Implement allocation size ranges and use for gang leaves (#17111)

When forced to resort to ganging, ZFS currently allocates three child
blocks, each one third of the size of the original. This is true
regardless of whether larger allocations could be made, which would
allow us to have fewer gang leaves. This improves performance when
fragmentation is high enough to require ganging, but not so high that
all the free ranges are only just big enough to hold a third of the
recordsize. This is also useful for improving the behavior of a future
change to allow larger gang headers.

We add the ability for the allocation codepath to allocate a range of
sizes instead of a single fixed size. We then use this to pre-allocate
the DVAs for the gang children. If those allocations fail, we fall back
to the normal write path, which will likely re-gang.

Signed-off-by: Paul Dagnelie <paul.dagnelie@klarasystems.com>
Co-authored-by: Paul Dagnelie <paul.dagnelie@klarasystems.com>
Reviewed-by: Alexander Motin <mav@FreeBSD.org>
Reviewed-by: Tony Hutter <hutter2@llnl.gov>

include/sys/metaslab.h

@@ -41,7 +41,7 @@ extern "C" {
 
 typedef struct metaslab_ops {
 	const char *msop_name;
-	uint64_t (*msop_alloc)(metaslab_t *, uint64_t);
+	uint64_t (*msop_alloc)(metaslab_t *, uint64_t, uint64_t, uint64_t *);
 } metaslab_ops_t;
 
 
@@ -82,6 +82,9 @@ uint64_t metaslab_largest_allocatable(metaslab_t *);
 
 int metaslab_alloc(spa_t *, metaslab_class_t *, uint64_t, blkptr_t *, int,
     uint64_t, blkptr_t *, int, zio_alloc_list_t *, int, const void *);
+int metaslab_alloc_range(spa_t *, metaslab_class_t *, uint64_t, uint64_t,
+    blkptr_t *, int, uint64_t, blkptr_t *, int, zio_alloc_list_t *,
+    int, const void *, uint64_t *);
 int metaslab_alloc_dva(spa_t *, metaslab_class_t *, uint64_t,
     dva_t *, int, dva_t *, uint64_t, int, zio_alloc_list_t *, int);
 void metaslab_free(spa_t *, const blkptr_t *, uint64_t, boolean_t);
@@ -95,6 +98,7 @@ void metaslab_check_free(spa_t *, const blkptr_t *);
 
 void metaslab_stat_init(void);
 void metaslab_stat_fini(void);
+void metaslab_trace_move(zio_alloc_list_t *, zio_alloc_list_t *);
 void metaslab_trace_init(zio_alloc_list_t *);
 void metaslab_trace_fini(zio_alloc_list_t *);
 
@@ -127,6 +131,8 @@ uint64_t metaslab_group_get_space(metaslab_group_t *);
 void metaslab_group_histogram_verify(metaslab_group_t *);
 uint64_t metaslab_group_fragmentation(metaslab_group_t *);
 void metaslab_group_histogram_remove(metaslab_group_t *, metaslab_t *);
+void metaslab_group_alloc_increment_all(spa_t *, blkptr_t *, int, int,
+    uint64_t, const void *);
 void metaslab_group_alloc_decrement(spa_t *, uint64_t, int, int, uint64_t,
     const void *);
 void metaslab_recalculate_weight_and_sort(metaslab_t *);

include/sys/vdev.h

@@ -134,6 +134,8 @@ extern void vdev_space_update(vdev_t *vd,
 
 extern int64_t vdev_deflated_space(vdev_t *vd, int64_t space);
 
+extern uint64_t vdev_asize_to_psize_txg(vdev_t *vd, uint64_t asize,
+    uint64_t txg);
 extern uint64_t vdev_psize_to_asize_txg(vdev_t *vd, uint64_t psize,
     uint64_t txg);
 extern uint64_t vdev_psize_to_asize(vdev_t *vd, uint64_t psize);

include/sys/vdev_draid.h

@@ -95,7 +95,7 @@ extern int vdev_draid_generate_perms(const draid_map_t *, uint8_t **);
  */
 extern boolean_t vdev_draid_readable(vdev_t *, uint64_t);
 extern boolean_t vdev_draid_missing(vdev_t *, uint64_t, uint64_t, uint64_t);
-extern uint64_t vdev_draid_asize_to_psize(vdev_t *, uint64_t);
+extern uint64_t vdev_draid_asize_to_psize(vdev_t *, uint64_t, uint64_t);
 extern void vdev_draid_map_alloc_empty(zio_t *, struct raidz_row *);
 extern int vdev_draid_map_verify_empty(zio_t *, struct raidz_row *);
 extern nvlist_t *vdev_draid_read_config_spare(vdev_t *);

include/sys/vdev_impl.h

@@ -103,7 +103,8 @@ typedef const struct vdev_ops {
 	vdev_fini_func_t		*vdev_op_fini;
 	vdev_open_func_t		*vdev_op_open;
 	vdev_close_func_t		*vdev_op_close;
-	vdev_asize_func_t		*vdev_op_asize;
+	vdev_asize_func_t		*vdev_op_psize_to_asize;
+	vdev_asize_func_t		*vdev_op_asize_to_psize;
 	vdev_min_asize_func_t		*vdev_op_min_asize;
 	vdev_min_alloc_func_t		*vdev_op_min_alloc;
 	vdev_io_start_func_t		*vdev_op_io_start;
@@ -615,6 +616,7 @@ extern vdev_ops_t vdev_indirect_ops;
  */
 extern void vdev_default_xlate(vdev_t *vd, const zfs_range_seg64_t *logical_rs,
     zfs_range_seg64_t *physical_rs, zfs_range_seg64_t *remain_rs);
+extern uint64_t vdev_default_psize(vdev_t *vd, uint64_t asize, uint64_t txg);
 extern uint64_t vdev_default_asize(vdev_t *vd, uint64_t psize, uint64_t txg);
 extern uint64_t vdev_default_min_asize(vdev_t *vd);
 extern uint64_t vdev_get_min_asize(vdev_t *vd);

include/sys/zio.h

@@ -227,6 +227,7 @@ typedef uint64_t zio_flag_t;
 #define	ZIO_FLAG_REEXECUTED	(1ULL << 30)
 #define	ZIO_FLAG_DELEGATED	(1ULL << 31)
 #define	ZIO_FLAG_DIO_CHKSUM_ERR	(1ULL << 32)
+#define	ZIO_FLAG_PREALLOCATED	(1ULL << 33)
 
 #define	ZIO_ALLOCATOR_NONE	(-1)
 #define	ZIO_HAS_ALLOCATOR(zio)	((zio)->io_allocator != ZIO_ALLOCATOR_NONE)