DDT: Move logs searches out of the lock

Postponing entry removal from the DDT log in case of hit till later
single-threaded sync stage allows to make ddl_tree stable during
multi-threaded ZIO processing stage.  It allows to drop the DDT lock
before the search instead of after, reducing the contention a lot.

Actually ddt_log_update_entry() was already handling the case of
entry present in the active log, so we only need to remove it from
flushing log, if the entry happen to be there.

My tests with parallel 4KB block writes show throughput increase
from 480MB/s (122K blocks/s) to 827MB/s (212K blocks/s), even
though still limited by the global DDT lock contention.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Alexander Motin <alexander.motin@TrueNAS.com>
Closes #18044

include/sys/ddt.h

@@ -213,6 +213,7 @@ typedef enum {
 #define	DDE_FLAG_LOADED		(1 << 0)	/* entry ready for use */
 #define	DDE_FLAG_OVERQUOTA	(1 << 1)	/* entry unusable, no space */
 #define	DDE_FLAG_LOGGED		(1 << 2)	/* loaded from log */
+#define	DDE_FLAG_FROM_FLUSHING	(1 << 3)	/* loaded from flushing log */
 
 /*
  * Additional data to support entry update or repair. This is fixed size
@@ -280,13 +281,14 @@ typedef struct {
  */
 typedef struct {
 	kmutex_t	ddt_lock;	/* protects changes to all fields */
-
 	avl_tree_t	ddt_tree;	/* "live" (changed) entries this txg */
-	avl_tree_t	ddt_log_tree;	/* logged entries */
-
 	avl_tree_t	ddt_repair_tree;	/* entries being repaired */
 
-	ddt_log_t	ddt_log[2];		/* active/flushing logs */
+	/*
+	 * Log trees are stable during I/O, and only modified during sync
+	 * with exclusive access.
+	 */
+	ddt_log_t	ddt_log[2] ____cacheline_aligned; /* logged entries */
 	ddt_log_t	*ddt_log_active;	/* pointers into ddt_log */
 	ddt_log_t	*ddt_log_flushing;	/* swapped when flush starts */
 

include/sys/ddt_impl.h

@@ -69,8 +69,8 @@ extern "C" {
  * the live tree.
  */
 typedef struct {
-	ddt_key_t	ddle_key;	/* ddt_log_tree key */
-	avl_node_t	ddle_node;	/* ddt_log_tree node */
+	ddt_key_t	ddle_key;	/* ddl_tree key */
+	avl_node_t	ddle_node;	/* ddl_tree node */
 
 	ddt_type_t	ddle_type;	/* storage type */
 	ddt_class_t	ddle_class;	/* storage class */
@@ -193,7 +193,7 @@ extern boolean_t ddt_log_take_first(ddt_t *ddt, ddt_log_t *ddl,
     ddt_lightweight_entry_t *ddlwe);
 
 extern boolean_t ddt_log_find_key(ddt_t *ddt, const ddt_key_t *ddk,
-    ddt_lightweight_entry_t *ddlwe);
+    ddt_lightweight_entry_t *ddlwe, boolean_t *from_flushing);
 extern boolean_t ddt_log_remove_key(ddt_t *ddt, ddt_log_t *ddl,
     const ddt_key_t *ddk);