SSV-25461 High latency due to ZFSin driver

include/os/windows/spl/sys/sysmacros.h

@@ -81,9 +81,9 @@ extern unsigned int num_ecores;
  * swap priority is at 92. Most ZFS priorities should probably
  * stay below this, but kmem_reap needs to be higher.
  */
-#define	minclsyspri  81 /* BASEPRI_KERNEL */
-#define	defclsyspri  81 /* BASEPRI_KERNEL */
-#define	maxclsyspri  89
+#define	minclsyspri  8 /* BASEPRI_KERNEL */
+#define	defclsyspri  8 /* BASEPRI_KERNEL */
+#define	maxclsyspri  12
 
 #define	NICE_TO_PRIO(nice)		(MAX_RT_PRIO + (nice) + 20)
 #define	PRIO_TO_NICE(prio)		((prio) - MAX_RT_PRIO - 20)

include/os/windows/zfs/sys/kstat_windows.h

@@ -88,6 +88,7 @@ typedef struct windows_kstat {
 	kstat_named_t spa_mode_global;
 	kstat_named_t zfs_flags;
 	kstat_named_t zfs_txg_timeout;
+	kstat_named_t zfs_adc_enable;
 	kstat_named_t zfs_vdev_cache_max;
 	kstat_named_t zfs_vdev_cache_size;
 	kstat_named_t zfs_vdev_cache_bshift;
@@ -150,6 +151,8 @@ typedef struct windows_kstat {
 	kstat_named_t zfs_removal_suspend_progress;
 	kstat_named_t cpu_avx_supported;
 	kstat_named_t zvol_io_threads;
+	kstat_named_t zfs_prealloc_percent;
+	kstat_named_t zfs_adc_target_sync_pct;
 } windows_kstat_t;
 
 
@@ -261,12 +264,15 @@ extern int zfs_autoimport_disable;
 extern int zfs_removal_suspend_progress;
 extern int cpu_avx_supported;
 extern int zvol_threads;
+extern int zfs_prealloc_percent;
+extern uint_t zfs_adc_target_sync_pct;
 
 int  kstat_windows_init(void *);
 void kstat_windows_fini(void);
 
 int arc_kstat_update(kstat_t *ksp, int rw);
 int arc_kstat_update_windows(kstat_t *ksp, int rw);
 int spl_kstat_registry(void *pRegistryPath, kstat_t *ksp);
+int dynamic_dirty_data_kstat_update(kstat_t* ksp, int rw);
 
 #endif

include/sys/arc.h

@@ -78,6 +78,7 @@ typedef void arc_prune_func_t(int64_t bytes, void *priv);
 
 /* Shared module parameters */
 extern int zfs_arc_average_blocksize;
+extern uint64_t dirty_ceil_bytes;
 
 /* generic arc_done_func_t's which you can use */
 arc_read_done_func_t arc_bcopy_func;

include/sys/txg.h

@@ -139,7 +139,15 @@ extern void *txg_list_next(txg_list_t *tl, void *p, uint64_t txg);
 
 /* Global tuning */
 extern int zfs_txg_timeout;
-
+extern int zfs_adc_enable;
+extern uint_t zfs_adc_target_sync_pct;
+
+typedef struct dynamic_dirty_data_stats {
+    kstat_named_t adc_target;
+    kstat_named_t spa_sync_time;
+    kstat_named_t data_flushed_per_sync;
+    kstat_named_t total_dirty_data;
+} dynamic_dirty_data_stats_t;
 
 #ifdef ZFS_DEBUG
 #define	TXG_VERIFY(spa, txg)		txg_verify(spa, txg)

module/os/windows/spl/spl-kmem.c

@@ -76,6 +76,7 @@ static volatile _Atomic int64_t spl_free = 0;
 int64_t spl_free_delta_ema;
 
 static boolean_t spl_event_thread_exit = FALSE;
+static boolean_t spl_abd_prealloc_thread_exit = FALSE;
 PKEVENT low_mem_event = NULL;
 
 static volatile _Atomic int64_t spl_free_manual_pressure = 0;
@@ -131,6 +132,10 @@ extern uint64_t		zfs_active_rwlock;
 extern uint64_t		total_memory;
 extern uint64_t		real_total_memory;
 
+extern kmem_cache_t *abd_chunk_cache;
+extern uint64_t zfs_arc_max;
+extern int zfs_prealloc_percent;
+
 #define	MULT 1
 
 static const char *KMEM_VA_PREFIX = "kmem_va";
@@ -4230,7 +4235,7 @@ spl_free_wrapper(void)
 int64_t
 spl_free_manual_pressure_wrapper(void)
 {
-	return (spl_free_manual_pressure);
+	return (0);
 }
 
 uint64_t
@@ -4479,6 +4484,7 @@ spl_free_thread()
 		    spl_vm_pressure_level != MAGIC_PRESSURE_UNAVAILABLE) {
 			/* there is pressure */
 			lowmem = true;
+			KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL, "spl_vm_pressure_level: %lu\n",spl_vm_pressure_level));
 			new_spl_free = -(2LL * PAGE_SIZE * spl_vm_pages_wanted);
 			if (spl_vm_pressure_level > 1) {
 				emergency_lowmem = true;
@@ -4533,6 +4539,7 @@ spl_free_thread()
 			int64_t old_pressure = spl_free_manual_pressure;
 			new_spl_free -= old_pressure * 2LL;
 			lowmem = true;
+			KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL, "spl_free_manual_pressure: %llu\n",spl_free_manual_pressure));
 			if (spl_free_fast_pressure) {
 				emergency_lowmem = true;
 				new_spl_free -= old_pressure * 4LL;
@@ -4630,6 +4637,7 @@ spl_free_thread()
 			new_spl_free += bminus;
 			lowmem = true;
 			emergency_lowmem = true;
+			KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL, "spl_vm_pages_wanted %lu\n", spl_vm_pages_wanted));
 			// atomic swaps to set these variables used in arc.c
 			int64_t previous_highest_pressure = 0;
 			int64_t new_p = -bminus;
@@ -4650,6 +4658,7 @@ spl_free_thread()
 			new_spl_free -= bytes_wanted;
 			if (reserve_low && !early_lots_free) {
 				lowmem = true;
+				KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL, "spl_vm_pages_wanted: %lu reserve_low: %lu early_lots_free: %lu\n", spl_vm_pages_wanted, reserve_low, early_lots_free));
 				if (recent_lowmem == 0) {
 					recent_lowmem = time_now;
 				}
@@ -4760,6 +4769,7 @@ spl_free_thread()
 			    real_total_memory) > 75) {
 				new_spl_free -= total_mem_used / 32;
 				lowmem = true;
+				KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL, "segkmem_total_mem_allocated: %llu real_total_memory: %llu\n", segkmem_total_mem_allocated, real_total_memory));
 			}
 		}
 
@@ -4914,6 +4924,53 @@ spl_event_thread(void *notused)
 	thread_exit();
 }
 
+static void
+spl_abd_prealloc_thread(void *notused)
+{
+	NTSTATUS Status;
+
+	typedef struct abd_prealloc_node {
+		list_node_t node;
+	} abd_prealloc_node_t;
+
+	abd_prealloc_node_t *node;
+	list_t abd_prealloc_list;
+
+	KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL, "SPL: abd prealloc start segkmem_total_mem_allocated: %lld total_memory: %lld\n",
+	    segkmem_total_mem_allocated, total_memory));
+
+	dprintf("SPL: beginning spl_abd_prealloc_thread() loop\n");
+
+	list_create(&abd_prealloc_list, sizeof (abd_prealloc_node_t), offsetof(abd_prealloc_node_t, node));
+
+	while (!spl_abd_prealloc_thread_exit) {
+
+		if (!abd_chunk_cache || !zfs_arc_max) {
+			delay(hz);
+			continue;
+		}
+
+		if (segkmem_total_mem_allocated >=
+                    (zfs_arc_max * zfs_prealloc_percent) / 100) {
+                        break;
+                }
+
+		node = (abd_prealloc_node_t *)kmem_cache_alloc(abd_chunk_cache, KM_SLEEP);
+		list_insert_tail(&abd_prealloc_list, node);
+	}
+
+	while ((node = list_remove_head(&abd_prealloc_list)) != NULL) {
+		kmem_cache_free(abd_chunk_cache, node);
+	}
+
+	spl_abd_prealloc_thread_exit = FALSE;
+	dprintf("SPL: %s thread_exit\n", __func__);
+
+	KdPrintEx((DPFLTR_IHVDRIVER_ID, DPFLTR_ERROR_LEVEL, "SPL: abd prealloc done segkmem_total_mem_allocated: %lld total_memory: %lld zfs_arc_max: %llu zfs_prealloc_percent: %d%\n",
+	    segkmem_total_mem_allocated, total_memory, zfs_arc_max, zfs_prealloc_percent));
+	thread_exit();
+}
+
 
 static int
 spl_kstat_update(kstat_t *ksp, int rw)
@@ -5342,15 +5399,21 @@ spl_kmem_thread_init(void)
 	(void) thread_create(NULL, 0, spl_free_thread, 0, 0, 0, 0, 92);
 	spl_free_thread_running = TRUE;
 
-	spl_event_thread_exit = FALSE;
-	(void) thread_create(NULL, 0, spl_event_thread, 0, 0, 0, 0, 92);
+	if (zfs_prealloc_percent) {
+		spl_abd_prealloc_thread_exit = FALSE;
+		(void) thread_create(NULL, 0, spl_abd_prealloc_thread, 0, 0, 0, 0, 92);
+	} else {
+		spl_event_thread_exit = FALSE;
+		(void) thread_create(NULL, 0, spl_event_thread, 0, 0, 0, 0, 92);
+	}
 }
 
 void
 spl_kmem_thread_fini(void)
 {
 	shutting_down = 1;
 
+	spl_abd_prealloc_thread_exit = TRUE;
 	if (low_mem_event != NULL) {
 		dprintf("SPL: stopping spl_event_thread\n");
 		spl_event_thread_exit = TRUE;

module/os/windows/spl/spl-thread.c

@@ -39,7 +39,78 @@
 
 uint64_t zfs_threads = 0;
 
-kthread_t *
+
+kthread_t*
+spl_thread_create(
+    caddr_t     stk,
+    size_t      stksize,
+    void        (*proc)(void*),
+    void* arg,
+    size_t      len,
+    int state,
+#ifdef SPL_DEBUG_THREAD
+    char* filename,
+    int line,
+#endif
+    pri_t       pri)
+{
+    NTSTATUS    status;
+    HANDLE      hThread = NULL;
+    PETHREAD    eThread = NULL;
+
+#ifdef SPL_DEBUG_THREAD
+    dprintf("Start thread pri %d\n", pri);
+#endif
+
+    status = PsCreateSystemThread(
+	&hThread,
+	THREAD_ALL_ACCESS,
+	NULL,
+	NULL,
+	NULL,
+	proc,
+	arg);
+
+    if (!NT_SUCCESS(status))
+	return NULL;
+
+    /* Convert HANDLE ETHREAD */
+    status = ObReferenceObjectByHandle(
+	hThread,
+	THREAD_ALL_ACCESS,
+	*PsThreadType,
+	KernelMode,
+	(PVOID*)&eThread,
+	NULL);
+
+    /* We no longer need the handle */
+    ZwClose(hThread);
+
+    if (!NT_SUCCESS(status))
+	return NULL;
+
+    /* Clamp priority to safe Windows range */
+    KPRIORITY newPri = (KPRIORITY)pri;
+
+    if (newPri > maxclsyspri)
+	newPri = maxclsyspri;
+
+    if (newPri < minclsyspri)
+	newPri = minclsyspri;
+
+    /* Set absolute priority */
+    KeSetPriorityThread((PKTHREAD)eThread, newPri);
+
+#ifdef SPL_DEBUG_THREAD
+    dprintf("Thread created with priority %d\n", newPri);
+#endif
+
+    atomic_inc_64(&zfs_threads);
+
+    return (kthread_t*)eThread;
+}
+
+/*kthread_t*
 spl_thread_create(
     caddr_t	stk,
     size_t	stksize,
@@ -72,12 +143,7 @@ spl_thread_create(
 	if (result != STATUS_SUCCESS)
 		return (NULL);
 
-	/*
-	 * Improve the priority when asked to do so
-	 * Thread priorities range from 0 to 31, where 0 is the lowest
-	 * priority and 31 is the highest
-	 */
-
+
 	if (pri > minclsyspri) {
 		// thread_precedence_policy_data_t policy;
 		// policy.importance = pri - minclsyspri;
@@ -102,7 +168,7 @@ spl_thread_create(
 	ObDereferenceObject(eThread);
 	ZwClose(thread);
 	return ((kthread_t *)eThread);
-}
+}*/
 
 kthread_t *
 spl_current_thread(void)

module/os/windows/spl/spl-vmem.c

@@ -448,6 +448,8 @@ uint64_t spl_frag_walk_cnt = 0;
 extern void spl_free_set_emergency_pressure(int64_t p);
 extern uint64_t segkmem_total_mem_allocated;
 extern uint64_t total_memory;
+extern uint64_t zfs_arc_max;
+extern int zfs_prealloc_percent;
 
 /*
  * Get a vmem_seg_t from the global segfree list.
@@ -1732,12 +1734,23 @@ vmem_xfree(vmem_t *vmp, void *vaddr, size_t size)
 		vsp = vprev;
 	}
 
+	// calling vm_source_free will free the memory to windows, we
+	// don't want to do this unless we are crossing the arc limit when
+	// zfs_prealloc_percent is enabled.
+	boolean_t allow_vm_source_free = true;
+	if (zfs_prealloc_percent) {
+		if (segkmem_total_mem_allocated <
+			(zfs_arc_max * 102) / 100) {
+			allow_vm_source_free = false;
+		}
+	}
+
 	/*
 	 * If the entire span is free, return it to the source.
 	 */
 	if (vsp->vs_aprev->vs_import && vmp->vm_source_free != NULL &&
 	    vsp->vs_aprev->vs_type == VMEM_SPAN &&
-	    vsp->vs_anext->vs_type == VMEM_SPAN) {
+	    vsp->vs_anext->vs_type == VMEM_SPAN && allow_vm_source_free) {
 		vaddr = (void *)vsp->vs_start;
 		size = VS_SIZE(vsp);
 		ASSERT(size == VS_SIZE(vsp->vs_aprev));