fix controlnet offload

Author: Glaceon-Hyy

diffsynth_engine/pipelines/flux_image.py

@@ -921,8 +921,8 @@ def predict_multicontrolnet(
                 text_ids,
             )
             if self.offload_mode is not None:
-                empty_cache()
                 param.model.to("cpu")
+                empty_cache()
             double_block_output_results = accumulate(double_block_output_results, double_block_output)
             single_block_output_results = accumulate(single_block_output_results, single_block_output)
         return double_block_output_results, single_block_output_results

diffsynth_engine/pipelines/sd_image.py

@@ -310,8 +310,8 @@ def predict_multicontrolnet(
             )
             controlnet_res = [res * current_scale for res in controlnet_res]
             if self.offload_mode is not None:
+                param.model.to("cpu")        
                 empty_cache()
-                param.model.to("cpu")            
             controlnet_res_stack = accumulate(controlnet_res_stack, controlnet_res)
         return controlnet_res_stack
 

diffsynth_engine/pipelines/sdxl_image.py

@@ -335,8 +335,8 @@ def predict_multicontrolnet(
             )
             controlnet_res = [res * current_scale for res in controlnet_res]
             if self.offload_mode is not None:
-                empty_cache()
-                param.model.to("cpu")            
+                param.model.to("cpu") 
+                empty_cache()           
             controlnet_res_stack = accumulate(controlnet_res_stack, controlnet_res)
         return controlnet_res_stack