显存占用明显大于diffusers框架

[zengkaiya]

当使用Wan2.1-VACE-1.3B模型推理相同的视频时（81帧，480*832）时，diffsynth的显存占用（34G）显著高于diffusers框架推理（23G），请问可能是哪些原因导致的呢？感谢

[mi804]

请提供推理代码。如果显存不够，可以尝试开启低显存管理，参考：https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-1.3B.py

[zengkaiya]

请提供推理代码。如果显存不够，可以尝试开启低显存管理，参考：https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-1.3B.py

感谢回复，下面是推理代码，md格式有点调不正确，不好意思😭

diffsynth推理代码

`
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
import argparse
import pandas as pd
import torch
from diffsynth import save_video, VideoData
from diffsynth.models.utils import load_state_dict
from diffsynth.pipelines.wan_video_new import WanVideoPipeline, ModelConfig
import numpy as np
from PIL import Image
OUTPUT_DIR = "output/infer/vace_test_stage1_test_data"
csv_path = "data/video_text_data/stage1_test_data.csv"
vroot = ["data/video_text_data/stage1_test_data"]
num_frame = 81

print(OUTPUT_DIR)
os.makedirs(OUTPUT_DIR, exist_ok=True)
parser = argparse.ArgumentParser()
parser.add_argument("--start_index", type=int, default=0, help="Start row index")
parser.add_argument("--end_index", type=int, default=20, help="End row index (exclusive)")
args = parser.parse_args()

start_idx = args.start_index
end_idx = args.end_index
df = pd.read_csv(
csv_path,
skiprows=range(1, start_idx + 1), # 跳过前 start_idx 行（保留 header）
nrows=end_idx - start_idx
)
print(f"Loaded rows {start_idx} to {end_idx} → total {len(df)} rows")

pipe = WanVideoPipeline.from_pretrained(
torch_dtype=torch.bfloat16,
device="cuda",
model_configs=[
ModelConfig(path="models/Wan-AI/Wan2.1-VACE-1.3B/diffusion_pytorch_model.safetensors"),
ModelConfig(path="models/Wan-AI/Wan2.1-VACE-1.3B/models_t5_umt5-xxl-enc-bf16.pth"),
ModelConfig(path="models/Wan-AI/Wan2.1-VACE-1.3B/Wan2.1_VAE.pth")
],
)
pipe.to("cuda")

for idx, row in df.iterrows():
print(f"\n===========================")
print(f"▶ Processing row {idx}")
print("===========================\n")

vace_video_path = row["vace_video"]
mask_path = row["vace_video_mask"]
ref_image_path = row["vace_reference_image"]

control = VideoData(vace_video_path, height=480, width=832)
control = [control[i] for i in range(num_frame)]
mask = VideoData(mask_path, height=480, width=832)
mask = [mask[i] for i in range(num_frame)]
ref_image = Image.open(ref_image_path).convert("RGB")

# 现在只支持batch size=1
edited_frames = pipe(
    prompt=row["prompt"],
    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
    vace_video=control,
    vace_video_mask=mask,
    vace_reference_image=[ref_image],
    num_frames=num_frame,
    num_inference_steps=20,
    seed=1,
    tiled=True,
)
relative_video_path = row['video'].replace('/clip1', '').replace('/clip2', '')
for v in vroot:
    relative_video_path = relative_video_path.replace(v, '')

out_path = os.path.join(OUTPUT_DIR, relative_video_path.replace('/', '-'))
save_video(edited_frames, out_path, fps=16, quality=5)
print(f"✔ Saved: {out_path}")

print("\nAll tasks finished!\n")
`

diffusers框架推理代码

`
import os
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
import torch
from tqdm import tqdm
import argparse
import pandas as pd
from PIL import Image
from diffusers import AutoencoderKLWan, WanVACEPipeline
from diffusers.utils import export_to_video
from diffusers.schedulers import FlowMatchEulerDiscreteScheduler, UniPCMultistepScheduler
from dataset.vace.video import VideoData

csv_path = "data/video_text_data/stage1_test_data.csv"
vroot = ["data/video_text_data/stage1_test_data"]
output_dir = "output/infer/testdiffusers_float32"
num_frame = 81
print(output_dir)
os.makedirs(output_dir, exist_ok=True)

parser = argparse.ArgumentParser()
parser.add_argument("--start_index", type=int, default=0, help="Start row index")
parser.add_argument("--end_index", type=int, default=20, help="End row index (exclusive)")
args = parser.parse_args()

start_idx = args.start_index
end_idx = args.end_index
df = pd.read_csv(
csv_path,
skiprows=range(1, start_idx + 1), # 跳过前 start_idx 行（保留 header）
nrows=end_idx - start_idx
)
print(f"Loaded rows {start_idx} to {end_idx} → total {len(df)} rows")

model_id = "models/Wan-AI/Wan2.1-VACE-1.3B-diffusers"
vae = AutoencoderKLWan.from_pretrained(model_id, subfolder="vae", torch_dtype=torch.float32,)
pipe = WanVACEPipeline.from_pretrained(model_id, vae=vae, torch_dtype=torch.bfloat16,)
pipe.scheduler = FlowMatchEulerDiscreteScheduler(shift=5.0,)
pipe.to("cuda")

for idx, row in df.iterrows():
print(f"\n===========================")
print(f"▶ Processing row {idx}")
print("===========================\n")

vace_video_path = row["vace_video"]
mask_path = row["vace_video_mask"]
ref_image_path = row["vace_reference_image"]

vace_video = VideoData(vace_video_path, height=480, width=832)
vace_video = [vace_video[i] for i in range(num_frame)]
mask = VideoData(mask_path, height=480, width=832)
mask = [mask[i] for i in range(num_frame)]
ref_image = Image.open(ref_image_path).convert("RGB")

output = pipe(
    video=[vace_video],
    mask=[mask],
    reference_images=[[ref_image]],
    prompt=row["prompt"],
    negative_prompt="色调艳丽，过曝，静态，细节模糊不清，字幕，风格，作品，画作，画面，静止，整体发灰，最差质量，低质量，JPEG压缩残留，丑陋的，残缺的，多余的手指，画得不好的手部，画得不好的脸部，畸形的，毁容的，形态畸形的肢体，手指融合，静止不动的画面，杂乱的背景，三条腿，背景人很多，倒着走",
    height=480,
    width=832,
    num_frames=num_frame,
    num_inference_steps=20,
    guidance_scale=5.0,
    generator=torch.Generator().manual_seed(1),
).frames[0]

relative_video_path = row['video'].replace('/clip1', '').replace('/clip2', '')
for v in vroot:
    relative_video_path = relative_video_path.replace(v, '')
out_path = os.path.join(output_dir, relative_video_path.replace('/', '-'))
export_to_video(output, out_path, fps=16)
print(f"Video saved to {out_path}")

`

[zengkaiya]

请提供推理代码。如果显存不够，可以尝试开启低显存管理，参考：https://github.com/modelscope/DiffSynth-Studio/blob/main/examples/wanvideo/model_inference_low_vram/Wan2.1-VACE-1.3B.py

确实可以用offload等策略，但在训练时相比diffusers框架就慢了很多，所以想请教下有没有啥降低其他显存占用的办法/代码哪里有问题🌹

[mi804]

推理可以用将最大显存限制在vram_limit=20 GB内：

vram_config = {
    "offload_dtype": "cpu",
    "offload_device": "cpu",
    "onload_dtype": torch.bfloat16,
    "onload_device": "cpu",
    "preparing_dtype": torch.bfloat16,
    "preparing_device": "cuda",
    "computation_dtype": torch.bfloat16,
    "computation_device": "cuda",
}
vram_limit = 20 # GB
pipe = WanVideoPipeline.from_pretrained(
    torch_dtype=torch.bfloat16,
    device="cuda",
    model_configs=[
        ModelConfig(model_id="Wan-AI/Wan2.1-VACE-1.3B", origin_file_pattern="diffusion_pytorch_model*.safetensors", **vram_config),
        ModelConfig(model_id="Wan-AI/Wan2.1-VACE-1.3B", origin_file_pattern="models_t5_umt5-xxl-enc-bf16.pth", **vram_config),
        ModelConfig(model_id="Wan-AI/Wan2.1-VACE-1.3B", origin_file_pattern="Wan2.1_VAE.pth", **vram_config),
    ],
    tokenizer_config=ModelConfig(model_id="Wan-AI/Wan2.1-T2V-1.3B", origin_file_pattern="google/umt5-xxl/"),
    vram_limit=vram_limit - 2,
)

训练可以根据你的显存灵活配置use_gradient_checkpointing或者use_gradient_checkpointing_offload，参考docs/zh/Pipeline_Usage/Model_Training.md， 如果有多卡，可以使用deepspeed 如zero2。 还可以使用分步训练，将vae的encode与dit的训练分开，参考docs/zh/Training/Split_Training.md

[zengkaiya]

非常感谢～ diffusers之前23G显存，推理大概1min3s；diffsynth修改前34G，推理大概53s，修改后18G，推理1min，好于diffusers👍

[zengkaiya]

想再请教下，vram_limit适合用在训练场景吗，刚和accelerate结合训了下发现会有数据和参数不在一个device上的bug：RuntimeError: Input type (CUDABFloat16Type) and weight type (CPUBFloat16Type) should be the same

[mi804]

diffsynth的vram管理仅针对推理。vram_limit的目标是：限制推理时同时onload到GPU的模型权重总量，从而使总显存消耗始终位于vram_limit GB之下。

训练时，所有参数都应该在GPU上，频繁offload和onload会极大降低训练速度。想降低训练显存请考虑上面提到的方法。

[zengkaiya]

了解了，多谢解答🙏