From b454751f8f02f395e9710bb1c34774dcdf65bb20 Mon Sep 17 00:00:00 2001 From: "Zheng Zangwei (Alex Zheng)" Date: Mon, 25 Mar 2024 13:19:11 +0800 Subject: [PATCH] Docs/fix (#211) * fix #210 * fix #209 --- configs/opensora/train/16x512x512.py | 2 +- configs/opensora/train/360x512x512.py | 2 +- configs/opensora/train/64x512x512-sp.py | 2 +- configs/opensora/train/64x512x512.py | 2 +- configs/pixart/inference/16x256x256.py | 2 +- configs/pixart/inference/1x1024MS.py | 2 +- configs/pixart/inference/1x256x256.py | 2 +- configs/pixart/inference/1x512x512.py | 2 +- configs/pixart/train/16x256x256.py | 2 +- configs/pixart/train/1x512x512.py | 2 +- configs/pixart/train/64x512x512.py | 2 +- docs/report_v1.md | 2 +- docs/structure.md | 4 ++-- docs/zh_CN/structure.md | 4 ++-- 14 files changed, 16 insertions(+), 16 deletions(-) diff --git a/configs/opensora/train/16x512x512.py b/configs/opensora/train/16x512x512.py index 885aad1..22031dc 100644 --- a/configs/opensora/train/16x512x512.py +++ b/configs/opensora/train/16x512x512.py @@ -30,7 +30,7 @@ vae = dict( ) text_encoder = dict( type="t5", - from_pretrained="./pretrained_models/t5_ckpts", + from_pretrained="DeepFloyd/t5-v1_1-xxl", model_max_length=120, shardformer=True, ) diff --git a/configs/opensora/train/360x512x512.py b/configs/opensora/train/360x512x512.py index 7a6f759..0f4e170 100644 --- a/configs/opensora/train/360x512x512.py +++ b/configs/opensora/train/360x512x512.py @@ -31,7 +31,7 @@ vae = dict( ) text_encoder = dict( type="t5", - from_pretrained="./pretrained_models/t5_ckpts", + from_pretrained="DeepFloyd/t5-v1_1-xxl", model_max_length=120, shardformer=True, ) diff --git a/configs/opensora/train/64x512x512-sp.py b/configs/opensora/train/64x512x512-sp.py index b0b9062..f9b138e 100644 --- a/configs/opensora/train/64x512x512-sp.py +++ b/configs/opensora/train/64x512x512-sp.py @@ -30,7 +30,7 @@ vae = dict( ) text_encoder = dict( type="t5", - from_pretrained="./pretrained_models/t5_ckpts", + from_pretrained="DeepFloyd/t5-v1_1-xxl", model_max_length=120, shardformer=True, ) diff --git a/configs/opensora/train/64x512x512.py b/configs/opensora/train/64x512x512.py index dfcdcc0..81162de 100644 --- a/configs/opensora/train/64x512x512.py +++ b/configs/opensora/train/64x512x512.py @@ -30,7 +30,7 @@ vae = dict( ) text_encoder = dict( type="t5", - from_pretrained="./pretrained_models/t5_ckpts", + from_pretrained="DeepFloyd/t5-v1_1-xxl", model_max_length=120, shardformer=True, ) diff --git a/configs/pixart/inference/16x256x256.py b/configs/pixart/inference/16x256x256.py index 6fc8ee6..fed26f5 100644 --- a/configs/pixart/inference/16x256x256.py +++ b/configs/pixart/inference/16x256x256.py @@ -15,7 +15,7 @@ vae = dict( ) text_encoder = dict( type="t5", - from_pretrained="./pretrained_models/t5_ckpts", + from_pretrained="DeepFloyd/t5-v1_1-xxl", model_max_length=120, ) scheduler = dict( diff --git a/configs/pixart/inference/1x1024MS.py b/configs/pixart/inference/1x1024MS.py index 41cc97a..3b34ebf 100644 --- a/configs/pixart/inference/1x1024MS.py +++ b/configs/pixart/inference/1x1024MS.py @@ -17,7 +17,7 @@ vae = dict( ) text_encoder = dict( type="t5", - from_pretrained="./pretrained_models/t5_ckpts", + from_pretrained="DeepFloyd/t5-v1_1-xxl", model_max_length=120, ) scheduler = dict( diff --git a/configs/pixart/inference/1x256x256.py b/configs/pixart/inference/1x256x256.py index 11e06d7..44ce10b 100644 --- a/configs/pixart/inference/1x256x256.py +++ b/configs/pixart/inference/1x256x256.py @@ -16,7 +16,7 @@ vae = dict( ) text_encoder = dict( type="t5", - from_pretrained="./pretrained_models/t5_ckpts", + from_pretrained="DeepFloyd/t5-v1_1-xxl", model_max_length=120, ) scheduler = dict( diff --git a/configs/pixart/inference/1x512x512.py b/configs/pixart/inference/1x512x512.py index 5674259..243af4a 100644 --- a/configs/pixart/inference/1x512x512.py +++ b/configs/pixart/inference/1x512x512.py @@ -16,7 +16,7 @@ vae = dict( ) text_encoder = dict( type="t5", - from_pretrained="./pretrained_models/t5_ckpts", + from_pretrained="DeepFloyd/t5-v1_1-xxl", model_max_length=120, ) scheduler = dict( diff --git a/configs/pixart/train/16x256x256.py b/configs/pixart/train/16x256x256.py index b47731e..dc91937 100644 --- a/configs/pixart/train/16x256x256.py +++ b/configs/pixart/train/16x256x256.py @@ -29,7 +29,7 @@ vae = dict( ) text_encoder = dict( type="t5", - from_pretrained="./pretrained_models/t5_ckpts", + from_pretrained="DeepFloyd/t5-v1_1-xxl", model_max_length=120, shardformer=True, ) diff --git a/configs/pixart/train/1x512x512.py b/configs/pixart/train/1x512x512.py index 619c9aa..ec73b8c 100644 --- a/configs/pixart/train/1x512x512.py +++ b/configs/pixart/train/1x512x512.py @@ -30,7 +30,7 @@ vae = dict( ) text_encoder = dict( type="t5", - from_pretrained="./pretrained_models/t5_ckpts", + from_pretrained="DeepFloyd/t5-v1_1-xxl", model_max_length=120, shardformer=True, ) diff --git a/configs/pixart/train/64x512x512.py b/configs/pixart/train/64x512x512.py index 628cf25..deebdf6 100644 --- a/configs/pixart/train/64x512x512.py +++ b/configs/pixart/train/64x512x512.py @@ -30,7 +30,7 @@ vae = dict( ) text_encoder = dict( type="t5", - from_pretrained="./pretrained_models/t5_ckpts", + from_pretrained="DeepFloyd/t5-v1_1-xxl", model_max_length=120, shardformer=True, ) diff --git a/docs/report_v1.md b/docs/report_v1.md index b3b8073..edb3644 100644 --- a/docs/report_v1.md +++ b/docs/report_v1.md @@ -14,7 +14,7 @@ As shown in the figure, we insert a temporal attention right after each spatial To focus on video generation, we hope to train the model based on a powerful image generation model. [PixArt-α](https://github.com/PixArt-alpha/PixArt-alpha) is an efficiently trained high-quality image generation model with T5-conditioned DiT structure. We initialize our model with PixArt-α and initialize the projection layer of inserted temporal attention with zero. This initialization preserves model's ability of image generation at beginning, while Latte's architecture cannot. The inserted attention increases the number of parameter from 580M to 724M. -![Architecture](https://i0.imgs.ovh/2024/03/16/erC1d.png) +![Architecture](https://image.jiqizhixin.com/uploads/editor/ff49eaba-6b19-43d7-b65d-ad2ecdb9d555/640.jpeg) Drawing from the success of PixArt-α and Stable Video Diffusion, we also adopt a progressive training strategy: 16x256x256 on 366K pretraining datasets, and then 16x256x256, 16x512x512, and 64x512x512 on 20K datasets. With scaled position embedding, this strategy greatly reduces the computational cost. diff --git a/docs/structure.md b/docs/structure.md index 2d83620..3eb7c69 100644 --- a/docs/structure.md +++ b/docs/structure.md @@ -100,7 +100,7 @@ vae = dict( ) text_encoder = dict( type="t5", # Select text encoder type (t5, clip) - from_pretrained="./pretrained_models/t5_ckpts", # Load from pretrained text encoder + from_pretrained="DeepFloyd/t5-v1_1-xxl", # Load from pretrained text encoder model_max_length=120, # Maximum length of input text ) scheduler = dict( @@ -153,7 +153,7 @@ vae = dict( ) text_encoder = dict( type="t5", - from_pretrained="./pretrained_models/t5_ckpts", + from_pretrained="DeepFloyd/t5-v1_1-xxl", model_max_length=120, shardformer=True, # Enable shardformer for T5 acceleration ) diff --git a/docs/zh_CN/structure.md b/docs/zh_CN/structure.md index 2d83620..3eb7c69 100644 --- a/docs/zh_CN/structure.md +++ b/docs/zh_CN/structure.md @@ -100,7 +100,7 @@ vae = dict( ) text_encoder = dict( type="t5", # Select text encoder type (t5, clip) - from_pretrained="./pretrained_models/t5_ckpts", # Load from pretrained text encoder + from_pretrained="DeepFloyd/t5-v1_1-xxl", # Load from pretrained text encoder model_max_length=120, # Maximum length of input text ) scheduler = dict( @@ -153,7 +153,7 @@ vae = dict( ) text_encoder = dict( type="t5", - from_pretrained="./pretrained_models/t5_ckpts", + from_pretrained="DeepFloyd/t5-v1_1-xxl", model_max_length=120, shardformer=True, # Enable shardformer for T5 acceleration )