update installation

configs/opensora-v1-2/train/stage1.py

@@ -4,7 +4,7 @@ dataset = dict(
     transform_name="resize_crop",
 )
 
-# h800
+# backup
 # bucket_config = {  # 20s/it
 #     "144p": {1: (1.0, 100), 51: (1.0, 30), 102: (1.0, 20), 204: (1.0, 8), 408: (1.0, 4)},
 #     # ---

configs/opensora-v1-2/train/stage2.py

@@ -0,0 +1,90 @@
+# Dataset settings
+dataset = dict(
+    type="VariableVideoTextDataset",
+    transform_name="resize_crop",
+)
+
+# webvid
+bucket_config = {  # 12s/it
+    "144p": {1: (1.0, 475), 51: (1.0, 51), 102: ((1.0, 0.33), 27), 204: ((1.0, 0.1), 13), 408: ((1.0, 0.1), 6)},
+    # ---
+    "256": {1: (0.4, 297), 51: (0.5, 20), 102: ((0.5, 0.33), 10), 204: ((0.5, 0.1), 5), 408: ((0.5, 0.1), 2)},
+    "240p": {1: (0.3, 297), 51: (0.4, 20), 102: ((0.4, 0.33), 10), 204: ((0.4, 0.1), 5), 408: ((0.4, 0.1), 2)},
+    # ---
+    "360p": {1: (0.2, 141), 51: (0.15, 8), 102: ((0.15, 0.33), 4), 204: ((0.15, 0.1), 2), 408: ((0.15, 0.1), 1)},
+    "512": {1: (0.2, 141), 51: (0.15, 8), 102: ((0.15, 0.33), 4), 204: ((0.15, 0.1), 2), 408: ((0.15, 0.1), 1)},
+    # ---
+    "480p": {1: (0.1, 89), 51: (0.1, 5), 102: (0.1, 2), 204: (0.1, 1)},
+    # ---
+    "720p": {1: (0.05, 36), 51: (0.1, 1)},
+    "1024": {1: (0.05, 36), 51: (0.1, 1)},
+    # ---
+    "1080p": {1: (0.1, 5)},
+    # ---
+    "2048": {1: (0.1, 5)},
+}
+
+grad_checkpoint = True
+
+# Acceleration settings
+num_workers = 8
+num_bucket_build_workers = 16
+dtype = "bf16"
+plugin = "zero2"
+
+# Model settings
+model = dict(
+    type="STDiT3-XL/2",
+    from_pretrained=None,
+    qk_norm=True,
+    enable_flash_attn=True,
+    enable_layernorm_kernel=True,
+    freeze_y_embedder=True,
+)
+vae = dict(
+    type="OpenSoraVAE_V1_2",
+    from_pretrained="/mnt/jfs/sora_checkpoints/vae-pipeline",
+    micro_frame_size=17,
+    micro_batch_size=4,
+)
+text_encoder = dict(
+    type="t5",
+    from_pretrained="DeepFloyd/t5-v1_1-xxl",
+    model_max_length=300,
+    shardformer=True,
+    local_files_only=True,
+)
+scheduler = dict(
+    type="rflow",
+    use_timestep_transform=True,
+    sample_method="logit-normal",
+)
+
+# Mask settings
+mask_ratios = {
+    "random": 0.05,
+    "intepolate": 0.005,
+    "quarter_random": 0.005,
+    "quarter_head": 0.005,
+    "quarter_tail": 0.005,
+    "quarter_head_tail": 0.005,
+    "image_random": 0.025,
+    "image_head": 0.05,
+    "image_tail": 0.025,
+    "image_head_tail": 0.025,
+}
+
+# Log settings
+seed = 42
+outputs = "outputs"
+wandb = False
+epochs = 1000
+log_every = 10
+ckpt_every = 200
+
+# optimization settings
+load = None
+grad_clip = 1.0
+lr = 1e-4
+ema_decay = 0.99
+adam_eps = 1e-15

docs/installation.md

@@ -26,6 +26,7 @@ pip install -r requirements/requirements-cu121.txt
 ```
 
 If you are using different CUDA versions, you need to manually install `torch`, `torchvision` and `xformers`. You can find the compatible distributions according to the links below.
+
 - PyTorch: choose install commands from [PyTorch installation page](https://pytorch.org/get-started/locally/) based on your own CUDA version.
 - xformers: choose install commands from [xformers repo](https://github.com/facebookresearch/xformers?tab=readme-ov-file#installing-xformers) based on your own CUDA version.
 
@@ -57,12 +58,10 @@ pip install flash-attn --no-build-isolation
 pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation --config-settings "--build-option=--cpp_ext" --config-settings "--build-option=--cuda_ext" git+https://github.com/NVIDIA/apex.git
 ```
 
-
-
 ## Evaluation
 
+### Step 1: Install Requirements
 
-## Step 1: Install Requirements
 To conduct evaluation, run the following command to install requirements:
 
 ```bash
@@ -70,7 +69,8 @@ pip install -v .[eval]
 # For development:`pip install -v -e .[eval]`
 ```
 
-## Step 2: Install VBench
+### Step 2: Install VBench
+
 You need to manually install [VBench](https://github.com/Vchitect/VBench):
 
 ```bash
@@ -79,7 +79,7 @@ pip install --no-deps vbench==0.1.1
 export PATH="/path/to/vbench:$PATH"
 ```
 
-#### Step 3: Install `cupy` for Potential VAE Errors
+### Step 3: Install `cupy` for Potential VAE Errors
 
 You need to mannually install [cupy](https://docs.cupy.dev/en/stable/install.html).
 
@@ -95,10 +95,8 @@ import torchvision.transforms.functional_tensor as F_t
 import torchvision.transforms._functional_tensor as F_t
 ```
 
-
 ## Data Processing
 
-
 ### Step 1: Install Requirements
 
 First, run the following command to install requirements:

requirements/requirements-eval.txt

@@ -3,9 +3,9 @@ detectron2 @ git+https://github.com/facebookresearch/detectron2.git@ff53992
 imageio>=2.34.1
 pyiqa==0.1.10
 scikit-learn>=1.4.2
-scikit-image==0.23.2
+scikit-image>=0.20.0
 lvis==0.5.3
-boto3==1.34.113
+boto3>=1.34.113
 
 # [vae]
 decord==0.6.0

requirements/requirements.txt

@@ -9,7 +9,7 @@ accelerate==0.29.2 # for t5
 av>=12.0.0 # for video loading
 
 # [gradio]
-gradio>=4.31.3
+gradio>=4.26.0
 spaces>=0.28.3
 
 # [notebook]