mirror of
https://github.com/hpcaitech/Open-Sora.git
synced 2026-05-21 11:59:01 +02:00
Merge branch 'dev/v1.2' into vbench_trials
This commit is contained in:
Shen-Chenhui
commit
7c6ef24e9a
|
|
@ -5,7 +5,10 @@ LABEL org.opencontainers.image.source = "https://github.com/hpcaitech/Open-Sora"
|
|||
LABEL org.opencontainers.image.licenses = "Apache License 2.0"
|
||||
LABEL org.opencontainers.image.base.name = "docker.io/library/hpcaitech/pytorch-cuda:2.1.0-12.1.0"
|
||||
|
||||
COPY . /workspace/Open-Sora
|
||||
# Set the working directory
|
||||
WORKDIR /workspace/Open-Sora
|
||||
# Copy the current directory contents into the container at /workspace/Open-Sora
|
||||
COPY . .
|
||||
|
||||
# inatall library dependencies
|
||||
RUN apt-get update && apt-get install ffmpeg libsm6 libxext6 -y
|
||||
|
|
@ -20,6 +23,4 @@ RUN pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolati
|
|||
RUN pip install xformers --index-url https://download.pytorch.org/whl/cu121
|
||||
|
||||
# install this project
|
||||
RUN git clone https://github.com/hpcaitech/Open-Sora && \
|
||||
cd Open-Sora && \
|
||||
pip install -v .
|
||||
RUN pip install -v .
|
||||
3
LICENSE
3
LICENSE
|
|
@ -1,3 +1,4 @@
|
|||
Copyright 2024 HPC-AI Technology Inc. All rights reserved.
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
|
@ -186,7 +187,7 @@
|
|||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
Copyright 2024 HPC-AI Technology Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
|
|
|
|||
183
README.md
183
README.md
|
|
@ -24,22 +24,30 @@ With Open-Sora, our goal is to foster innovation, creativity, and inclusivity wi
|
|||
|
||||
## 📰 News
|
||||
|
||||
* **[2024.06.17]** 🔥 We released **Open-Sora 1.2**, which includes **3D-VAE**, **rectified flow**, and **score condition**. The video quality is greatly improved. [[checkpoints]](#open-sora-10-model-weights) [[report]](/docs/report_03.md)
|
||||
* **[2024.04.25]** 🤗 We released the [Gradio demo for Open-Sora](https://huggingface.co/spaces/hpcai-tech/open-sora) on Hugging Face Spaces.
|
||||
* **[2024.04.25]** We released **Open-Sora 1.1**, which supports **2s~15s, 144p to 720p, any aspect ratio** text-to-image, **text-to-video, image-to-video, video-to-video, infinite time** generation. In addition, a full video processing pipeline is released. [[checkpoints]]() [[report]](/docs/report_02.md)
|
||||
* **[2024.03.18]** We released **Open-Sora 1.0**, a fully open-source project for video generation.
|
||||
- **[2024.06.17]** 🔥 We released **Open-Sora 1.2**, which includes **3D-VAE**, **rectified flow**, and **score condition**. The video quality is greatly improved. [[checkpoints]](#open-sora-10-model-weights) [[report]](/docs/report_03.md)
|
||||
- **[2024.04.25]** 🤗 We released the [Gradio demo for Open-Sora](https://huggingface.co/spaces/hpcai-tech/open-sora) on Hugging Face Spaces.
|
||||
- **[2024.04.25]** We released **Open-Sora 1.1**, which supports **2s~15s, 144p to 720p, any aspect ratio** text-to-image, **text-to-video, image-to-video, video-to-video, infinite time** generation. In addition, a full video processing pipeline is released. [[checkpoints]]() [[report]](/docs/report_02.md)
|
||||
- **[2024.03.18]** We released **Open-Sora 1.0**, a fully open-source project for video generation.
|
||||
Open-Sora 1.0 supports a full pipeline of video data preprocessing, training with
|
||||
<a href="https://github.com/hpcaitech/ColossalAI"><img src="assets/readme/colossal_ai.png" width="8%" ></a>
|
||||
acceleration,
|
||||
inference, and more. Our model can produce 2s 512x512 videos with only 3 days training. [[checkpoints]](#open-sora-10-model-weights)
|
||||
[[blog]](https://hpc-ai.com/blog/open-sora-v1.0) [[report]](/docs/report_01.md)
|
||||
* **[2024.03.04]** Open-Sora provides training with 46% cost reduction.
|
||||
- **[2024.03.04]** Open-Sora provides training with 46% cost reduction.
|
||||
[[blog]](https://hpc-ai.com/blog/open-sora)
|
||||
|
||||
## 🎥 Latest Demo
|
||||
|
||||
🔥 You can experience Open-Sora on our [🤗 Gradio application on Hugging Face](https://huggingface.co/spaces/hpcai-tech/open-sora). More samples are available in our [Gallery](https://hpcaitech.github.io/Open-Sora/).
|
||||
|
||||
|
||||
| **4s 720×1280** | **4s 720×1280** | **4s 720×1280** |
|
||||
| ---------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
| [<img src="assets/demo/v1.2/sample_0013.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/7895aab6-ed23-488c-8486-091480c26327) | [<img src="assets/demo/v1.2/sample_1718.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/20f07c7b-182b-4562-bbee-f1df74c86c9a) | [<img src="assets/demo/v1.2/sample_0087.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/3d897e0d-dc21-453a-b911-b3bda838acc2) |
|
||||
| [<img src="assets/demo/v1.2/sample_0052.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/644bf938-96ce-44aa-b797-b3c0b513d64c) | [<img src="assets/demo/v1.2/sample_1719.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/272d88ac-4b4a-484d-a665-8d07431671d0) | [<img src="assets/demo/v1.2/sample_0002.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/ebbac621-c34e-4bb4-9543-1c34f8989764) |
|
||||
| [<img src="assets/demo/v1.2/sample_0011.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/a1e3a1a3-4abd-45f5-8df2-6cced69da4ca) | [<img src="assets/demo/v1.2/sample_0004.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/d6ce9c13-28e1-4dff-9644-cc01f5f11926) | [<img src="assets/demo/v1.2/sample_0061.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/561978f8-f1b0-4f4d-ae7b-45bec9001b4a) |
|
||||
|
||||
|
||||
<details>
|
||||
<summary>OpenSora 1.1 Demo</summary>
|
||||
|
||||
|
|
@ -75,42 +83,42 @@ see [here](/assets/texts/t2v_samples.txt) for full prompts.
|
|||
|
||||
## 🔆 New Features/Updates
|
||||
|
||||
* 📍 **Open-Sora 1.2** released. Model weights are available [here](#model-weights). See our **[report 1.2](docs/report_03.md)** for more details.
|
||||
* ✅ Support rectified flow scheduling.
|
||||
* ✅ Support more conditioning including fps, aesthetic score, motion strength and camera motion.
|
||||
* ✅ Trained our 3D-VAE for temporal dimension compression.
|
||||
* 📍 **Open-Sora 1.1** released. Model weights are available [here](#model-weights). It is trained on **0s~15s, 144p to 720p, various aspect ratios** videos. See our **[report 1.1](/docs/report_02.md)** for more discussions.
|
||||
* 🔧 **Data processing pipeline v1.1** is released. An automatic [processing pipeline](#data-processing) from raw videos to (text, video clip) pairs is provided, including scene cutting $\rightarrow$ filtering(aesthetic, optical flow, OCR, etc.) $\rightarrow$ captioning $\rightarrow$ managing. With this tool, you can easily build your video dataset.
|
||||
* ✅ Improved ST-DiT architecture includes rope positional encoding, qk norm, longer text length, etc.
|
||||
* ✅ Support training with any resolution, aspect ratio, and duration (including images).
|
||||
* ✅ Support image and video conditioning and video editing, and thus support animating images, connecting videos, etc.
|
||||
* 📍 **Open-Sora 1.0** released. Model weights are available [here](#model-weights). With only 400K video clips and 200 H800
|
||||
days (compared with 152M samples in Stable Video Diffusion), we are able to generate 2s 512×512 videos. See our **[report 1.0](docs/report_01.md)** for more discussions.
|
||||
* ✅ Three-stage training from an image diffusion model to a video diffusion model. We provide the weights for each
|
||||
stage.
|
||||
* ✅ Support training acceleration including accelerated transformer, faster T5 and VAE, and sequence parallelism.
|
||||
Open-Sora improves **55%** training speed when training on 64x512x512 videos. Details locates
|
||||
at [acceleration.md](docs/acceleration.md).
|
||||
* 🔧 **Data preprocessing pipeline v1.0**,
|
||||
including [downloading](tools/datasets/README.md), [video cutting](tools/scene_cut/README.md),
|
||||
and [captioning](tools/caption/README.md) tools. Our data collection plan can be found
|
||||
at [datasets.md](docs/datasets.md).
|
||||
- 📍 **Open-Sora 1.2** released. Model weights are available [here](#model-weights). See our **[report 1.2](/docs/report_03.md)** for more details.
|
||||
- ✅ Support rectified flow scheduling.
|
||||
- ✅ Support more conditioning including fps, aesthetic score, motion strength and camera motion.
|
||||
- ✅ Trained our 3D-VAE for temporal dimension compression.
|
||||
- 📍 **Open-Sora 1.1** released. Model weights are available [here](#model-weights). It is trained on **0s~15s, 144p to 720p, various aspect ratios** videos. See our **[report 1.1](/docs/report_02.md)** for more discussions.
|
||||
- 🔧 **Data processing pipeline v1.1** is released. An automatic [processing pipeline](#data-processing) from raw videos to (text, video clip) pairs is provided, including scene cutting $\rightarrow$ filtering(aesthetic, optical flow, OCR, etc.) $\rightarrow$ captioning $\rightarrow$ managing. With this tool, you can easily build your video dataset.
|
||||
|
||||
<details>
|
||||
<summary>View more</summary>
|
||||
|
||||
* ✅ We find VQ-VAE from [VideoGPT](https://wilson1yan.github.io/videogpt/index.html) has a low quality and thus adopt a
|
||||
- ✅ Improved ST-DiT architecture includes rope positional encoding, qk norm, longer text length, etc.
|
||||
- ✅ Support training with any resolution, aspect ratio, and duration (including images).
|
||||
- ✅ Support image and video conditioning and video editing, and thus support animating images, connecting videos, etc.
|
||||
- 📍 **Open-Sora 1.0** released. Model weights are available [here](#model-weights). With only 400K video clips and 200 H800
|
||||
days (compared with 152M samples in Stable Video Diffusion), we are able to generate 2s 512×512 videos. See our **[report 1.0](docs/report_01.md)** for more discussions.
|
||||
- ✅ Three-stage training from an image diffusion model to a video diffusion model. We provide the weights for each
|
||||
stage.
|
||||
- ✅ Support training acceleration including accelerated transformer, faster T5 and VAE, and sequence parallelism.
|
||||
Open-Sora improves **55%** training speed when training on 64x512x512 videos. Details locates
|
||||
at [acceleration.md](docs/acceleration.md).
|
||||
- 🔧 **Data preprocessing pipeline v1.0**,
|
||||
including [downloading](tools/datasets/README.md), [video cutting](tools/scene_cut/README.md),
|
||||
and [captioning](tools/caption/README.md) tools. Our data collection plan can be found
|
||||
at [datasets.md](docs/datasets.md).
|
||||
- ✅ We find VQ-VAE from [VideoGPT](https://wilson1yan.github.io/videogpt/index.html) has a low quality and thus adopt a
|
||||
better VAE from [Stability-AI](https://huggingface.co/stabilityai/sd-vae-ft-mse-original). We also find patching in
|
||||
the time dimension deteriorates the quality. See our **[report](docs/report_01.md)** for more discussions.
|
||||
* ✅ We investigate different architectures including DiT, Latte, and our proposed STDiT. Our **STDiT** achieves a better
|
||||
- ✅ We investigate different architectures including DiT, Latte, and our proposed STDiT. Our **STDiT** achieves a better
|
||||
trade-off between quality and speed. See our **[report](docs/report_01.md)** for more discussions.
|
||||
* ✅ Support clip and T5 text conditioning.
|
||||
* ✅ By viewing images as one-frame videos, our project supports training DiT on both images and videos (e.g., ImageNet &
|
||||
- ✅ Support clip and T5 text conditioning.
|
||||
- ✅ By viewing images as one-frame videos, our project supports training DiT on both images and videos (e.g., ImageNet &
|
||||
UCF101). See [commands.md](docs/commands.md) for more instructions.
|
||||
* ✅ Support inference with official weights
|
||||
- ✅ Support inference with official weights
|
||||
from [DiT](https://github.com/facebookresearch/DiT), [Latte](https://github.com/Vchitect/Latte),
|
||||
and [PixArt](https://pixart-alpha.github.io/).
|
||||
* ✅ Refactor the codebase. See [structure.md](docs/structure.md) to learn the project structure and how to use the
|
||||
- ✅ Refactor the codebase. See [structure.md](docs/structure.md) to learn the project structure and how to use the
|
||||
config files.
|
||||
|
||||
</details>
|
||||
|
|
@ -120,40 +128,44 @@ see [here](/assets/texts/t2v_samples.txt) for full prompts.
|
|||
<details>
|
||||
<summary>View more</summary>
|
||||
|
||||
* [x] Training Video-VAE and adapt our model to new VAE.
|
||||
* [x] Scaling model parameters and dataset size.
|
||||
* [x] Incoporate a better scheduler (rectified flow).
|
||||
* [x] Evaluation pipeline.
|
||||
* [x] Complete the data processing pipeline (including dense optical flow, aesthetics scores, text-image similarity, etc.). See [the dataset](/docs/datasets.md) for more information
|
||||
* [x] Support image and video conditioning.
|
||||
* [x] Support variable aspect ratios, resolutions, durations.
|
||||
- [x] Training Video-VAE and adapt our model to new VAE.
|
||||
- [x] Scaling model parameters and dataset size.
|
||||
- [x] Incoporate a better scheduler (rectified flow).
|
||||
- [x] Evaluation pipeline.
|
||||
- [x] Complete the data processing pipeline (including dense optical flow, aesthetics scores, text-image similarity, etc.). See [the dataset](/docs/datasets.md) for more information
|
||||
- [x] Support image and video conditioning.
|
||||
- [x] Support variable aspect ratios, resolutions, durations.
|
||||
|
||||
</details>
|
||||
|
||||
## Contents
|
||||
|
||||
* [Installation](#installation)
|
||||
* [Model Weights](#model-weights)
|
||||
* [Gradio Demo](#gradio-demo)
|
||||
* [Inference](#inference)
|
||||
* [Data Processing](#data-processing)
|
||||
* [Training](#training)
|
||||
* [Evaluation](#evaluation)
|
||||
* [VAE Training & Evaluation](#vae-training--evaluation)
|
||||
* [Contribution](#contribution)
|
||||
* [Citation](#citation)
|
||||
* [Acknowledgement](#acknowledgement)
|
||||
- [Installation](#installation)
|
||||
- [Model Weights](#model-weights)
|
||||
- [Gradio Demo](#gradio-demo)
|
||||
- [Inference](#inference)
|
||||
- [Data Processing](#data-processing)
|
||||
- [Training](#training)
|
||||
- [Evaluation](#evaluation)
|
||||
- [VAE Training & Evaluation](#vae-training--evaluation)
|
||||
- [Contribution](#contribution)
|
||||
- [Citation](#citation)
|
||||
- [Acknowledgement](#acknowledgement)
|
||||
|
||||
Other useful documents and links are listed below.
|
||||
|
||||
* Report: [report 1.2](docs/report_03.md), [report 1.1](docs/report_02.md), [report 1.0](docs/report_01.md), [acceleration.md](docs/acceleration.md)
|
||||
* Repo structure: [structure.md](docs/structure.md)
|
||||
* Config file explanation: [config.md](docs/config.md)
|
||||
* Useful commands: [commands.md](docs/commands.md)
|
||||
* Data processing pipeline and dataset: [datasets.md](docs/datasets.md)
|
||||
* Each data processing tool's README: [dataset conventions and management](/tools/datasets/README.md), [scene cutting](/tools/scene_cut/README.md), [scoring](/tools/scoring/README.md), [caption](/tools/caption/README.md)
|
||||
* Evaluation: [eval/README.md](/eval/README.md)
|
||||
* Gallery: [gallery](https://hpcaitech.github.io/Open-Sora/)
|
||||
- Report: each version is trained from a image base seperately (not continuously trained), while a newer version will incorporate the techniques from the previous version.
|
||||
- [report 1.2](docs/report_03.md): rectified flow, 3d-VAE, score condition, evaluation, etc.
|
||||
- [report 1.1](docs/report_02.md): multi-resolution/length/aspect-ratio, image/video conditioning/editing, data preprocessing, etc.
|
||||
- [report 1.0](docs/report_01.md): architecture, captioning, etc.
|
||||
- [acceleration.md](docs/acceleration.md)
|
||||
- Repo structure: [structure.md](docs/structure.md)
|
||||
- Config file explanation: [config.md](docs/config.md)
|
||||
- Useful commands: [commands.md](docs/commands.md)
|
||||
- Data processing pipeline and dataset: [datasets.md](docs/datasets.md)
|
||||
- Each data processing tool's README: [dataset conventions and management](/tools/datasets/README.md), [scene cutting](/tools/scene_cut/README.md), [scoring](/tools/scoring/README.md), [caption](/tools/caption/README.md)
|
||||
- Evaluation: [eval/README.md](/eval/README.md)
|
||||
- Gallery: [gallery](https://hpcaitech.github.io/Open-Sora/)
|
||||
|
||||
## Installation
|
||||
|
||||
|
|
@ -166,13 +178,13 @@ For CUDA 12.1, you can install the dependencies with the following commands. Oth
|
|||
conda create -n opensora python=3.9
|
||||
conda activate opensora
|
||||
|
||||
# install torch, torchvision and xformers
|
||||
pip install -r requirements/requirements-cu121.txt
|
||||
|
||||
# download the repo
|
||||
git clone https://github.com/hpcaitech/Open-Sora
|
||||
cd Open-Sora
|
||||
|
||||
# install torch, torchvision and xformers
|
||||
pip install -r requirements/requirements-cu121.txt
|
||||
|
||||
# the default installation is for inference only
|
||||
pip install -v . # for development mode, `pip install -v -e .`
|
||||
```
|
||||
|
|
@ -195,13 +207,13 @@ pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation -
|
|||
Run the following command to build a docker image from Dockerfile provided.
|
||||
|
||||
```bash
|
||||
docker build -t opensora ./docker
|
||||
docker build -t opensora .
|
||||
```
|
||||
|
||||
Run the following command to start the docker container in interactive mode.
|
||||
|
||||
```bash
|
||||
docker run -ti --gpus all -v {MOUNT_DIR}:/data opensora
|
||||
docker run -ti --gpus all -v .:/workspace/Open-Sora opensora
|
||||
```
|
||||
|
||||
## Model Weights
|
||||
|
|
@ -215,6 +227,8 @@ docker run -ti --gpus all -v {MOUNT_DIR}:/data opensora
|
|||
|
||||
See our **[report 1.2](docs/report_03.md)** for more infomation. Weight will be automatically downloaded when you run the inference script.
|
||||
|
||||
> For users from mainland China, try `export HF_ENDPOINT=https://hf-mirror.com` to successfully download the weights.
|
||||
|
||||
### Open-Sora 1.1 Model Weights
|
||||
|
||||
<details>
|
||||
|
|
@ -291,7 +305,7 @@ Then, you can choose the **resolution**, **duration**, and **aspect ratio** of t
|
|||
| 480p | 2s, 24G | 29s, 31G | 55s, 30G | 108s, 32G | 219s, 36G |
|
||||
| 720p | 6s, 27G | 68s, 41G | 130s, 39G | 260s, 45G | 547s, 67G |
|
||||
|
||||
Note that besides text to video, you can also use image to video generation. You can upload an image and then click the "**Generate video**" button to generate a video with the image as the first frame. Or you can fill in the text prompt and click the "**Generate image**" button to generate an image with the text prompt, and then click the "**Generate video**" button to generate a video with the image generated with the same model.
|
||||
Note that besides text to video, you can also use **image to video generation**. You can upload an image and then click the "**Generate video**" button to generate a video with the image as the first frame. Or you can fill in the text prompt and click the "**Generate image**" button to generate an image with the text prompt, and then click the "**Generate video**" button to generate a video with the image generated with the same model.
|
||||
|
||||
|
||||
|
||||
|
|
@ -336,6 +350,19 @@ python scripts/inference.py configs/opensora-v1-2/inference/sample.py \
|
|||
--prompt "a beautiful waterfall"
|
||||
```
|
||||
|
||||
### Sequence Parallelism Inference
|
||||
|
||||
To enable sequence parallelism, you need to use `torchrun` to run the inference script. The following command will run the inference with 2 GPUs.
|
||||
|
||||
```bash
|
||||
# text to video
|
||||
CUDA_VISIBLE_DEVICES=0,1 torchrun --nproc_per_node 2 scripts/inference.py configs/opensora-v1-2/inference/sample.py \
|
||||
--num-frames 4s --resolution 720p --aspect-ratio 9:16 \
|
||||
--prompt "a beautiful waterfall"
|
||||
```
|
||||
|
||||
:warning: **LIMITATION**: The sequence parallelism is not supported for gradio deployment. For now, the sequence parallelism is only supported when the dimension can be divided by the number of GPUs. Thus, it may fail for some cases. We tested 4 GPUs for 720p and 2 GPUs for 480p.
|
||||
|
||||
### GPT-4o Prompt Refinement
|
||||
|
||||
We find that GPT-4o can refine the prompt and improve the quality of the generated video. With this feature, you can also use other language (e.g., Chinese) as the prompt. To enable this feature, you need prepare your openai api key in the environment:
|
||||
|
|
@ -472,10 +499,10 @@ For training other models and advanced usage, see [here](docs/commands.md) for m
|
|||
|
||||
We support evaluation based on:
|
||||
|
||||
* Validation loss
|
||||
* VBench score
|
||||
* VBench-i2v score
|
||||
* Batch generation for human evaluation
|
||||
- Validation loss
|
||||
- [VBench](https://github.com/Vchitect/VBench/tree/master) score
|
||||
- VBench-i2v score
|
||||
- Batch generation for human evaluation
|
||||
|
||||
All the evaluation code is released in `eval` folder. Check the [README](/eval/README.md) for more details. Our [report](/docs/report_03.md#evaluation) also provides more information about the evaluation during training. The following table shows Open-Sora 1.2 greatly improves Open-Sora 1.0.
|
||||
|
||||
|
|
@ -529,19 +556,19 @@ If you wish to contribute to this project, please refer to the [Contribution Gui
|
|||
|
||||
Here we only list a few of the projects. For other works and datasets, please refer to our report.
|
||||
|
||||
* [ColossalAI](https://github.com/hpcaitech/ColossalAI): A powerful large model parallel acceleration and optimization
|
||||
- [ColossalAI](https://github.com/hpcaitech/ColossalAI): A powerful large model parallel acceleration and optimization
|
||||
system.
|
||||
* [DiT](https://github.com/facebookresearch/DiT): Scalable Diffusion Models with Transformers.
|
||||
* [OpenDiT](https://github.com/NUS-HPC-AI-Lab/OpenDiT): An acceleration for DiT training. We adopt valuable acceleration
|
||||
- [DiT](https://github.com/facebookresearch/DiT): Scalable Diffusion Models with Transformers.
|
||||
- [OpenDiT](https://github.com/NUS-HPC-AI-Lab/OpenDiT): An acceleration for DiT training. We adopt valuable acceleration
|
||||
strategies for training progress from OpenDiT.
|
||||
* [PixArt](https://github.com/PixArt-alpha/PixArt-alpha): An open-source DiT-based text-to-image model.
|
||||
* [Latte](https://github.com/Vchitect/Latte): An attempt to efficiently train DiT for video.
|
||||
* [StabilityAI VAE](https://huggingface.co/stabilityai/sd-vae-ft-mse-original): A powerful image VAE model.
|
||||
* [CLIP](https://github.com/openai/CLIP): A powerful text-image embedding model.
|
||||
* [T5](https://github.com/google-research/text-to-text-transfer-transformer): A powerful text encoder.
|
||||
* [LLaVA](https://github.com/haotian-liu/LLaVA): A powerful image captioning model based on [Mistral-7B](https://huggingface.co/mistralai/Mistral-7B-v0.1) and [Yi-34B](https://huggingface.co/01-ai/Yi-34B).
|
||||
* [PLLaVA](https://github.com/magic-research/PLLaVA): A powerful video captioning model.
|
||||
* [MiraData](https://github.com/mira-space/MiraData): A large-scale video dataset with long durations and structured caption.
|
||||
- [PixArt](https://github.com/PixArt-alpha/PixArt-alpha): An open-source DiT-based text-to-image model.
|
||||
- [Latte](https://github.com/Vchitect/Latte): An attempt to efficiently train DiT for video.
|
||||
- [StabilityAI VAE](https://huggingface.co/stabilityai/sd-vae-ft-mse-original): A powerful image VAE model.
|
||||
- [CLIP](https://github.com/openai/CLIP): A powerful text-image embedding model.
|
||||
- [T5](https://github.com/google-research/text-to-text-transfer-transformer): A powerful text encoder.
|
||||
- [LLaVA](https://github.com/haotian-liu/LLaVA): A powerful image captioning model based on [Mistral-7B](https://huggingface.co/mistralai/Mistral-7B-v0.1) and [Yi-34B](https://huggingface.co/01-ai/Yi-34B).
|
||||
- [PLLaVA](https://github.com/magic-research/PLLaVA): A powerful video captioning model.
|
||||
- [MiraData](https://github.com/mira-space/MiraData): A large-scale video dataset with long durations and structured caption.
|
||||
|
||||
We are grateful for their exceptional work and generous contribution to open source. Special thanks go to the authors of [MiraData](https://github.com/mira-space/MiraData) and [Rectified Flow](https://github.com/gnobitab/RectifiedFlow) for their valuable advice and help. We wish to express gratitude towards AK for sharing this project on social media and Hugging Face for providing free GPU resources for our online Gradio demo.
|
||||
|
||||
|
|
|
|||
|
|
@ -19,12 +19,14 @@ model = dict(
|
|||
qk_norm=True,
|
||||
enable_flash_attn=True,
|
||||
enable_layernorm_kernel=True,
|
||||
force_huggingface=True,
|
||||
)
|
||||
vae = dict(
|
||||
type="OpenSoraVAE_V1_2",
|
||||
from_pretrained="hpcai-tech/OpenSora-VAE-v1.2",
|
||||
micro_frame_size=17,
|
||||
micro_batch_size=4,
|
||||
force_huggingface=True,
|
||||
)
|
||||
text_encoder = dict(
|
||||
type="t5",
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ model = dict(
|
|||
)
|
||||
vae = dict(
|
||||
type="OpenSoraVAE_V1_2",
|
||||
from_pretrained="pretrained_models/vae-pipeline",
|
||||
from_pretrained="hpcai-tech/OpenSora-VAE-v1.2",
|
||||
micro_frame_size=17,
|
||||
micro_batch_size=4,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ model = dict(
|
|||
)
|
||||
vae = dict(
|
||||
type="OpenSoraVAE_V1_2",
|
||||
from_pretrained="pretrained_models/vae-pipeline",
|
||||
from_pretrained="hpcai-tech/OpenSora-VAE-v1.2",
|
||||
micro_frame_size=17,
|
||||
micro_batch_size=4,
|
||||
local_files_only=True,
|
||||
|
|
|
|||
|
|
@ -36,14 +36,14 @@ wandb = False
|
|||
# Model settings
|
||||
model = dict(
|
||||
type="STDiT3-XL/2",
|
||||
from_pretrained="/mnt/nfs-206/zangwei/opensora/outputs/1091-STDiT3-XL-2/epoch0-global_step8500",
|
||||
from_pretrained="hpcai-tech/OpenSora-STDiT-v3",
|
||||
qk_norm=True,
|
||||
enable_flash_attn=True,
|
||||
enable_layernorm_kernel=True,
|
||||
)
|
||||
vae = dict(
|
||||
type="OpenSoraVAE_V1_2",
|
||||
from_pretrained="pretrained_models/vae-pipeline",
|
||||
from_pretrained="hpcai-tech/OpenSora-VAE-v1.2",
|
||||
micro_frame_size=17,
|
||||
micro_batch_size=32,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ model = dict(
|
|||
)
|
||||
vae = dict(
|
||||
type="OpenSoraVAE_V1_2",
|
||||
from_pretrained="pretrained_models/vae-pipeline",
|
||||
from_pretrained="hpcai-tech/OpenSora-VAE-v1.2",
|
||||
micro_frame_size=17,
|
||||
micro_batch_size=4,
|
||||
)
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ model = dict(
|
|||
)
|
||||
vae = dict(
|
||||
type="OpenSoraVAE_V1_2",
|
||||
from_pretrained="pretrained_models/vae-pipeline",
|
||||
from_pretrained="hpcai-tech/OpenSora-VAE-v1.2",
|
||||
micro_frame_size=17,
|
||||
micro_batch_size=4,
|
||||
)
|
||||
|
|
@ -46,7 +46,6 @@ text_encoder = dict(
|
|||
from_pretrained="DeepFloyd/t5-v1_1-xxl",
|
||||
model_max_length=300,
|
||||
shardformer=True,
|
||||
local_files_only=True,
|
||||
)
|
||||
scheduler = dict(
|
||||
type="rflow",
|
||||
|
|
|
|||
|
|
@ -63,7 +63,7 @@ model = dict(
|
|||
)
|
||||
vae = dict(
|
||||
type="OpenSoraVAE_V1_2",
|
||||
from_pretrained="/mnt/jfs/sora_checkpoints/vae-pipeline",
|
||||
from_pretrained="hpcai-tech/OpenSora-VAE-v1.2",
|
||||
micro_frame_size=17,
|
||||
micro_batch_size=4,
|
||||
)
|
||||
|
|
@ -72,7 +72,6 @@ text_encoder = dict(
|
|||
from_pretrained="DeepFloyd/t5-v1_1-xxl",
|
||||
model_max_length=300,
|
||||
shardformer=True,
|
||||
local_files_only=True,
|
||||
)
|
||||
scheduler = dict(
|
||||
type="rflow",
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ model = dict(
|
|||
)
|
||||
vae = dict(
|
||||
type="OpenSoraVAE_V1_2",
|
||||
from_pretrained="/mnt/jfs/sora_checkpoints/vae-pipeline",
|
||||
from_pretrained="hpcai-tech/OpenSora-VAE-v1.2",
|
||||
micro_frame_size=17,
|
||||
micro_batch_size=4,
|
||||
)
|
||||
|
|
@ -52,7 +52,6 @@ text_encoder = dict(
|
|||
from_pretrained="DeepFloyd/t5-v1_1-xxl",
|
||||
model_max_length=300,
|
||||
shardformer=True,
|
||||
local_files_only=True,
|
||||
)
|
||||
scheduler = dict(
|
||||
type="rflow",
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ model = dict(
|
|||
)
|
||||
vae = dict(
|
||||
type="OpenSoraVAE_V1_2",
|
||||
from_pretrained="/mnt/jfs/sora_checkpoints/vae-pipeline",
|
||||
from_pretrained="hpcai-tech/OpenSora-VAE-v1.2",
|
||||
micro_frame_size=17,
|
||||
micro_batch_size=4,
|
||||
)
|
||||
|
|
@ -52,7 +52,6 @@ text_encoder = dict(
|
|||
from_pretrained="DeepFloyd/t5-v1_1-xxl",
|
||||
model_max_length=300,
|
||||
shardformer=True,
|
||||
local_files_only=True,
|
||||
)
|
||||
scheduler = dict(
|
||||
type="rflow",
|
||||
|
|
|
|||
73
configs/opensora-v1-2/train/stage3_480p.py
Normal file
73
configs/opensora-v1-2/train/stage3_480p.py
Normal file
|
|
@ -0,0 +1,73 @@
|
|||
# Dataset settings
|
||||
dataset = dict(
|
||||
type="VariableVideoTextDataset",
|
||||
transform_name="resize_crop",
|
||||
)
|
||||
|
||||
# webvid
|
||||
bucket_config = {"480p": {51: (0.5, 5)}}
|
||||
grad_checkpoint = True
|
||||
|
||||
# Acceleration settings
|
||||
num_workers = 0
|
||||
num_bucket_build_workers = 16
|
||||
dtype = "bf16"
|
||||
plugin = "zero2"
|
||||
|
||||
# Model settings
|
||||
model = dict(
|
||||
type="STDiT3-XL/2",
|
||||
from_pretrained=None,
|
||||
qk_norm=True,
|
||||
enable_flash_attn=True,
|
||||
enable_layernorm_kernel=True,
|
||||
freeze_y_embedder=True,
|
||||
)
|
||||
vae = dict(
|
||||
type="OpenSoraVAE_V1_2",
|
||||
from_pretrained="hpcai-tech/OpenSora-VAE-v1.2",
|
||||
micro_frame_size=17,
|
||||
micro_batch_size=4,
|
||||
)
|
||||
text_encoder = dict(
|
||||
type="t5",
|
||||
from_pretrained="DeepFloyd/t5-v1_1-xxl",
|
||||
model_max_length=300,
|
||||
shardformer=True,
|
||||
)
|
||||
scheduler = dict(
|
||||
type="rflow",
|
||||
use_timestep_transform=True,
|
||||
sample_method="logit-normal",
|
||||
)
|
||||
|
||||
# Mask settings
|
||||
# 25%
|
||||
mask_ratios = {
|
||||
"random": 0.01,
|
||||
"intepolate": 0.002,
|
||||
"quarter_random": 0.002,
|
||||
"quarter_head": 0.002,
|
||||
"quarter_tail": 0.002,
|
||||
"quarter_head_tail": 0.002,
|
||||
"image_random": 0.0,
|
||||
"image_head": 0.22,
|
||||
"image_tail": 0.005,
|
||||
"image_head_tail": 0.005,
|
||||
}
|
||||
|
||||
# Log settings
|
||||
seed = 42
|
||||
outputs = "outputs"
|
||||
wandb = False
|
||||
epochs = 1000
|
||||
log_every = 10
|
||||
ckpt_every = 200
|
||||
|
||||
# optimization settings
|
||||
load = None
|
||||
grad_clip = 1.0
|
||||
lr = 1e-4
|
||||
ema_decay = 0.99
|
||||
adam_eps = 1e-15
|
||||
warmup_steps = 1000
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
image_size = (256, 256)
|
||||
num_frames = 17
|
||||
num_frames = 51
|
||||
|
||||
dtype = "bf16"
|
||||
batch_size = 1
|
||||
|
|
|
|||
|
|
@ -18,20 +18,10 @@ plugin = "zero2"
|
|||
|
||||
# Define model
|
||||
model = dict(
|
||||
type="VideoAutoencoderPipeline",
|
||||
type="OpenSoraVAE_V1_2",
|
||||
freeze_vae_2d=True,
|
||||
from_pretrained=None,
|
||||
cal_loss=True,
|
||||
vae_2d=dict(
|
||||
type="VideoAutoencoderKL",
|
||||
from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
|
||||
subfolder="vae",
|
||||
local_files_only=True,
|
||||
),
|
||||
vae_temporal=dict(
|
||||
type="VAE_Temporal_SD",
|
||||
from_pretrained=None,
|
||||
),
|
||||
)
|
||||
|
||||
# loss weights
|
||||
|
|
|
|||
|
|
@ -18,20 +18,10 @@ plugin = "zero2"
|
|||
|
||||
# Define model
|
||||
model = dict(
|
||||
type="VideoAutoencoderPipeline",
|
||||
type="OpenSoraVAE_V1_2",
|
||||
freeze_vae_2d=False,
|
||||
from_pretrained="outputs/vae_stage1",
|
||||
cal_loss=True,
|
||||
vae_2d=dict(
|
||||
type="VideoAutoencoderKL",
|
||||
from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
|
||||
subfolder="vae",
|
||||
local_files_only=True,
|
||||
),
|
||||
vae_temporal=dict(
|
||||
type="VAE_Temporal_SD",
|
||||
from_pretrained=None,
|
||||
),
|
||||
)
|
||||
|
||||
# loss weights
|
||||
|
|
|
|||
|
|
@ -18,20 +18,10 @@ plugin = "zero2"
|
|||
|
||||
# Define model
|
||||
model = dict(
|
||||
type="VideoAutoencoderPipeline",
|
||||
type="OpenSoraVAE_V1_2",
|
||||
freeze_vae_2d=False,
|
||||
from_pretrained="outputs/vae_stage2",
|
||||
cal_loss=True,
|
||||
vae_2d=dict(
|
||||
type="VideoAutoencoderKL",
|
||||
from_pretrained="PixArt-alpha/pixart_sigma_sdxlvae_T5_diffusers",
|
||||
subfolder="vae",
|
||||
local_files_only=True,
|
||||
),
|
||||
vae_temporal=dict(
|
||||
type="VAE_Temporal_SD",
|
||||
from_pretrained=None,
|
||||
),
|
||||
)
|
||||
|
||||
# loss weights
|
||||
|
|
|
|||
|
|
@ -28,6 +28,8 @@ python scripts/inference.py configs/opensora-v1-2/inference/sample.py \
|
|||
--image-size 480 853
|
||||
```
|
||||
|
||||
In this version, we have merged all functions in previous `inference-long.py` into `inference.py`. The command line arguments are the same as before (only note that the frame index and length is calculated with 4x compressed).
|
||||
|
||||
### Inference with Open-Sora 1.1
|
||||
|
||||
Since Open-Sora 1.1 supports inference with dynamic input size, you can pass the input size as an argument.
|
||||
|
|
|
|||
|
|
@ -17,11 +17,12 @@ We use the [training-10M subset](https://github.com/snap-research/Panda-70M/tree
|
|||
which contains ~10M videos of better quality.
|
||||
|
||||
### Mixkit
|
||||
|
||||
[Mixkit](https://mixkit.co/) is a video website where we obtained 9k videos.
|
||||
|
||||
### Pixabay
|
||||
[Pixabay](https://pixabay.com/videos/) is video website where we obtained 60.5k videos.
|
||||
|
||||
[Pixabay](https://pixabay.com/videos/) is video website where we obtained 60.5k videos.
|
||||
|
||||
### Pexels
|
||||
|
||||
|
|
@ -64,7 +65,7 @@ This dataset is proposed for exploring the relationship of prompts and high-qual
|
|||
Moreover, this dataset is divided into two subsets: original and upscale.
|
||||
This dataset is proposed for enabling research on text-to-image model prompting.
|
||||
|
||||
### upsplash-lite
|
||||
### Unsplash-lite
|
||||
|
||||
The [Unsplash-lite](https://github.com/unsplash/datasets) Dataset comprises 25k nature-themed Unsplash photos, 25k keywords, and 1M searches.
|
||||
This dataset covers a vast range of uses and contexts. Its extensive scope in intent and semantics opens new avenues for research and learning.
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ We made the following modifications to the original ST-DiT for better training s
|
|||
|
||||
- **[Rope embedding](https://arxiv.org/abs/2104.09864) for temporal attention**: Following LLM's best practice, we change the sinusoidal positional encoding to rope embedding for temporal attention since it is also a sequence prediction task.
|
||||
- **AdaIN and Layernorm for temporal attention**: we wrap the temporal attention with AdaIN and layernorm as the spatial attention to stabilize the training.
|
||||
- **[QK-normalization](https://arxiv.org/abs/2302.05442) with [RMSNorm](https://arxiv.org/abs/1910.07467)**: Following [SD3](https://arxiv.org/pdf/2403.03206.pdf), we appy QK-normalization to the all attention for better training stability in half-precision.
|
||||
- **[QK-normalization](https://arxiv.org/abs/2302.05442) with [RMSNorm](https://arxiv.org/abs/1910.07467)**: Following [SD3](https://arxiv.org/pdf/2403.03206.pdf), we apply QK-normalization to the all attention for better training stability in half-precision.
|
||||
- **Dynamic input size support and video infomation condition**: To support multi-resolution, aspect ratio, and fps training, we make ST-DiT-2 to accept any input size, and automatically scale positional embeddings. Extending [PixArt-alpha](https://github.com/PixArt-alpha/PixArt-alpha)'s idea, we conditioned on video's height, width, aspect ratio, frame length, and fps.
|
||||
- **Extending T5 tokens from 120 to 200**: our caption is usually less than 200 tokens, and we find the model can handle longer text well.
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@
|
|||
- [Evaluation](#evaluation)
|
||||
- [Sequence parallelism](#sequence-parallelism)
|
||||
|
||||
In Open-Sora 1.2 release, we train a 1.1B models on >20M data, with training cost 35k H100 GPU hours, supporting 0s~16s, 144p to 720p, various aspect ratios video generation. Our configurations is listed below. Following our 1.1 version, Open-Sora 1.2 can also do image-to-video generation and video extension.
|
||||
In Open-Sora 1.2 release, we train a 1.1B models on >30M data (~80k hours), with training cost 35k H100 GPU hours, supporting 0s~16s, 144p to 720p, various aspect ratios video generation. Our configurations is listed below. Following our 1.1 version, Open-Sora 1.2 can also do image-to-video generation and video extension.
|
||||
|
||||
| | image | 2s | 4s | 8s | 16s |
|
||||
| ---- | ----- | --- | --- | --- | --- |
|
||||
|
|
@ -57,13 +57,13 @@ When using the VAE for diffusion model, our stacked VAE requires small memory as
|
|||
|
||||
Lastest diffusion model like Stable Diffusion 3 adopts the [rectified flow](https://github.com/gnobitab/RectifiedFlow) instead of DDPM for better performance. Pitiably, SD3's rectified flow training code is not open-sourced. However, Open-Sora 1.2 provides the training code following SD3's paper, including:
|
||||
|
||||
- Basic rectified flow training
|
||||
- Logit-norm sampling for training acceleration
|
||||
- Resolution and video length aware timestep sampling
|
||||
- Basic rectified flow training ([original rectified flow paper](https://arxiv.org/abs/2209.03003))
|
||||
- Logit-norm sampling for training acceleration ([SD3 paper](https://arxiv.org/pdf/2403.03206) Section 3.1, intuitively it is more likely to sample timesteps at middle noise level)
|
||||
- Resolution and video length aware timestep sampling ([SD3 paper](https://arxiv.org/pdf/2403.03206) Section 5.3.2, intuitively it is more likely to sample timesteps with more noise for larger resolution, and we extend it to longer video)
|
||||
|
||||
For the resolution-aware timestep sampling, we should use more noise for images with larger resolution. We extend this idea to video generation and use more noise for videos with longer length.
|
||||
|
||||
Open-Sora 1.2 starts from the [PixArt-Σ 2K](https://github.com/PixArt-alpha/PixArt-sigma) checkpoint. Note that this model is trained with DDPM and SDXL VAE, also a much higher resolution. We find finetuning on a small dataset can easily adapt the model for our video generation setting. The adaptation process is as follows, all training is done on 8 GPUs:
|
||||
Open-Sora 1.2 starts from the [PixArt-Σ 2K](https://github.com/PixArt-alpha/PixArt-sigma) checkpoint. Note that this model is trained with DDPM and SDXL VAE, also a much higher resolution. We find finetuning on a small dataset can easily adapt the model for our video generation setting. The adaptation process is as follows, all training is done on 8 GPUs (the adaptation for the diffusion model is quite fast and straightforward):
|
||||
|
||||
1. Multi-resolution image generation ability: we train the model to generate different resolution ranging from 144p to 2K for 20k steps.
|
||||
2. QK-norm: we add the QK-norm to the model and train for 18k steps.
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ We train the VAE using 20% images and 80% videos with 17 frames.
|
|||
torchrun --nnodes=1 --nproc_per_node=8 scripts/train_vae.py configs/vae/train/stage1.py --data-path YOUR_CSV_PATH
|
||||
```
|
||||
|
||||
Next, we remove the indentity loss and train the 3D VAE pipeline to reconstructe the 2D-compressed videos for 260k steps.
|
||||
Next, we remove the identity loss and train the 3D VAE pipeline to reconstructe the 2D-compressed videos for 260k steps.
|
||||
|
||||
```bash
|
||||
torchrun --nnodes=1 --nproc_per_node=8 scripts/train_vae.py configs/vae/train/stage2.py --data-path YOUR_CSV_PATH
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
<p align="center">
|
||||
<img src="./assets/readme/icon.png" width="250"/>
|
||||
<img src="../../assets/readme/icon.png" width="250"/>
|
||||
</p>
|
||||
<div align="center">
|
||||
<a href="https://github.com/hpcaitech/Open-Sora/stargazers"><img src="https://img.shields.io/github/stars/hpcaitech/Open-Sora?style=social"></a>
|
||||
|
|
@ -175,13 +175,13 @@ pip install -v --disable-pip-version-check --no-cache-dir --no-build-isolation -
|
|||
运行以下命令从提供的Dockerfile 构建docker 镜像。
|
||||
|
||||
```bash
|
||||
docker build -t opensora ./docker
|
||||
docker build -t opensora .
|
||||
```
|
||||
|
||||
运行以下命令以交互模式启动docker容器。
|
||||
|
||||
```bash
|
||||
docker run -ti --gpus all -v {MOUNT_DIR}:/data opensora
|
||||
docker run -ti --gpus all -v .:/workspace/Open-Sora opensora
|
||||
```
|
||||
|
||||
## 模型权重
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@
|
|||
|
||||
我们的训练包括三个阶段：
|
||||
|
||||
1.对于前 380k 步，我们冻结 2D VAE并在 8 个 GPU 上进行训练。训练目标包括重建 2D VAE 的压缩特征（图中粉红色），并添加损失以使 3D VAE 的特征与 2D VAE 的特征相似（粉红色和绿色，称为identity loss）。我们发现后者的损失可以快速使整个 VAE 在图像上取得良好的性能，并在下一阶段更快地收敛。
|
||||
1. 对于前 380k 步，我们冻结 2D VAE并在 8 个 GPU 上进行训练。训练目标包括重建 2D VAE 的压缩特征（图中粉红色），并添加损失以使 3D VAE 的特征与 2D VAE 的特征相似（粉红色和绿色，称为identity loss）。我们发现后者的损失可以快速使整个 VAE 在图像上取得良好的性能，并在下一阶段更快地收敛。
|
||||
2. 对于接下来的 260k 步，我们消除identity loss并仅学习 3D VAE。
|
||||
3. 对于最后 540k 步，由于我们发现仅重建 2D VAE 的特征无法带来进一步的改进，因此我们移除了loss并训练整个 VAE 来重建原始视频。此阶段在 24 个 GPU 上进行训练。
|
||||
|
||||
|
|
|
|||
|
|
@ -42,17 +42,15 @@ def install_dependencies(enable_optimization=False):
|
|||
except (ImportError, ModuleNotFoundError):
|
||||
return False
|
||||
|
||||
# flash attention is needed no matter optimization is enabled or not
|
||||
# because Hugging Face transformers detects flash_attn is a dependency in STDiT
|
||||
# thus, we need to install it no matter what
|
||||
if not _is_package_available("flash_attn"):
|
||||
subprocess.run(
|
||||
f"{sys.executable} -m pip install flash-attn --no-build-isolation",
|
||||
env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"},
|
||||
shell=True,
|
||||
)
|
||||
|
||||
if enable_optimization:
|
||||
# install flash attention
|
||||
if not _is_package_available("flash_attn"):
|
||||
subprocess.run(
|
||||
f"{sys.executable} -m pip install flash-attn --no-build-isolation",
|
||||
env={"FLASH_ATTENTION_SKIP_CUDA_BUILD": "TRUE"},
|
||||
shell=True,
|
||||
)
|
||||
|
||||
# install apex for fused layernorm
|
||||
if not _is_package_available("apex"):
|
||||
subprocess.run(
|
||||
|
|
@ -102,7 +100,8 @@ def build_models(model_type, config, enable_optimization=False):
|
|||
# handle model download logic in HuggingFace Space
|
||||
from opensora.models.stdit.stdit3 import STDiT3
|
||||
|
||||
stdit = STDiT3.from_pretrained(HF_STDIT_MAP[model_type])
|
||||
model_kwargs = {k: v for k, v in config.model.items() if k not in ("type", "from_pretrained")}
|
||||
stdit = STDiT3.from_pretrained(HF_STDIT_MAP[model_type], **model_kwargs)
|
||||
stdit = stdit.cuda()
|
||||
|
||||
# build scheduler
|
||||
|
|
@ -514,8 +513,10 @@ def main():
|
|||
with gr.Row():
|
||||
with gr.Column():
|
||||
prompt_text = gr.Textbox(label="Prompt", placeholder="Describe your video here", lines=4)
|
||||
refine_prompt = gr.Checkbox(value=True, label="Refine prompt with GPT4o")
|
||||
random_prompt_btn = gr.Button("Random Prompt By GPT4o")
|
||||
refine_prompt = gr.Checkbox(
|
||||
value=has_openai_key(), label="Refine prompt with GPT4o", interactive=has_openai_key()
|
||||
)
|
||||
random_prompt_btn = gr.Button("Random Prompt By GPT4o", interactive=has_openai_key())
|
||||
|
||||
gr.Markdown("## Basic Settings")
|
||||
resolution = gr.Radio(
|
||||
|
|
@ -647,7 +648,8 @@ def main():
|
|||
random_prompt_btn.click(fn=generate_random_prompt, outputs=prompt_text)
|
||||
|
||||
# launch
|
||||
demo.launch(server_port=args.port, server_name=args.host, share=args.share)
|
||||
demo.queue(max_size=5, default_concurrency_limit=1)
|
||||
demo.launch(server_port=args.port, server_name=args.host, share=args.share, max_threads=1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -1,3 +1,3 @@
|
|||
xformers
|
||||
transformers
|
||||
git+https://github.com/hpcaitech/Open-Sora.git#egg=opensora
|
||||
git+https://github.com/hpcaitech/Open-Sora.git
|
||||
|
|
|
|||
|
|
@ -225,7 +225,7 @@ ASPECT_RATIO_2880 = {
|
|||
"0.32": (1600, 4992),
|
||||
"0.33": (1600, 4800),
|
||||
"0.34": (1600, 4672),
|
||||
"0.4": (1792, 4480),
|
||||
"0.40": (1792, 4480),
|
||||
"0.42": (1792, 4288),
|
||||
"0.47": (1920, 4096),
|
||||
"0.49": (1920, 3904),
|
||||
|
|
@ -238,26 +238,26 @@ ASPECT_RATIO_2880 = {
|
|||
"0.83": (2496, 3008),
|
||||
"0.89": (2688, 3008),
|
||||
"0.93": (2688, 2880),
|
||||
"1.0": (2880, 2880),
|
||||
"1.00": (2880, 2880),
|
||||
"1.07": (2880, 2688),
|
||||
"1.12": (3008, 2688),
|
||||
"1.21": (3008, 2496),
|
||||
"1.28": (3200, 2496),
|
||||
"1.39": (3200, 2304),
|
||||
"1.47": (3392, 2304),
|
||||
"1.7": (3584, 2112),
|
||||
"1.70": (3584, 2112),
|
||||
"1.82": (3840, 2112),
|
||||
"2.03": (3904, 1920),
|
||||
"2.13": (4096, 1920),
|
||||
"2.39": (4288, 1792),
|
||||
"2.5": (4480, 1792),
|
||||
"2.50": (4480, 1792),
|
||||
"2.92": (4672, 1600),
|
||||
"3.0": (4800, 1600),
|
||||
"3.00": (4800, 1600),
|
||||
"3.12": (4992, 1600),
|
||||
"3.68": (5184, 1408),
|
||||
"3.82": (5376, 1408),
|
||||
"3.95": (5568, 1408),
|
||||
"4.0": (5760, 1408),
|
||||
"4.00": (5760, 1408),
|
||||
}
|
||||
|
||||
# S = 4194304
|
||||
|
|
@ -269,20 +269,20 @@ ASPECT_RATIO_2048 = {
|
|||
"0.32": (1152, 3584),
|
||||
"0.33": (1152, 3456),
|
||||
"0.35": (1152, 3328),
|
||||
"0.4": (1280, 3200),
|
||||
"0.40": (1280, 3200),
|
||||
"0.42": (1280, 3072),
|
||||
"0.48": (1408, 2944),
|
||||
"0.5": (1408, 2816),
|
||||
"0.50": (1408, 2816),
|
||||
"0.52": (1408, 2688),
|
||||
"0.57": (1536, 2688),
|
||||
"0.6": (1536, 2560),
|
||||
"0.60": (1536, 2560),
|
||||
"0.68": (1664, 2432),
|
||||
"0.72": (1664, 2304),
|
||||
"0.78": (1792, 2304),
|
||||
"0.82": (1792, 2176),
|
||||
"0.88": (1920, 2176),
|
||||
"0.94": (1920, 2048),
|
||||
"1.0": (2048, 2048),
|
||||
"1.00": (2048, 2048),
|
||||
"1.07": (2048, 1920),
|
||||
"1.13": (2176, 1920),
|
||||
"1.21": (2176, 1792),
|
||||
|
|
@ -291,17 +291,17 @@ ASPECT_RATIO_2048 = {
|
|||
"1.46": (2432, 1664),
|
||||
"1.67": (2560, 1536),
|
||||
"1.75": (2688, 1536),
|
||||
"2.0": (2816, 1408),
|
||||
"2.00": (2816, 1408),
|
||||
"2.09": (2944, 1408),
|
||||
"2.4": (3072, 1280),
|
||||
"2.5": (3200, 1280),
|
||||
"2.40": (3072, 1280),
|
||||
"2.50": (3200, 1280),
|
||||
"2.89": (3328, 1152),
|
||||
"3.0": (3456, 1152),
|
||||
"3.00": (3456, 1152),
|
||||
"3.11": (3584, 1152),
|
||||
"3.62": (3712, 1024),
|
||||
"3.75": (3840, 1024),
|
||||
"3.88": (3968, 1024),
|
||||
"4.0": (4096, 1024),
|
||||
"4.00": (4096, 1024),
|
||||
}
|
||||
|
||||
# S = 1048576
|
||||
|
|
@ -313,20 +313,20 @@ ASPECT_RATIO_1024 = {
|
|||
"0.32": (576, 1792),
|
||||
"0.33": (576, 1728),
|
||||
"0.35": (576, 1664),
|
||||
"0.4": (640, 1600),
|
||||
"0.40": (640, 1600),
|
||||
"0.42": (640, 1536),
|
||||
"0.48": (704, 1472),
|
||||
"0.5": (704, 1408),
|
||||
"0.50": (704, 1408),
|
||||
"0.52": (704, 1344),
|
||||
"0.57": (768, 1344),
|
||||
"0.6": (768, 1280),
|
||||
"0.60": (768, 1280),
|
||||
"0.68": (832, 1216),
|
||||
"0.72": (832, 1152),
|
||||
"0.78": (896, 1152),
|
||||
"0.82": (896, 1088),
|
||||
"0.88": (960, 1088),
|
||||
"0.94": (960, 1024),
|
||||
"1.0": (1024, 1024),
|
||||
"1.00": (1024, 1024),
|
||||
"1.07": (1024, 960),
|
||||
"1.13": (1088, 960),
|
||||
"1.21": (1088, 896),
|
||||
|
|
@ -335,17 +335,17 @@ ASPECT_RATIO_1024 = {
|
|||
"1.46": (1216, 832),
|
||||
"1.67": (1280, 768),
|
||||
"1.75": (1344, 768),
|
||||
"2.0": (1408, 704),
|
||||
"2.00": (1408, 704),
|
||||
"2.09": (1472, 704),
|
||||
"2.4": (1536, 640),
|
||||
"2.5": (1600, 640),
|
||||
"2.40": (1536, 640),
|
||||
"2.50": (1600, 640),
|
||||
"2.89": (1664, 576),
|
||||
"3.0": (1728, 576),
|
||||
"3.00": (1728, 576),
|
||||
"3.11": (1792, 576),
|
||||
"3.62": (1856, 512),
|
||||
"3.75": (1920, 512),
|
||||
"3.88": (1984, 512),
|
||||
"4.0": (2048, 512),
|
||||
"4.00": (2048, 512),
|
||||
}
|
||||
|
||||
# S = 262144
|
||||
|
|
@ -357,20 +357,20 @@ ASPECT_RATIO_512 = {
|
|||
"0.32": (288, 896),
|
||||
"0.33": (288, 864),
|
||||
"0.35": (288, 832),
|
||||
"0.4": (320, 800),
|
||||
"0.40": (320, 800),
|
||||
"0.42": (320, 768),
|
||||
"0.48": (352, 736),
|
||||
"0.5": (352, 704),
|
||||
"0.50": (352, 704),
|
||||
"0.52": (352, 672),
|
||||
"0.57": (384, 672),
|
||||
"0.6": (384, 640),
|
||||
"0.60": (384, 640),
|
||||
"0.68": (416, 608),
|
||||
"0.72": (416, 576),
|
||||
"0.78": (448, 576),
|
||||
"0.82": (448, 544),
|
||||
"0.88": (480, 544),
|
||||
"0.94": (480, 512),
|
||||
"1.0": (512, 512),
|
||||
"1.00": (512, 512),
|
||||
"1.07": (512, 480),
|
||||
"1.13": (544, 480),
|
||||
"1.21": (544, 448),
|
||||
|
|
@ -379,17 +379,17 @@ ASPECT_RATIO_512 = {
|
|||
"1.46": (608, 416),
|
||||
"1.67": (640, 384),
|
||||
"1.75": (672, 384),
|
||||
"2.0": (704, 352),
|
||||
"2.00": (704, 352),
|
||||
"2.09": (736, 352),
|
||||
"2.4": (768, 320),
|
||||
"2.5": (800, 320),
|
||||
"2.40": (768, 320),
|
||||
"2.50": (800, 320),
|
||||
"2.89": (832, 288),
|
||||
"3.0": (864, 288),
|
||||
"3.00": (864, 288),
|
||||
"3.11": (896, 288),
|
||||
"3.62": (928, 256),
|
||||
"3.75": (960, 256),
|
||||
"3.88": (992, 256),
|
||||
"4.0": (1024, 256),
|
||||
"4.00": (1024, 256),
|
||||
}
|
||||
|
||||
# S = 65536
|
||||
|
|
@ -401,20 +401,20 @@ ASPECT_RATIO_256 = {
|
|||
"0.32": (144, 448),
|
||||
"0.33": (144, 432),
|
||||
"0.35": (144, 416),
|
||||
"0.4": (160, 400),
|
||||
"0.40": (160, 400),
|
||||
"0.42": (160, 384),
|
||||
"0.48": (176, 368),
|
||||
"0.5": (176, 352),
|
||||
"0.50": (176, 352),
|
||||
"0.52": (176, 336),
|
||||
"0.57": (192, 336),
|
||||
"0.6": (192, 320),
|
||||
"0.60": (192, 320),
|
||||
"0.68": (208, 304),
|
||||
"0.72": (208, 288),
|
||||
"0.78": (224, 288),
|
||||
"0.82": (224, 272),
|
||||
"0.88": (240, 272),
|
||||
"0.94": (240, 256),
|
||||
"1.0": (256, 256),
|
||||
"1.00": (256, 256),
|
||||
"1.07": (256, 240),
|
||||
"1.13": (272, 240),
|
||||
"1.21": (272, 224),
|
||||
|
|
@ -423,17 +423,17 @@ ASPECT_RATIO_256 = {
|
|||
"1.46": (304, 208),
|
||||
"1.67": (320, 192),
|
||||
"1.75": (336, 192),
|
||||
"2.0": (352, 176),
|
||||
"2.00": (352, 176),
|
||||
"2.09": (368, 176),
|
||||
"2.4": (384, 160),
|
||||
"2.5": (400, 160),
|
||||
"2.40": (384, 160),
|
||||
"2.50": (400, 160),
|
||||
"2.89": (416, 144),
|
||||
"3.0": (432, 144),
|
||||
"3.00": (432, 144),
|
||||
"3.11": (448, 144),
|
||||
"3.62": (464, 128),
|
||||
"3.75": (480, 128),
|
||||
"3.88": (496, 128),
|
||||
"4.0": (512, 128),
|
||||
"4.00": (512, 128),
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -152,6 +152,8 @@ class VariableVideoTextDataset(VideoTextDataset):
|
|||
# Sampling video frames
|
||||
video = temporal_random_crop(vframes, num_frames, self.frame_interval)
|
||||
|
||||
video_fps = video_fps // self.frame_interval
|
||||
|
||||
# transform
|
||||
transform = get_transforms_video(self.transform_name, (height, width))
|
||||
video = transform(video) # T C H W
|
||||
|
|
|
|||
|
|
@ -52,8 +52,7 @@ def read_file(input_path):
|
|||
|
||||
def download_url(input_path):
|
||||
output_dir = "cache"
|
||||
if not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
base_name = os.path.basename(input_path)
|
||||
output_path = os.path.join(output_dir, base_name)
|
||||
img_data = requests.get(input_path).content
|
||||
|
|
|
|||
|
|
@ -267,10 +267,10 @@ class STDiT(nn.Module):
|
|||
Returns:
|
||||
x (torch.Tensor): output latent representation; of shape [B, C, T, H, W]
|
||||
"""
|
||||
|
||||
x = x.to(self.dtype)
|
||||
timestep = timestep.to(self.dtype)
|
||||
y = y.to(self.dtype)
|
||||
dtype = self.x_embedder.proj.weight.dtype
|
||||
x = x.to(dtype)
|
||||
timestep = timestep.to(dtype)
|
||||
y = y.to(dtype)
|
||||
|
||||
# embedding
|
||||
x = self.x_embedder(x) # [B, N, C]
|
||||
|
|
|
|||
|
|
@ -447,7 +447,8 @@ class STDiT3(PreTrainedModel):
|
|||
|
||||
@MODELS.register_module("STDiT3-XL/2")
|
||||
def STDiT3_XL_2(from_pretrained=None, **kwargs):
|
||||
if from_pretrained is not None and not os.path.isdir(from_pretrained):
|
||||
force_huggingface = kwargs.pop("force_huggingface", False)
|
||||
if force_huggingface or from_pretrained is not None and not os.path.isdir(from_pretrained):
|
||||
model = STDiT3.from_pretrained(from_pretrained, **kwargs)
|
||||
else:
|
||||
config = STDiT3Config(depth=28, hidden_size=1152, patch_size=(1, 2, 2), num_heads=16, **kwargs)
|
||||
|
|
@ -459,7 +460,6 @@ def STDiT3_XL_2(from_pretrained=None, **kwargs):
|
|||
|
||||
@MODELS.register_module("STDiT3-3B/2")
|
||||
def STDiT3_3B_2(from_pretrained=None, **kwargs):
|
||||
# check if from_pretrained is a path
|
||||
if from_pretrained is not None and not os.path.isdir(from_pretrained):
|
||||
model = STDiT3.from_pretrained(from_pretrained, **kwargs)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -32,8 +32,6 @@ from opensora.registry import MODELS
|
|||
|
||||
|
||||
class T5Embedder:
|
||||
available_models = ["DeepFloyd/t5-v1_1-xxl"]
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
device,
|
||||
|
|
@ -99,7 +97,6 @@ class T5Embedder:
|
|||
self.use_text_preprocessing = use_text_preprocessing
|
||||
self.hf_token = hf_token
|
||||
|
||||
# assert from_pretrained in self.available_models
|
||||
self.tokenizer = AutoTokenizer.from_pretrained(
|
||||
from_pretrained,
|
||||
cache_dir=cache_dir,
|
||||
|
|
|
|||
|
|
@ -145,7 +145,6 @@ class VideoAutoencoderPipelineConfig(PretrainedConfig):
|
|||
super().__init__(**kwargs)
|
||||
|
||||
|
||||
@MODELS.register_module()
|
||||
class VideoAutoencoderPipeline(PreTrainedModel):
|
||||
config_class = VideoAutoencoderPipelineConfig
|
||||
|
||||
|
|
@ -253,6 +252,7 @@ def OpenSoraVAE_V1_2(
|
|||
local_files_only=False,
|
||||
freeze_vae_2d=False,
|
||||
cal_loss=False,
|
||||
force_huggingface=False,
|
||||
):
|
||||
vae_2d = dict(
|
||||
type="VideoAutoencoderKL",
|
||||
|
|
@ -277,7 +277,7 @@ def OpenSoraVAE_V1_2(
|
|||
scale=scale,
|
||||
)
|
||||
|
||||
if from_pretrained is not None and not os.path.isdir(from_pretrained):
|
||||
if force_huggingface or (from_pretrained is not None and not os.path.isdir(from_pretrained)):
|
||||
model = VideoAutoencoderPipeline.from_pretrained(from_pretrained, **kwargs)
|
||||
else:
|
||||
config = VideoAutoencoderPipelineConfig(**kwargs)
|
||||
|
|
|
|||
Loading…
Reference in a new issue