mirror of
https://github.com/hpcaitech/Open-Sora.git
synced 2026-04-12 05:46:22 +02:00
format
This commit is contained in:
Shen-Chenhui
parent
01fee497f3
commit
53b9ccba1d
|
|
@ -71,14 +71,34 @@ pip install -v .[eval]
|
|||
|
||||
### Step 2: Install VBench
|
||||
|
||||
You need to manually install [VBench](https://github.com/Vchitect/VBench):
|
||||
<!-- You need to manually install [VBench](https://github.com/Vchitect/VBench):
|
||||
|
||||
```bash
|
||||
pip install --no-deps vbench==0.1.2
|
||||
pip install --no-deps vbench==0.1.1
|
||||
# If the installation shows a warning about the intalled vbench not in PATH, you need to add it by:
|
||||
export PATH="/path/to/vbench:$PATH"
|
||||
``` -->
|
||||
|
||||
You need to install VBench mannually by:
|
||||
```bash
|
||||
# first clone their repo
|
||||
cd ..
|
||||
git clone https://github.com/Vchitect/VBench.git
|
||||
cd VBench
|
||||
git checkout v0.1.2
|
||||
|
||||
# next, fix their hard-coded path isse
|
||||
vim vbench2_beta_i2v/utils.py
|
||||
# find `image_root` in the `load_i2v_dimension_info` function, change it to point to your appropriate image folder
|
||||
|
||||
# last, create softlinks
|
||||
cd Open-Sora # or `cd Open-Sora-dev` for development
|
||||
ln -s ../VBench/vbench vbench
|
||||
ln -s ../VBench/vbench2_beta_i2v vbench2_beta_i2v
|
||||
# later you need to make sure to call evaluatio from your Open-Sora folder, else vbench, vbench2_beta_i2v cannot be found
|
||||
```
|
||||
|
||||
|
||||
### Step 3: Install `cupy` for Potential VAE Errors
|
||||
|
||||
You need to mannually install [cupy](https://docs.cupy.dev/en/stable/install.html).
|
||||
|
|
|
|||
|
|
@ -54,8 +54,12 @@ bash eval/vbench/launch.sh /path/to/ckpt num_frames model_name
|
|||
|
||||
After generation, install the VBench package following our [installation](../docs/installation.md)'s sections of "Evaluation Dependencies". Then, run the following commands to evaluate the generated samples.
|
||||
|
||||
```bash
|
||||
<!-- ```bash
|
||||
bash eval/vbench/vbench.sh /path/to/video_folder /path/to/model/ckpt
|
||||
``` -->
|
||||
|
||||
```bash
|
||||
python eval/vbench/calc_vbench.py /path/to/video_folder /path/to/model/ckpt
|
||||
```
|
||||
|
||||
Finally, we obtain the scaled scores for the model by:
|
||||
|
|
@ -66,7 +70,7 @@ python eval/vbench/tabulate_vbench_scores.py --score_dir path/to/score/dir
|
|||
## VBench-i2v
|
||||
|
||||
[VBench-i2v](https://github.com/Vchitect/VBench/tree/master/vbench2_beta_i2v) is a benchmark for short image to video generation (beta version).
|
||||
Similarly, install the VBench package following our [installation](../docs/installation.md)'s sections of "Evaluation Dependencies". Then, run the following commands to evaluate the generated samples.
|
||||
|
||||
|
||||
```bash
|
||||
# Step 1: generate the relevant videos
|
||||
|
|
@ -76,12 +80,12 @@ bash eval/sample.sh /path/to/ckpt num_frames model_name_for_log -5a
|
|||
bash eval/vbench_i2v/launch.sh /path/to/ckpt num_frames model_name
|
||||
|
||||
# Step 2: run vbench to evaluate the generated samples
|
||||
python eval/vbench_i2v/vbench_i2v.py
|
||||
python eval/vbench_i2v/vbench_i2v.py /path/to/video_folder /path/to/model/ckpt
|
||||
# Note that if you need to go to `VBench/vbench2_beta_i2v/utils.py` and change the harded-coded var `image_root` in the `load_i2v_dimension_info` function to your corresponding image folder.
|
||||
|
||||
# Step 3: obtain the scaled scores
|
||||
def load_i2v_dimension_info(json_dir, dimension, lang, resolution):
|
||||
# Note that if you need to go to `your_conda_env_path/lib/python3.x/site-packages/vbench2_beta_i2v/utils.py` and change the harded-coded var `image_root` in the `load_i2v_dimension_info` function to your appropriate image folder.
|
||||
python eval/vbench_i2v/tabulate_vbench_i2v_scores.py --score_dir path/to/evaluation_results/dir
|
||||
python eval/vbench_i2v/tabulate_vbench_i2v_scores.py path/to/videos/folder path/to/your/model/ckpt
|
||||
# this will store the results under `eval/vbench_i2v` in the path/to/your/model/ckpt
|
||||
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@ from ast import literal_eval
|
|||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--log_dir", type=str)
|
||||
parser.add_argument("--ckpt_name", type=str)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
|
@ -30,7 +29,8 @@ if __name__ == "__main__":
|
|||
os.makedirs(output_dir)
|
||||
|
||||
files = os.listdir(args.log_dir)
|
||||
files = ["img.log", "video_144p.log", "video_240p.log", "video_360p.log", "video_480p.log", "video_720p.log"]
|
||||
# files = ["img.log", "video_144p.log", "video_240p.log", "video_360p.log", "video_480p.log", "video_720p.log"]
|
||||
files = ["img.log", "144p_vid.log", "240p_vid.log", "360p_vid.log", "480p_vid.log", "720p_vid.log"]
|
||||
|
||||
loss_info = {}
|
||||
|
||||
|
|
@ -47,7 +47,7 @@ if __name__ == "__main__":
|
|||
loss_info[resolution][frame] = format(loss, ".4f")
|
||||
|
||||
# Convert and write JSON object to file
|
||||
output_file_path = os.path.join(output_dir, args.ckpt_name + "_loss.json")
|
||||
output_file_path = os.path.join(output_dir, "loss.json")
|
||||
with open(output_file_path, "w") as outfile:
|
||||
json.dump(loss_info, outfile, indent=4, sort_keys=True)
|
||||
print(f"results saved to: {output_file_path}")
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ set -e
|
|||
# Base path for videos
|
||||
videos_path=$1
|
||||
model_path=$2
|
||||
output_path=${model_path}/eval/vbench
|
||||
output_path=$(dirname $model_path)/eval/vbench
|
||||
mkdir -p ${output_path}
|
||||
# videos_base=$(basename $videos_path)
|
||||
json_path=./eval/vbench/VBench_full_info.json
|
||||
|
|
|
|||
45
eval/vbench_i2v/calc_vbench_i2v.py
Normal file
45
eval/vbench_i2v/calc_vbench_i2v.py
Normal file
|
|
@ -0,0 +1,45 @@
|
|||
import argparse
|
||||
import os
|
||||
|
||||
from vbench import VBench
|
||||
from vbench2_beta_i2v import VBenchI2V
|
||||
|
||||
full_info_path = "eval/vbench_i2v/vbench2_i2v_full_info.json"
|
||||
video_quality_dimensions = [
|
||||
"subject_consistency",
|
||||
"background_consistency",
|
||||
"motion_smoothness",
|
||||
"dynamic_degree",
|
||||
"aesthetic_quality",
|
||||
"imaging_quality",
|
||||
"temporal_flickering",
|
||||
]
|
||||
i2v_dimensions = ["i2v_subject", "i2v_background", "camera_motion"]
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("video_folder", type=str) # samples/samples..._vbench_i2v/
|
||||
parser.add_argument("model_ckpt", type=str)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
output_dir = os.path.join(args.model_ckpt, "eval", "vbench_i2v")
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
video_path = args.video_folder
|
||||
|
||||
my_VBench_I2V = VBenchI2V("cuda", full_info_path, output_dir)
|
||||
for i2v_dim in i2v_dimensions:
|
||||
my_VBench_I2V.evaluate(videos_path=video_path, name=i2v_dim, dimension_list=[i2v_dim], resolution="1-1")
|
||||
|
||||
kwargs = {}
|
||||
kwargs["imaging_quality_preprocessing_mode"] = "longer" # use VBench/evaluate.py default
|
||||
|
||||
my_VBench = VBench("cuda", full_info_path, output_dir)
|
||||
for quality_dim in video_quality_dimensions:
|
||||
my_VBench.evaluate(
|
||||
videos_path=video_path, name=quality_dim, dimension_list=[quality_dim], mode="vbench_standard", **kwargs
|
||||
)
|
||||
|
|
@ -6,44 +6,44 @@ I2V_WEIGHT = 1.0
|
|||
I2V_QUALITY_WEIGHT = 1.0
|
||||
|
||||
I2V_LIST = [
|
||||
"Video-Image Subject Consistency",
|
||||
"Video-Image Background Consistency",
|
||||
"i2v_subject",
|
||||
"i2v_background",
|
||||
]
|
||||
|
||||
I2V_QUALITY_LIST = [
|
||||
"Subject Consistency",
|
||||
"Background Consistency",
|
||||
"Motion Smoothness",
|
||||
"Dynamic Degree",
|
||||
"Aesthetic Quality",
|
||||
"Imaging Quality",
|
||||
"Temporal Flickering",
|
||||
"subject_consistency",
|
||||
"background_consistency",
|
||||
"temporal_flickering",
|
||||
"motion_smoothness",
|
||||
"aesthetic_quality",
|
||||
"imaging_quality",
|
||||
"dynamic_degree",
|
||||
]
|
||||
|
||||
DIM_WEIGHT_I2V = {
|
||||
"Video-Text Camera Motion": 0.1,
|
||||
"Video-Image Subject Consistency": 1,
|
||||
"Video-Image Background Consistency": 1,
|
||||
"Subject Consistency": 1,
|
||||
"Background Consistency": 1,
|
||||
"Motion Smoothness": 1,
|
||||
"Dynamic Degree": 0.5,
|
||||
"Aesthetic Quality": 1,
|
||||
"Imaging Quality": 1,
|
||||
"Temporal Flickering": 1,
|
||||
"camera_motion": 0.1,
|
||||
"i2v_subject": 1,
|
||||
"i2v_background_consistency": 1,
|
||||
"subject_consistency": 1,
|
||||
"background_consistency": 1,
|
||||
"motion_smoothness": 1,
|
||||
"dynamic_degree": 0.5,
|
||||
"aesthetic_quality": 1,
|
||||
"imaging_quality": 1,
|
||||
"temporal_flickering": 1,
|
||||
}
|
||||
|
||||
NORMALIZE_DIC_I2V = {
|
||||
"Video-Text Camera Motion": {"Min": 0.0, "Max": 1.0},
|
||||
"Video-Image Subject Consistency": {"Min": 0.1462, "Max": 1.0},
|
||||
"Video-Image Background Consistency": {"Min": 0.2615, "Max": 1.0},
|
||||
"Subject Consistency": {"Min": 0.1462, "Max": 1.0},
|
||||
"Background Consistency": {"Min": 0.2615, "Max": 1.0},
|
||||
"Motion Smoothness": {"Min": 0.7060, "Max": 0.9975},
|
||||
"Dynamic Degree": {"Min": 0.0, "Max": 1.0},
|
||||
"Aesthetic Quality": {"Min": 0.0, "Max": 1.0},
|
||||
"Imaging Quality": {"Min": 0.0, "Max": 1.0},
|
||||
"Temporal Flickering": {"Min": 0.6293, "Max": 1.0},
|
||||
"camera_motion": {"Min": 0.0, "Max": 1.0},
|
||||
"i2v_subject": {"Min": 0.1462, "Max": 1.0},
|
||||
"i2v_background": {"Min": 0.2615, "Max": 1.0},
|
||||
"subject_consistency": {"Min": 0.1462, "Max": 1.0},
|
||||
"background_consistency": {"Min": 0.2615, "Max": 1.0},
|
||||
"motion_smoothness": {"Min": 0.7060, "Max": 0.9975},
|
||||
"dynamic_degree": {"Min": 0.0, "Max": 1.0},
|
||||
"aesthetic_quality": {"Min": 0.0, "Max": 1.0},
|
||||
"imaging_quality": {"Min": 0.0, "Max": 1.0},
|
||||
"temporal_flickering": {"Min": 0.6293, "Max": 1.0},
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,41 +0,0 @@
|
|||
import argparse
|
||||
import os
|
||||
|
||||
from vbench2_beta_i2v import VBenchI2V
|
||||
|
||||
FULL_INFO_PATH = "eval/vbench_i2v/vbench2_i2v_full_info.json"
|
||||
VIDEO_QUALITY_DIMENSIONS = [
|
||||
"subject_consistency",
|
||||
"background_consistency",
|
||||
"motion_smoothness",
|
||||
"dynamic_degree",
|
||||
"aesthetic_quality",
|
||||
"imaging_quality",
|
||||
"temporal_flickering",
|
||||
]
|
||||
I2V_DIMENSIONS = ["i2v_subject", "i2v_background", "camera_motion"]
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("video_folder", type=str) # samples/samples..._vbench_i2v/
|
||||
parser.add_argument("model_ckpt", type=str)
|
||||
args = parser.parse_args()
|
||||
return args
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
args = parse_args()
|
||||
output_dir = os.path.join(args.model_ckpt, "eval", "vbench_i2v")
|
||||
os.makedirs(output_dir, exist_ok=True)
|
||||
video_path = args.video_folder
|
||||
|
||||
my_VBench_I2V = VBenchI2V("cuda", FULL_INFO_PATH, output_dir)
|
||||
my_VBench_I2V.evaluate(videos_path=video_path, name="vbench_i2v", dimension_list=I2V_DIMENSIONS, resolution="1-1")
|
||||
|
||||
# my_VBench = VBench("cuda", FULL_INFO_PATH, output_dir)
|
||||
# my_VBench.evaluate(
|
||||
# videos_path=video_path,
|
||||
# name="vbench_video_quality",
|
||||
# dimension_list=VIDEO_QUALITY_DIMENSIONS,
|
||||
# )
|
||||
Loading…
Reference in a new issue