From 53b9ccba1d14208be5d0bfe8cf3eae21496551da Mon Sep 17 00:00:00 2001 From: Shen-Chenhui Date: Thu, 6 Jun 2024 07:06:26 +0000 Subject: [PATCH] format --- docs/installation.md | 24 +++++++- eval/README.md | 16 +++-- eval/loss/tabulate_rl_loss.py | 6 +- eval/vbench/vbench.sh | 2 +- eval/vbench_i2v/calc_vbench_i2v.py | 45 ++++++++++++++ eval/vbench_i2v/tabulate_vbench_i2v_scores.py | 58 +++++++++---------- eval/vbench_i2v/vbench_i2v.py | 41 ------------- 7 files changed, 110 insertions(+), 82 deletions(-) create mode 100644 eval/vbench_i2v/calc_vbench_i2v.py delete mode 100644 eval/vbench_i2v/vbench_i2v.py diff --git a/docs/installation.md b/docs/installation.md index 71dae21..e17d0ef 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -71,14 +71,34 @@ pip install -v .[eval] ### Step 2: Install VBench -You need to manually install [VBench](https://github.com/Vchitect/VBench): + + +You need to install VBench mannually by: +```bash +# first clone their repo +cd .. +git clone https://github.com/Vchitect/VBench.git +cd VBench +git checkout v0.1.2 + +# next, fix their hard-coded path isse +vim vbench2_beta_i2v/utils.py +# find `image_root` in the `load_i2v_dimension_info` function, change it to point to your appropriate image folder + +# last, create softlinks +cd Open-Sora # or `cd Open-Sora-dev` for development +ln -s ../VBench/vbench vbench +ln -s ../VBench/vbench2_beta_i2v vbench2_beta_i2v +# later you need to make sure to call evaluatio from your Open-Sora folder, else vbench, vbench2_beta_i2v cannot be found ``` + ### Step 3: Install `cupy` for Potential VAE Errors You need to mannually install [cupy](https://docs.cupy.dev/en/stable/install.html). diff --git a/eval/README.md b/eval/README.md index 15852bf..9bc70d0 100644 --- a/eval/README.md +++ b/eval/README.md @@ -54,8 +54,12 @@ bash eval/vbench/launch.sh /path/to/ckpt num_frames model_name After generation, install the VBench package following our [installation](../docs/installation.md)'s sections of "Evaluation Dependencies". Then, run the following commands to evaluate the generated samples. -```bash + + +```bash +python eval/vbench/calc_vbench.py /path/to/video_folder /path/to/model/ckpt ``` Finally, we obtain the scaled scores for the model by: @@ -66,7 +70,7 @@ python eval/vbench/tabulate_vbench_scores.py --score_dir path/to/score/dir ## VBench-i2v [VBench-i2v](https://github.com/Vchitect/VBench/tree/master/vbench2_beta_i2v) is a benchmark for short image to video generation (beta version). -Similarly, install the VBench package following our [installation](../docs/installation.md)'s sections of "Evaluation Dependencies". Then, run the following commands to evaluate the generated samples. + ```bash # Step 1: generate the relevant videos @@ -76,12 +80,12 @@ bash eval/sample.sh /path/to/ckpt num_frames model_name_for_log -5a bash eval/vbench_i2v/launch.sh /path/to/ckpt num_frames model_name # Step 2: run vbench to evaluate the generated samples -python eval/vbench_i2v/vbench_i2v.py +python eval/vbench_i2v/vbench_i2v.py /path/to/video_folder /path/to/model/ckpt +# Note that if you need to go to `VBench/vbench2_beta_i2v/utils.py` and change the harded-coded var `image_root` in the `load_i2v_dimension_info` function to your corresponding image folder. # Step 3: obtain the scaled scores -def load_i2v_dimension_info(json_dir, dimension, lang, resolution): -# Note that if you need to go to `your_conda_env_path/lib/python3.x/site-packages/vbench2_beta_i2v/utils.py` and change the harded-coded var `image_root` in the `load_i2v_dimension_info` function to your appropriate image folder. -python eval/vbench_i2v/tabulate_vbench_i2v_scores.py --score_dir path/to/evaluation_results/dir +python eval/vbench_i2v/tabulate_vbench_i2v_scores.py path/to/videos/folder path/to/your/model/ckpt +# this will store the results under `eval/vbench_i2v` in the path/to/your/model/ckpt ``` diff --git a/eval/loss/tabulate_rl_loss.py b/eval/loss/tabulate_rl_loss.py index d9b3a19..48c930e 100644 --- a/eval/loss/tabulate_rl_loss.py +++ b/eval/loss/tabulate_rl_loss.py @@ -15,7 +15,6 @@ from ast import literal_eval def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("--log_dir", type=str) - parser.add_argument("--ckpt_name", type=str) args = parser.parse_args() return args @@ -30,7 +29,8 @@ if __name__ == "__main__": os.makedirs(output_dir) files = os.listdir(args.log_dir) - files = ["img.log", "video_144p.log", "video_240p.log", "video_360p.log", "video_480p.log", "video_720p.log"] + # files = ["img.log", "video_144p.log", "video_240p.log", "video_360p.log", "video_480p.log", "video_720p.log"] + files = ["img.log", "144p_vid.log", "240p_vid.log", "360p_vid.log", "480p_vid.log", "720p_vid.log"] loss_info = {} @@ -47,7 +47,7 @@ if __name__ == "__main__": loss_info[resolution][frame] = format(loss, ".4f") # Convert and write JSON object to file - output_file_path = os.path.join(output_dir, args.ckpt_name + "_loss.json") + output_file_path = os.path.join(output_dir, "loss.json") with open(output_file_path, "w") as outfile: json.dump(loss_info, outfile, indent=4, sort_keys=True) print(f"results saved to: {output_file_path}") diff --git a/eval/vbench/vbench.sh b/eval/vbench/vbench.sh index 3e1ff49..9b66791 100644 --- a/eval/vbench/vbench.sh +++ b/eval/vbench/vbench.sh @@ -5,7 +5,7 @@ set -e # Base path for videos videos_path=$1 model_path=$2 -output_path=${model_path}/eval/vbench +output_path=$(dirname $model_path)/eval/vbench mkdir -p ${output_path} # videos_base=$(basename $videos_path) json_path=./eval/vbench/VBench_full_info.json diff --git a/eval/vbench_i2v/calc_vbench_i2v.py b/eval/vbench_i2v/calc_vbench_i2v.py new file mode 100644 index 0000000..fbbdd11 --- /dev/null +++ b/eval/vbench_i2v/calc_vbench_i2v.py @@ -0,0 +1,45 @@ +import argparse +import os + +from vbench import VBench +from vbench2_beta_i2v import VBenchI2V + +full_info_path = "eval/vbench_i2v/vbench2_i2v_full_info.json" +video_quality_dimensions = [ + "subject_consistency", + "background_consistency", + "motion_smoothness", + "dynamic_degree", + "aesthetic_quality", + "imaging_quality", + "temporal_flickering", +] +i2v_dimensions = ["i2v_subject", "i2v_background", "camera_motion"] + + +def parse_args(): + parser = argparse.ArgumentParser() + parser.add_argument("video_folder", type=str) # samples/samples..._vbench_i2v/ + parser.add_argument("model_ckpt", type=str) + args = parser.parse_args() + return args + + +if __name__ == "__main__": + args = parse_args() + output_dir = os.path.join(args.model_ckpt, "eval", "vbench_i2v") + os.makedirs(output_dir, exist_ok=True) + video_path = args.video_folder + + my_VBench_I2V = VBenchI2V("cuda", full_info_path, output_dir) + for i2v_dim in i2v_dimensions: + my_VBench_I2V.evaluate(videos_path=video_path, name=i2v_dim, dimension_list=[i2v_dim], resolution="1-1") + + kwargs = {} + kwargs["imaging_quality_preprocessing_mode"] = "longer" # use VBench/evaluate.py default + + my_VBench = VBench("cuda", full_info_path, output_dir) + for quality_dim in video_quality_dimensions: + my_VBench.evaluate( + videos_path=video_path, name=quality_dim, dimension_list=[quality_dim], mode="vbench_standard", **kwargs + ) diff --git a/eval/vbench_i2v/tabulate_vbench_i2v_scores.py b/eval/vbench_i2v/tabulate_vbench_i2v_scores.py index 758171c..fb6a518 100644 --- a/eval/vbench_i2v/tabulate_vbench_i2v_scores.py +++ b/eval/vbench_i2v/tabulate_vbench_i2v_scores.py @@ -6,44 +6,44 @@ I2V_WEIGHT = 1.0 I2V_QUALITY_WEIGHT = 1.0 I2V_LIST = [ - "Video-Image Subject Consistency", - "Video-Image Background Consistency", + "i2v_subject", + "i2v_background", ] I2V_QUALITY_LIST = [ - "Subject Consistency", - "Background Consistency", - "Motion Smoothness", - "Dynamic Degree", - "Aesthetic Quality", - "Imaging Quality", - "Temporal Flickering", + "subject_consistency", + "background_consistency", + "temporal_flickering", + "motion_smoothness", + "aesthetic_quality", + "imaging_quality", + "dynamic_degree", ] DIM_WEIGHT_I2V = { - "Video-Text Camera Motion": 0.1, - "Video-Image Subject Consistency": 1, - "Video-Image Background Consistency": 1, - "Subject Consistency": 1, - "Background Consistency": 1, - "Motion Smoothness": 1, - "Dynamic Degree": 0.5, - "Aesthetic Quality": 1, - "Imaging Quality": 1, - "Temporal Flickering": 1, + "camera_motion": 0.1, + "i2v_subject": 1, + "i2v_background_consistency": 1, + "subject_consistency": 1, + "background_consistency": 1, + "motion_smoothness": 1, + "dynamic_degree": 0.5, + "aesthetic_quality": 1, + "imaging_quality": 1, + "temporal_flickering": 1, } NORMALIZE_DIC_I2V = { - "Video-Text Camera Motion": {"Min": 0.0, "Max": 1.0}, - "Video-Image Subject Consistency": {"Min": 0.1462, "Max": 1.0}, - "Video-Image Background Consistency": {"Min": 0.2615, "Max": 1.0}, - "Subject Consistency": {"Min": 0.1462, "Max": 1.0}, - "Background Consistency": {"Min": 0.2615, "Max": 1.0}, - "Motion Smoothness": {"Min": 0.7060, "Max": 0.9975}, - "Dynamic Degree": {"Min": 0.0, "Max": 1.0}, - "Aesthetic Quality": {"Min": 0.0, "Max": 1.0}, - "Imaging Quality": {"Min": 0.0, "Max": 1.0}, - "Temporal Flickering": {"Min": 0.6293, "Max": 1.0}, + "camera_motion": {"Min": 0.0, "Max": 1.0}, + "i2v_subject": {"Min": 0.1462, "Max": 1.0}, + "i2v_background": {"Min": 0.2615, "Max": 1.0}, + "subject_consistency": {"Min": 0.1462, "Max": 1.0}, + "background_consistency": {"Min": 0.2615, "Max": 1.0}, + "motion_smoothness": {"Min": 0.7060, "Max": 0.9975}, + "dynamic_degree": {"Min": 0.0, "Max": 1.0}, + "aesthetic_quality": {"Min": 0.0, "Max": 1.0}, + "imaging_quality": {"Min": 0.0, "Max": 1.0}, + "temporal_flickering": {"Min": 0.6293, "Max": 1.0}, } diff --git a/eval/vbench_i2v/vbench_i2v.py b/eval/vbench_i2v/vbench_i2v.py deleted file mode 100644 index ac82ab9..0000000 --- a/eval/vbench_i2v/vbench_i2v.py +++ /dev/null @@ -1,41 +0,0 @@ -import argparse -import os - -from vbench2_beta_i2v import VBenchI2V - -FULL_INFO_PATH = "eval/vbench_i2v/vbench2_i2v_full_info.json" -VIDEO_QUALITY_DIMENSIONS = [ - "subject_consistency", - "background_consistency", - "motion_smoothness", - "dynamic_degree", - "aesthetic_quality", - "imaging_quality", - "temporal_flickering", -] -I2V_DIMENSIONS = ["i2v_subject", "i2v_background", "camera_motion"] - - -def parse_args(): - parser = argparse.ArgumentParser() - parser.add_argument("video_folder", type=str) # samples/samples..._vbench_i2v/ - parser.add_argument("model_ckpt", type=str) - args = parser.parse_args() - return args - - -if __name__ == "__main__": - args = parse_args() - output_dir = os.path.join(args.model_ckpt, "eval", "vbench_i2v") - os.makedirs(output_dir, exist_ok=True) - video_path = args.video_folder - - my_VBench_I2V = VBenchI2V("cuda", FULL_INFO_PATH, output_dir) - my_VBench_I2V.evaluate(videos_path=video_path, name="vbench_i2v", dimension_list=I2V_DIMENSIONS, resolution="1-1") - - # my_VBench = VBench("cuda", FULL_INFO_PATH, output_dir) - # my_VBench.evaluate( - # videos_path=video_path, - # name="vbench_video_quality", - # dimension_list=VIDEO_QUALITY_DIMENSIONS, - # )