Open-Sora/eval/vbench_i2v/calc_vbench_i2v.py

import argparse
import os
import time

import torch
from vbench import VBench
from vbench2_beta_i2v import VBenchI2V

full_info_path = "eval/vbench_i2v/vbench2_i2v_full_info.json"
video_quality_dimensions = [
    "subject_consistency",
    "background_consistency",
    "motion_smoothness",
    "dynamic_degree",
    "aesthetic_quality",
    "imaging_quality",
    "temporal_flickering",
]
i2v_dimensions = ["i2v_subject", "i2v_background", "camera_motion"]


def str2bool(v):
    if isinstance(v, bool):
        return v
    if v.lower() in ("yes", "true", "t", "y", "1"):
        return True
    elif v.lower() in ("no", "false", "f", "n", "0"):
        return False
    else:
        raise argparse.ArgumentTypeError("Boolean value expected.")


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument("video_folder", type=str)  # samples/samples..._vbench_i2v/
    parser.add_argument("model_ckpt", type=str)
    parser.add_argument("--start", type=int, default=0)  # start index of dimension to be evaluated
    parser.add_argument("--end", type=int, default=-1)  # start index of dimension to be evaluated
    parser.add_argument("--calc_i2v", type=str2bool, default=True)
    parser.add_argument("--calc_quality", type=str2bool, default=True)
    args = parser.parse_args()
    return args


if __name__ == "__main__":
    args = parse_args()
    output_dir = os.path.join(args.model_ckpt, "vbench_i2v")
    os.makedirs(output_dir, exist_ok=True)
    video_path = args.video_folder

    start_time = time.time()

    if args.calc_i2v:
        my_VBench_I2V = VBenchI2V(torch.device("cuda"), full_info_path, output_dir)
        end = args.end if args.end != -1 else len(i2v_dimensions)
        for i2v_dim in i2v_dimensions[args.start : end]:
            my_VBench_I2V.evaluate(videos_path=video_path, name=i2v_dim, dimension_list=[i2v_dim], resolution="1-1")

    kwargs = {}
    kwargs["imaging_quality_preprocessing_mode"] = "longer"  # use VBench/evaluate.py default

    if args.calc_quality:
        my_VBench = VBench(torch.device("cuda"), full_info_path, output_dir)
        end = args.end if args.end != -1 else len(video_quality_dimensions)
        for quality_dim in video_quality_dimensions[args.start : end]:
            my_VBench.evaluate(
                videos_path=video_path, name=quality_dim, dimension_list=[quality_dim], mode="vbench_standard", **kwargs
            )

    print("Runtime: %s seconds " % (time.time() - start_time))