diff --git a/eval/sample.sh b/eval/sample.sh index 1c52506..eff1368 100644 --- a/eval/sample.sh +++ b/eval/sample.sh @@ -260,10 +260,17 @@ function run_vbench() { --resolution $VBENCH_RES --aspect-ratio $VBENCH_ASP_RATIO --num-sampling-steps ${NUM_SAMPLING_STEPS} --flow ${FLOW} \ --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --start-index $1 --end-index $2 else - eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \ - --prompt-path assets/texts/VBench/all_dimension.txt \ - --resolution $VBENCH_RES --aspect-ratio $VBENCH_ASP_RATIO --num-sampling-steps ${NUM_SAMPLING_STEPS} --flow ${FLOW} --llm-refine ${LLM_REFINE} \ - --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --start-index $1 --end-index $2 + if [ "${FLOW}" = "None" ]; then + eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \ + --prompt-path assets/texts/VBench/all_dimension.txt \ + --resolution $VBENCH_RES --aspect-ratio $VBENCH_ASP_RATIO --num-sampling-steps ${NUM_SAMPLING_STEPS} --llm-refine ${LLM_REFINE} \ + --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --start-index $1 --end-index $2 + else + eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \ + --prompt-path assets/texts/VBench/all_dimension.txt \ + --resolution $VBENCH_RES --aspect-ratio $VBENCH_ASP_RATIO --num-sampling-steps ${NUM_SAMPLING_STEPS} --flow ${FLOW} --llm-refine ${LLM_REFINE} \ + --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --start-index $1 --end-index $2 + fi fi fi fi diff --git a/eval/vbench/calc_vbench.py b/eval/vbench/calc_vbench.py index b2505bf..e5570a1 100644 --- a/eval/vbench/calc_vbench.py +++ b/eval/vbench/calc_vbench.py @@ -1,37 +1,46 @@ import argparse import os -from vbench import VBench -import torch import time +import torch + +from vbench import VBench + full_info_path = "eval/vbench/VBench_full_info.json" dimensions = [ - # Quality Score - "subject_consistency", - "background_consistency", - "motion_smoothness", - "dynamic_degree", - "aesthetic_quality", - "imaging_quality", - "temporal_flickering", - # Semantic Score - "object_class", - "multiple_objects", - "color", - "spatial_relationship", - "scene", - "temporal_style", - "overall_consistency", - "human_action", - "appearance_style", + # a: 10min + "subject_consistency", # 4min + "imaging_quality", # 6min + # b: 12min + "background_consistency", # 2min + "motion_smoothness", # 5min + "overall_consistency", # 2min + "human_action", # 3min + # c: 14min + "multiple_objects", # 14min + # d: 14min + "spatial_relationship", # 14min + # e: 12min + "object_class", # 12min + # f: 12min + "color", # 12min + # g: 10.5min + "aesthetic_quality", # 2.5min + "appearance_style", # 6min + "temporal_flickering", # 2min + # h: 9min + "scene", # 3min + "temporal_style", # 2min + "dynamic_degree", # 4min ] + def parse_args(): parser = argparse.ArgumentParser() parser.add_argument("video_folder", type=str) # samples/samples..._vbench/eval parser.add_argument("model_ckpt", type=str) - parser.add_argument("--start", type=int, default=0) # start index of dimension to be evaluated - parser.add_argument("--end", type=int, default=-1) # start index of dimension to be evaluated + parser.add_argument("--start", type=int, default=0) # start index of dimension to be evaluated + parser.add_argument("--end", type=int, default=-1) # start index of dimension to be evaluated args = parser.parse_args() return args @@ -44,23 +53,23 @@ if __name__ == "__main__": video_path = args.video_folder kwargs = {} - kwargs['imaging_quality_preprocessing_mode'] = 'longer' # use VBench/evaluate.py default + kwargs["imaging_quality_preprocessing_mode"] = "longer" # use VBench/evaluate.py default start_time = time.time() # NOTE: important to use torch.device("cuda"), else will have issue with object_class third_party module my_VBench = VBench(torch.device("cuda"), full_info_path, output_dir) - if args.end == -1: # adjust end accordingly + if args.end == -1: # adjust end accordingly args.end = len(dimensions) - for dim in dimensions[args.start:args.end]: + for dim in dimensions[args.start : args.end]: my_VBench.evaluate( videos_path=video_path, name=dim, local=False, read_frame=False, dimension_list=[dim], - mode='vbench_standard', - **kwargs + mode="vbench_standard", + **kwargs, ) - print("Runtime: %s seconds " % (time.time() - start_time)) \ No newline at end of file + print("Runtime: %s seconds " % (time.time() - start_time)) diff --git a/eval/vbench/launch_calc.sh b/eval/vbench/launch_calc.sh index 53114b9..9f14ce5 100644 --- a/eval/vbench/launch_calc.sh +++ b/eval/vbench/launch_calc.sh @@ -7,11 +7,10 @@ mkdir -p $LOG_BASE echo "Logging to $LOG_BASE" GPUS=(0 1 2 3 4 5 6 7) -START_INDEX_LIST=(0 2 4 6 8 10 12 14) -END_INDEX_LIST=(2 4 6 8 10 12 14 16) +START_INDEX_LIST=(0 2 6 7 8 9 10 13) +END_INDEX_LIST=(2 6 7 8 9 10 13 16) TASK_ID_LIST=(calc_vbench_a calc_vbench_b calc_vbench_c calc_vbench_d calc_vbench_e calc_vbench_f calc_vbench_g calc_vbench_h) # for log records only - for i in "${!GPUS[@]}"; do CUDA_VISIBLE_DEVICES=${GPUS[i]} python eval/vbench/calc_vbench.py $VIDEO_DIR $CKPT_DIR --start ${START_INDEX_LIST[i]} --end ${END_INDEX_LIST[i]} > ${LOG_BASE}/${TASK_ID_LIST[i]}.log 2>&1 & done