diff --git a/.gitignore b/.gitignore index 14dc0e3..ebf9f97 100644 --- a/.gitignore +++ b/.gitignore @@ -170,6 +170,7 @@ dataset/ runs/ checkpoints/ outputs/ +outputs samples/ samples logs/ diff --git a/eval/sample.sh b/eval/sample.sh index 5d94d23..0eae70d 100644 --- a/eval/sample.sh +++ b/eval/sample.sh @@ -160,55 +160,54 @@ function run_video_h() { # 23min # vbench has 950 samples VBENCH_BS=1 # 80GB -VBENCH_FRAMES=51 VBENCH_H=240 VBENCH_W=426 function run_vbenck_a() { # 2h eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \ --prompt-path assets/texts/VBench/all_dimension.txt \ - --batch-size $VBENCH_BS --num-frames $VBENCH_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 0 --end-index 120 + --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 0 --end-index 120 } function run_vbenck_b() { # 2h eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \ - --prompt-path assets/texts/VBench/all_dimension.txt --batch-size $VBENCH_BS --num-frames $VBENCH_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 120 --end-index 240 + --prompt-path assets/texts/VBench/all_dimension.txt --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 120 --end-index 240 } function run_vbenck_c() { # 2h eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \ --prompt-path assets/texts/VBench/all_dimension.txt \ - --batch-size $VBENCH_BS --num-frames $VBENCH_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 240 --end-index 360 + --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 240 --end-index 360 } function run_vbenck_d() { # 2h eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \ --prompt-path assets/texts/VBench/all_dimension.txt \ - --batch-size $VBENCH_BS --num-frames $VBENCH_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 360 --end-index 480 + --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 360 --end-index 480 } function run_vbenck_e() { # 2h eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \ --prompt-path assets/texts/VBench/all_dimension.txt \ - --batch-size $VBENCH_BS --num-frames $VBENCH_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 480 --end-index 600 + --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 480 --end-index 600 } function run_vbenck_f() { # 2h eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \ --prompt-path assets/texts/VBench/all_dimension.txt \ - --batch-size $VBENCH_BS --num-frames $VBENCH_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 600 --end-index 720 + --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 600 --end-index 720 } function run_vbenck_g() { # 2h eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \ --prompt-path assets/texts/VBench/all_dimension.txt \ - --batch-size $VBENCH_BS --num-frames $VBENCH_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 720 --end-index 840 + --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 720 --end-index 840 } function run_vbenck_h() { # 2h eval $CMD --ckpt-path $CKPT --save-dir ${OUTPUT}_vbench --prompt-as-path --num-sample 5 \ --prompt-path assets/texts/VBench/all_dimension.txt \ - --batch-size $VBENCH_BS --num-frames $VBENCH_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 840 + --batch-size $VBENCH_BS --num-frames $NUM_FRAMES --image-size $VBENCH_H $VBENCH_W --start-index 840 } # vbench-i2v has 1120 samples diff --git a/eval/vbench/launch.sh b/eval/vbench/launch.sh index acb99aa..0ee4009 100644 --- a/eval/vbench/launch.sh +++ b/eval/vbench/launch.sh @@ -5,6 +5,7 @@ set -e CKPT=$1 MODEL_NAME=$2 +NUM_FRAMES=51 if [[ $CKPT == *"ema"* ]]; then parentdir=$(dirname $CKPT) @@ -19,5 +20,5 @@ GPUS=(0 1 2 3 4 5 6 7) TASK_ID_LIST=(4a 4b 4c 4d 4e 4f 4g 4h) for i in "${!GPUS[@]}"; do - CUDA_VISIBLE_DEVICES=${GPUS[i]} bash eval/sample.sh $CKPT -${TASK_ID_LIST[i]} >${LOG_BASE}_${TASK_ID_LIST[i]}.log 2>&1 & + CUDA_VISIBLE_DEVICES=${GPUS[i]} bash eval/sample.sh $CKPT ${NUM_FRAMES} ${MODEL_NAME} -${TASK_ID_LIST[i]} >${LOG_BASE}_${TASK_ID_LIST[i]}.log 2>&1 & done