From 01fee497f3bcc98a22bef60657a34e6f53cfe606 Mon Sep 17 00:00:00 2001
From: Shen-Chenhui <shen_chenhui@u.nus.edu>
Date: Thu, 6 Jun 2024 03:34:41 +0000
Subject: [PATCH] format

---
 .gitignore                    |  4 ++++
 eval/README.md                |  2 ++
 eval/human_eval/launch.sh     |  1 +
 eval/loss/launch.sh           |  2 +-
 eval/vbench/launch.sh         |  9 +++++----
 eval/vbench_i2v/vbench_i2v.py | 17 ++++++++---------
 6 files changed, 21 insertions(+), 14 deletions(-)

diff --git a/.gitignore b/.gitignore
index ebf9f97..b4237e0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -196,3 +196,7 @@ package.json
 
 # PLLaVA
 tools/caption/pllava_dir/PLLaVA/
+
+# vbench
+vbench
+vbench2_beta_i2v
diff --git a/eval/README.md b/eval/README.md
index d30d594..15852bf 100644
--- a/eval/README.md
+++ b/eval/README.md
@@ -79,6 +79,8 @@ bash eval/vbench_i2v/launch.sh /path/to/ckpt num_frames model_name
 python eval/vbench_i2v/vbench_i2v.py
 
 # Step 3: obtain the scaled scores
+def load_i2v_dimension_info(json_dir, dimension, lang, resolution):
+# Note that if you need to go to `your_conda_env_path/lib/python3.x/site-packages/vbench2_beta_i2v/utils.py` and change the harded-coded var `image_root` in the `load_i2v_dimension_info` function to your appropriate image folder.
 python eval/vbench_i2v/tabulate_vbench_i2v_scores.py --score_dir path/to/evaluation_results/dir
 
 ```
diff --git a/eval/human_eval/launch.sh b/eval/human_eval/launch.sh
index 79095f1..c980267 100644
--- a/eval/human_eval/launch.sh
+++ b/eval/human_eval/launch.sh
@@ -16,6 +16,7 @@ fi
 # LOG_BASE=logs/samples/${MODEL_NAME}_${CKPT_BASE}
 # mkdir -p logs/samples
 LOG_BASE=$(dirname $CKPT)/eval
+mkdir -p ${LOG_BASE}
 echo "Logging to $LOG_BASE"
 
 GPUS=(0 1 2 3 4 5 6 7)
diff --git a/eval/loss/launch.sh b/eval/loss/launch.sh
index bf28b02..4eb5435 100644
--- a/eval/loss/launch.sh
+++ b/eval/loss/launch.sh
@@ -17,7 +17,7 @@ else
 fi
 # LOG_BASE=logs/loss/${MODEL_NAME}_${CKPT_BASE}
 # mkdir -p logs/loss
-LOG_BASE=$(dirname $CKPT)/eval
+LOG_BASE=$(dirname $CKPT_PATH)/eval
 echo "Logging to $LOG_BASE"
 
 
diff --git a/eval/vbench/launch.sh b/eval/vbench/launch.sh
index 436a937..c295fb5 100644
--- a/eval/vbench/launch.sh
+++ b/eval/vbench/launch.sh
@@ -4,8 +4,9 @@ set -x
 set -e
 
 CKPT=$1
-MODEL_NAME=$2
-NUM_FRAMES=$3
+NUM_FRAMES=$2
+MODEL_NAME=$3
+
 
 if [[ $CKPT == *"ema"* ]]; then
     parentdir=$(dirname $CKPT)
@@ -13,12 +14,12 @@ if [[ $CKPT == *"ema"* ]]; then
 else
     CKPT_BASE=$(basename $CKPT)
 fi
-LOG_BASE=logs/sample/${MODEL_NAME}_${CKPT_BASE}
+LOG_BASE=$(dirname $CKPT)/eval
 echo "Logging to $LOG_BASE"
 
 GPUS=(0 1 2 3 4 5 6 7)
 TASK_ID_LIST=(4a 4b 4c 4d 4e 4f 4g 4h)
 
 for i in "${!GPUS[@]}"; do
-    CUDA_VISIBLE_DEVICES=${GPUS[i]} bash eval/sample.sh $CKPT ${NUM_FRAMES} ${MODEL_NAME} -${TASK_ID_LIST[i]} >${LOG_BASE}_${TASK_ID_LIST[i]}.log 2>&1 &
+    CUDA_VISIBLE_DEVICES=${GPUS[i]} bash eval/sample.sh $CKPT ${NUM_FRAMES} ${MODEL_NAME} -${TASK_ID_LIST[i]} >${LOG_BASE}/${TASK_ID_LIST[i]}.log 2>&1 &
 done
diff --git a/eval/vbench_i2v/vbench_i2v.py b/eval/vbench_i2v/vbench_i2v.py
index c5ab048..ac82ab9 100644
--- a/eval/vbench_i2v/vbench_i2v.py
+++ b/eval/vbench_i2v/vbench_i2v.py
@@ -1,10 +1,9 @@
 import argparse
 import os
 
-from vbench import VBench
 from vbench2_beta_i2v import VBenchI2V
 
-FULL_INFO_PATH = "vbench2_beta_i2v/vbench2_i2v_full_info.json"
+FULL_INFO_PATH = "eval/vbench_i2v/vbench2_i2v_full_info.json"
 VIDEO_QUALITY_DIMENSIONS = [
     "subject_consistency",
     "background_consistency",
@@ -31,12 +30,12 @@ if __name__ == "__main__":
     os.makedirs(output_dir, exist_ok=True)
     video_path = args.video_folder
 
-    my_VBench_I2V = VBenchI2V("cuda", FULL_INFO_PATH, "evaluation_results")
+    my_VBench_I2V = VBenchI2V("cuda", FULL_INFO_PATH, output_dir)
     my_VBench_I2V.evaluate(videos_path=video_path, name="vbench_i2v", dimension_list=I2V_DIMENSIONS, resolution="1-1")
 
-    my_VBench = VBench("cuda", FULL_INFO_PATH, output_dir)
-    my_VBench.evaluate(
-        videos_path=video_path,
-        name="vbench_video_quality",
-        dimension_list=VIDEO_QUALITY_DIMENSIONS,
-    )
+    # my_VBench = VBench("cuda", FULL_INFO_PATH, output_dir)
+    # my_VBench.evaluate(
+    #     videos_path=video_path,
+    #     name="vbench_video_quality",
+    #     dimension_list=VIDEO_QUALITY_DIMENSIONS,
+    # )