From 9ea3d46a3325a34179dcff346cdcefeade07247a Mon Sep 17 00:00:00 2001 From: zhengzangw Date: Wed, 15 May 2024 16:35:53 +0000 Subject: [PATCH] [refactor] eval scripts --- eval/human_eval/launch.sh | 31 +++++ eval/launch.sh | 51 ------- eval/launch_per_gpu.sh | 40 ------ eval/multiple.sh | 281 -------------------------------------- eval/test.sh | 14 -- eval/vbench/launch.sh | 26 ++++ eval/vbench_i2v/launch.sh | 26 ++++ notebooks/inference.ipynb | 201 ++++++++++++++++++++++++--- 8 files changed, 268 insertions(+), 402 deletions(-) create mode 100644 eval/human_eval/launch.sh delete mode 100644 eval/launch.sh delete mode 100644 eval/launch_per_gpu.sh delete mode 100644 eval/multiple.sh delete mode 100644 eval/test.sh create mode 100644 eval/vbench/launch.sh create mode 100644 eval/vbench_i2v/launch.sh diff --git a/eval/human_eval/launch.sh b/eval/human_eval/launch.sh new file mode 100644 index 0000000..5b25abd --- /dev/null +++ b/eval/human_eval/launch.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +set -x +set -e + +CKPT=$1 +NUM_FRAMES=$2 +MODEL_NAME=$3 + +if [[ $CKPT == *"ema"* ]]; then + parentdir=$(dirname $CKPT) + CKPT_BASE=$(basename $parentdir)_ema +else + CKPT_BASE=$(basename $CKPT) +fi +LOG_BASE=logs/sample/${MODEL_NAME}_${CKPT_BASE} +echo "Logging to $LOG_BASE" + +# == sample & human evaluation == +CUDA_VISIBLE_DEVICES=0 bash eval/sample.sh $CKPT 1 $MODEL_NAME -1 >${LOG_BASE}_1.log 2>&1 & +CUDA_VISIBLE_DEVICES=1 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2a >${LOG_BASE}_2a.log 2>&1 & +CUDA_VISIBLE_DEVICES=2 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2b >${LOG_BASE}_2b.log 2>&1 & +CUDA_VISIBLE_DEVICES=3 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2c >${LOG_BASE}_2c.log 2>&1 & +CUDA_VISIBLE_DEVICES=4 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2d >${LOG_BASE}_2d.log 2>&1 & +CUDA_VISIBLE_DEVICES=5 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2e >${LOG_BASE}_2e.log 2>&1 & +CUDA_VISIBLE_DEVICES=6 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2f >${LOG_BASE}_2f.log 2>&1 & +CUDA_VISIBLE_DEVICES=7 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2g >${LOG_BASE}_2g.log 2>&1 & + +# CUDA_VISIBLE_DEVICES=7 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2h >${LOG_BASE}_2h.log 2>&1 & + +# kill all by: pkill -f "inference" diff --git a/eval/launch.sh b/eval/launch.sh deleted file mode 100644 index 6c55f7b..0000000 --- a/eval/launch.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash - -set -x -set -e - -CKPT=$1 -NUM_FRAMES=$2 -MODEL_NAME=$3 - -if [[ $CKPT == *"ema"* ]]; then - parentdir=$(dirname $CKPT) - CKPT_BASE=$(basename $parentdir)_ema -else - CKPT_BASE=$(basename $CKPT) -fi -LOG_BASE=logs/sample/${MODEL_NAME}_${CKPT_BASE} -echo "Logging to $LOG_BASE" - -# == sample & human evaluation == -CUDA_VISIBLE_DEVICES=0 bash eval/sample.sh $CKPT 1 $MODEL_NAME -1 >${LOG_BASE}_1.log 2>&1 & -CUDA_VISIBLE_DEVICES=1 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2a >${LOG_BASE}_2a.log 2>&1 & -CUDA_VISIBLE_DEVICES=2 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2b >${LOG_BASE}_2b.log 2>&1 & -CUDA_VISIBLE_DEVICES=3 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2c >${LOG_BASE}_2c.log 2>&1 & -CUDA_VISIBLE_DEVICES=4 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2d >${LOG_BASE}_2d.log 2>&1 & -CUDA_VISIBLE_DEVICES=5 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2e >${LOG_BASE}_2e.log 2>&1 & -CUDA_VISIBLE_DEVICES=6 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2f >${LOG_BASE}_2f.log 2>&1 & -CUDA_VISIBLE_DEVICES=7 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2g >${LOG_BASE}_2g.log 2>&1 & - -# CUDA_VISIBLE_DEVICES=7 bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2h >${LOG_BASE}_2h.log 2>&1 & - -# == vbench == -# CUDA_VISIBLE_DEVICES=0 bash eval/sample.sh $CKPT -4a >${LOG_BASE}_4a.log 2>&1 & -# CUDA_VISIBLE_DEVICES=1 bash eval/sample.sh $CKPT -4b >${LOG_BASE}_4b.log 2>&1 & -# CUDA_VISIBLE_DEVICES=2 bash eval/sample.sh $CKPT -4c >${LOG_BASE}_4c.log 2>&1 & -# CUDA_VISIBLE_DEVICES=3 bash eval/sample.sh $CKPT -4d >${LOG_BASE}_4d.log 2>&1 & -# CUDA_VISIBLE_DEVICES=4 bash eval/sample.sh $CKPT -4e >${LOG_BASE}_4e.log 2>&1 & -# CUDA_VISIBLE_DEVICES=5 bash eval/sample.sh $CKPT -4f >${LOG_BASE}_4f.log 2>&1 & -# CUDA_VISIBLE_DEVICES=6 bash eval/sample.sh $CKPT -4g >${LOG_BASE}_4g.log 2>&1 & -# CUDA_VISIBLE_DEVICES=7 bash eval/sample.sh $CKPT -4h >${LOG_BASE}_4h.log 2>&1 & - -# == vbench i2v == -# CUDA_VISIBLE_DEVICES=0 bash eval/sample.sh $CKPT -5a >${LOG_BASE}_5a.log 2>&1 & -# CUDA_VISIBLE_DEVICES=1 bash eval/sample.sh $CKPT -5b >${LOG_BASE}_5b.log 2>&1 & -# CUDA_VISIBLE_DEVICES=2 bash eval/sample.sh $CKPT -5c >${LOG_BASE}_5c.log 2>&1 & -# CUDA_VISIBLE_DEVICES=3 bash eval/sample.sh $CKPT -5d >${LOG_BASE}_5d.log 2>&1 & -# CUDA_VISIBLE_DEVICES=4 bash eval/sample.sh $CKPT -5e >${LOG_BASE}_5e.log 2>&1 & -# CUDA_VISIBLE_DEVICES=5 bash eval/sample.sh $CKPT -5f >${LOG_BASE}_5f.log 2>&1 & -# CUDA_VISIBLE_DEVICES=6 bash eval/sample.sh $CKPT -5g >${LOG_BASE}_5g.log 2>&1 & -# CUDA_VISIBLE_DEVICES=7 bash eval/sample.sh $CKPT -5h >${LOG_BASE}_5h.log 2>&1 & - -# kill all by: pkill -f "inference" diff --git a/eval/launch_per_gpu.sh b/eval/launch_per_gpu.sh deleted file mode 100644 index 2d65522..0000000 --- a/eval/launch_per_gpu.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash - -set -x -set -e - -CKPT=$1 -CUDA_NUM=$2 -NUM_FRAMES=$3 -MODEL_NAME=$4 - - -if [[ $CKPT == *"ema"* ]]; then - parentdir=$(dirname $CKPT) - CKPT_BASE=$(basename $parentdir)_ema -else - CKPT_BASE=$(basename $CKPT) -fi - -LOG_BASE=logs/sample/${MODEL_NAME}_${CKPT_BASE} -echo "Logging to $LOG_BASE" - -# == sample & human evaluation == -echo "running image task" -CUDA_VISIBLE_DEVICES=$CUDA_NUM bash eval/sample.sh $CKPT 1 $MODEL_NAME -1 >${LOG_BASE}_1.log 2>&1 -echo "running task 2a" -CUDA_VISIBLE_DEVICES=$CUDA_NUM bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2a >${LOG_BASE}_2a.log 2>&1 -# echo "running task 2b" -# CUDA_VISIBLE_DEVICES=$CUDA_NUM bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2b >${LOG_BASE}_2b.log 2>&1 -# echo "running task 2c" -# CUDA_VISIBLE_DEVICES=$CUDA_NUM bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2c >${LOG_BASE}_2c.log 2>&1 -# echo "running task 2d" -# CUDA_VISIBLE_DEVICES=$CUDA_NUM bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2d >${LOG_BASE}_2d.log 2>&1 -# echo "running task 2e" -# CUDA_VISIBLE_DEVICES=$CUDA_NUM bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2e >${LOG_BASE}_2e.log 2>&1 -# echo "running task 2f" -# CUDA_VISIBLE_DEVICES=$CUDA_NUM bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2f >${LOG_BASE}_2f.log 2>&1 -# echo "running task 2g" -# CUDA_VISIBLE_DEVICES=$CUDA_NUM bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2g >${LOG_BASE}_2g.log 2>&1 -# echo "running task 2h" -# CUDA_VISIBLE_DEVICES=$CUDA_NUM bash eval/sample.sh $CKPT $NUM_FRAMES $MODEL_NAME -2h >${LOG_BASE}_2h.log 2>&1 diff --git a/eval/multiple.sh b/eval/multiple.sh deleted file mode 100644 index 5e1dfe0..0000000 --- a/eval/multiple.sh +++ /dev/null @@ -1,281 +0,0 @@ -#!/bin/bash - -set -x -set -e - -CKPT=$1 -PROMPT=$2 -NUM_SAMPLE=3 -NAME=$(date +%Y%m%d%H%M%S) -CMD="python scripts/inference-long.py configs/opensora-v1-1/inference/sample.py" -if [[ $CKPT == *"ema"* ]]; then - parentdir=$(dirname $CKPT) - CKPT_BASE=$(basename $parentdir)_ema -else - CKPT_BASE=$(basename $CKPT) -fi -OUTPUT="./samples/samples_${CKPT_BASE}_${NAME}" -start=$(date +%s) - -# Generate samples - -# == 16x240p == -# 1:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x240p_1_1 \ - --num-frames 16 --image-size 320 320 --num-sample $NUM_SAMPLE -# 16:9 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x240p_16_9 \ - --num-frames 16 --image-size 240 426 --num-sample $NUM_SAMPLE -# 9:16 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x240p_9_16 \ - --num-frames 16 --image-size 426 240 --num-sample $NUM_SAMPLE -# 4:3 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x240p_4_3 \ - --num-frames 16 --image-size 276 368 --num-sample $NUM_SAMPLE -# 3:4 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x240p_3_4 \ - --num-frames 16 --image-size 368 276 --num-sample $NUM_SAMPLE -# 1:2 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x240p_1_2 \ - --num-frames 16 --image-size 226 452 --num-sample $NUM_SAMPLE -# 2:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x240p_2_1 \ - --num-frames 16 --image-size 452 226 --num-sample $NUM_SAMPLE - -# == 64x240p == -# 1:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x240p_1_1 \ - --num-frames 64 --image-size 320 320 --num-sample $NUM_SAMPLE -# 16:9 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x240p_16_9 \ - --num-frames 64 --image-size 240 426 --num-sample $NUM_SAMPLE -# 9:16 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x240p_9_16 \ - --num-frames 64 --image-size 426 240 --num-sample $NUM_SAMPLE -# 4:3 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x240p_4_3 \ - --num-frames 64 --image-size 276 368 --num-sample $NUM_SAMPLE -# 3:4 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x240p_3_4 \ - --num-frames 64 --image-size 368 276 --num-sample $NUM_SAMPLE -# 1:2 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x240p_1_2 \ - --num-frames 64 --image-size 226 452 --num-sample $NUM_SAMPLE -# 2:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x240p_2_1 \ - --num-frames 64 --image-size 452 226 --num-sample $NUM_SAMPLE - -# == 128x240p == -# 1:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x240p_1_1 \ - --num-frames 128 --image-size 320 320 --num-sample $NUM_SAMPLE -# 16:9 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x240p_16_9 \ - --num-frames 128 --image-size 240 426 --num-sample $NUM_SAMPLE -# 9:16 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x240p_9_16 \ - --num-frames 128 --image-size 426 240 --num-sample $NUM_SAMPLE -# 4:3 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x240p_4_3 \ - --num-frames 128 --image-size 276 368 --num-sample $NUM_SAMPLE -# 3:4 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x240p_3_4 \ - --num-frames 128 --image-size 368 276 --num-sample $NUM_SAMPLE -# 1:2 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x240p_1_2 \ - --num-frames 128 --image-size 226 452 --num-sample $NUM_SAMPLE -# 2:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x240p_2_1 \ - --num-frames 128 --image-size 452 226 --num-sample $NUM_SAMPLE - -# == 16x360p == -# 1:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x360p_1_1 \ - --num-frames 16 --image-size 480 480 --num-sample $NUM_SAMPLE -# 16:9 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x360p_16_9 \ - --num-frames 16 --image-size 360 640 --num-sample $NUM_SAMPLE -# 9:16 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x360p_9_16 \ - --num-frames 16 --image-size 640 360 --num-sample $NUM_SAMPLE -# 4:3 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x360p_4_3 \ - --num-frames 16 --image-size 416 554 --num-sample $NUM_SAMPLE -# 3:4 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x360p_3_4 \ - --num-frames 16 --image-size 554 416 --num-sample $NUM_SAMPLE -# 1:2 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x360p_1_2 \ - --num-frames 16 --image-size 360 640 --num-sample $NUM_SAMPLE -# 2:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x360p_2_1 \ - --num-frames 16 --image-size 640 360 --num-sample $NUM_SAMPLE - -# == 64x360p == -# 1:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x360p_1_1 \ - --num-frames 64 --image-size 480 480 --num-sample $NUM_SAMPLE -# 16:9 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x360p_16_9 \ - --num-frames 64 --image-size 360 640 --num-sample $NUM_SAMPLE -# 9:16 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x360p_9_16 \ - --num-frames 64 --image-size 640 360 --num-sample $NUM_SAMPLE -# 4:3 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x360p_4_3 \ - --num-frames 64 --image-size 416 554 --num-sample $NUM_SAMPLE -# 3:4 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x360p_3_4 \ - --num-frames 64 --image-size 554 416 --num-sample $NUM_SAMPLE -# 1:2 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x360p_1_2 \ - --num-frames 64 --image-size 360 640 --num-sample $NUM_SAMPLE -# 2:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x360p_2_1 \ - --num-frames 64 --image-size 640 360 --num-sample $NUM_SAMPLE - -# == 128x360p == -# 1:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x360p_1_1 \ - --num-frames 128 --image-size 480 480 --num-sample $NUM_SAMPLE -# 16:9 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x360p_16_9 \ - --num-frames 128 --image-size 360 640 --num-sample $NUM_SAMPLE -# 9:16 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x360p_9_16 \ - --num-frames 128 --image-size 640 360 --num-sample $NUM_SAMPLE -# 4:3 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x360p_4_3 \ - --num-frames 128 --image-size 416 554 --num-sample $NUM_SAMPLE -# 3:4 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x360p_3_4 \ - --num-frames 128 --image-size 554 416 --num-sample $NUM_SAMPLE -# 1:2 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x360p_1_2 \ - --num-frames 128 --image-size 360 640 --num-sample $NUM_SAMPLE -# 2:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 128x360p_2_1 \ - --num-frames 128 --image-size 640 360 --num-sample $NUM_SAMPLE - -# == 16x480p == -# 1:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x480p_1_1 \ - --num-frames 16 --image-size 640 640 --num-sample $NUM_SAMPLE -# 16:9 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x480p_16_9 \ - --num-frames 16 --image-size 480 854 --num-sample $NUM_SAMPLE -# 9:16 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x480p_9_16 \ - --num-frames 16 --image-size 854 480 --num-sample $NUM_SAMPLE -# 4:3 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x480p_4_3 \ - --num-frames 16 --image-size 554 738 --num-sample $NUM_SAMPLE -# 3:4 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x480p_3_4 \ - --num-frames 16 --image-size 738 554 --num-sample $NUM_SAMPLE -# 1:2 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x480p_1_2 \ - --num-frames 16 --image-size 452 904 --num-sample $NUM_SAMPLE -# 2:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x480p_2_1 \ - --num-frames 16 --image-size 904 452 --num-sample $NUM_SAMPLE - -# == 32x480p == -# 1:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x480p_1_1 \ - --num-frames 32 --image-size 640 640 --num-sample $NUM_SAMPLE -# 16:9 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x480p_16_9 \ - --num-frames 32 --image-size 480 854 --num-sample $NUM_SAMPLE -# 9:16 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x480p_9_16 \ - --num-frames 32 --image-size 854 480 --num-sample $NUM_SAMPLE -# 4:3 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x480p_4_3 \ - --num-frames 32 --image-size 554 738 --num-sample $NUM_SAMPLE -# 3:4 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x480p_3_4 \ - --num-frames 32 --image-size 738 554 --num-sample $NUM_SAMPLE -# 1:2 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x480p_1_2 \ - --num-frames 32 --image-size 452 904 --num-sample $NUM_SAMPLE -# 2:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x480p_2_1 \ - --num-frames 32 --image-size 904 452 --num-sample $NUM_SAMPLE - -# == 64x480p == -# 1:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x480p_1_1 \ - --num-frames 64 --image-size 640 640 --num-sample $NUM_SAMPLE -# 16:9 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x480p_16_9 \ - --num-frames 64 --image-size 480 854 --num-sample $NUM_SAMPLE -# 9:16 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x480p_9_16 \ - --num-frames 64 --image-size 854 480 --num-sample $NUM_SAMPLE -# 4:3 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x480p_4_3 \ - --num-frames 64 --image-size 554 738 --num-sample $NUM_SAMPLE -# 3:4 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x480p_3_4 \ - --num-frames 64 --image-size 738 554 --num-sample $NUM_SAMPLE -# 1:2 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x480p_1_2 \ - --num-frames 64 --image-size 452 904 --num-sample $NUM_SAMPLE -# 2:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 64x480p_2_1 \ - --num-frames 64 --image-size 904 452 --num-sample $NUM_SAMPLE - -# == 16x720p == -# 1:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x720p_1_1 \ - --num-frames 16 --image-size 960 960 --num-sample $NUM_SAMPLE -# 16:9 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x720p_16_9 \ - --num-frames 16 --image-size 720 1280 --num-sample $NUM_SAMPLE -# 9:16 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x720p_9_16 \ - --num-frames 16 --image-size 1280 720 --num-sample $NUM_SAMPLE -# 4:3 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x720p_4_3 \ - --num-frames 16 --image-size 832 1108 --num-sample $NUM_SAMPLE -# 3:4 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x720p_3_4 \ - --num-frames 16 --image-size 1108 832 --num-sample $NUM_SAMPLE -# 1:2 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x720p_1_2 \ - --num-frames 16 --image-size 1358 600 --num-sample $NUM_SAMPLE -# 2:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 16x720p_2_1 \ - --num-frames 16 --image-size 600 1358 - -# == 32x720p == -# 1:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x720p_1_1 \ - --num-frames 32 --image-size 960 960 --num-sample $NUM_SAMPLE -# 16:9 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x720p_16_9 \ - --num-frames 32 --image-size 720 1280 --num-sample $NUM_SAMPLE -# 9:16 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x720p_9_16 \ - --num-frames 32 --image-size 1280 720 --num-sample $NUM_SAMPLE -# 4:3 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x720p_4_3 \ - --num-frames 32 --image-size 832 1108 --num-sample $NUM_SAMPLE -# 3:4 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x720p_3_4 \ - --num-frames 32 --image-size 1108 832 --num-sample $NUM_SAMPLE -# 1:2 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x720p_1_2 \ - --num-frames 32 --image-size 1358 600 --num-sample $NUM_SAMPLE -# 2:1 -eval $CMD --ckpt-path $CKPT --prompt \"$PROMPT\" --save-dir $OUTPUT --sample-name 32x720p_2_1 \ - --num-frames 32 --image-size 600 1358 - -### End - -end=$(date +%s) - -runtime=$((end - start)) - -echo "Runtime: $runtime seconds" diff --git a/eval/test.sh b/eval/test.sh deleted file mode 100644 index 77e5623..0000000 --- a/eval/test.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash - -# set -x -set -e - -CKPT=$1 -NUM_FRAMES=$2 -MODEL_NAME=$3 - -let DOUBLE_FRAMES=$2*2 -let TRIPLE_FRAMES=$2*3 - -echo $DOUBLE_FRAMES -echo $TRIPLE_FRAMES diff --git a/eval/vbench/launch.sh b/eval/vbench/launch.sh new file mode 100644 index 0000000..ac25c01 --- /dev/null +++ b/eval/vbench/launch.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -x +set -e + +CKPT=$1 +NUM_FRAMES=$2 +MODEL_NAME=$3 + +if [[ $CKPT == *"ema"* ]]; then + parentdir=$(dirname $CKPT) + CKPT_BASE=$(basename $parentdir)_ema +else + CKPT_BASE=$(basename $CKPT) +fi +LOG_BASE=logs/sample/${MODEL_NAME}_${CKPT_BASE} +echo "Logging to $LOG_BASE" + +CUDA_VISIBLE_DEVICES=0 bash eval/sample.sh $CKPT -4a >${LOG_BASE}_4a.log 2>&1 & +CUDA_VISIBLE_DEVICES=1 bash eval/sample.sh $CKPT -4b >${LOG_BASE}_4b.log 2>&1 & +CUDA_VISIBLE_DEVICES=2 bash eval/sample.sh $CKPT -4c >${LOG_BASE}_4c.log 2>&1 & +CUDA_VISIBLE_DEVICES=3 bash eval/sample.sh $CKPT -4d >${LOG_BASE}_4d.log 2>&1 & +CUDA_VISIBLE_DEVICES=4 bash eval/sample.sh $CKPT -4e >${LOG_BASE}_4e.log 2>&1 & +CUDA_VISIBLE_DEVICES=5 bash eval/sample.sh $CKPT -4f >${LOG_BASE}_4f.log 2>&1 & +CUDA_VISIBLE_DEVICES=6 bash eval/sample.sh $CKPT -4g >${LOG_BASE}_4g.log 2>&1 & +CUDA_VISIBLE_DEVICES=7 bash eval/sample.sh $CKPT -4h >${LOG_BASE}_4h.log 2>&1 & diff --git a/eval/vbench_i2v/launch.sh b/eval/vbench_i2v/launch.sh new file mode 100644 index 0000000..82703d6 --- /dev/null +++ b/eval/vbench_i2v/launch.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -x +set -e + +CKPT=$1 +NUM_FRAMES=$2 +MODEL_NAME=$3 + +if [[ $CKPT == *"ema"* ]]; then + parentdir=$(dirname $CKPT) + CKPT_BASE=$(basename $parentdir)_ema +else + CKPT_BASE=$(basename $CKPT) +fi +LOG_BASE=logs/sample/${MODEL_NAME}_${CKPT_BASE} +echo "Logging to $LOG_BASE" + +CUDA_VISIBLE_DEVICES=0 bash eval/sample.sh $CKPT -5a >${LOG_BASE}_5a.log 2>&1 & +CUDA_VISIBLE_DEVICES=1 bash eval/sample.sh $CKPT -5b >${LOG_BASE}_5b.log 2>&1 & +CUDA_VISIBLE_DEVICES=2 bash eval/sample.sh $CKPT -5c >${LOG_BASE}_5c.log 2>&1 & +CUDA_VISIBLE_DEVICES=3 bash eval/sample.sh $CKPT -5d >${LOG_BASE}_5d.log 2>&1 & +CUDA_VISIBLE_DEVICES=4 bash eval/sample.sh $CKPT -5e >${LOG_BASE}_5e.log 2>&1 & +CUDA_VISIBLE_DEVICES=5 bash eval/sample.sh $CKPT -5f >${LOG_BASE}_5f.log 2>&1 & +CUDA_VISIBLE_DEVICES=6 bash eval/sample.sh $CKPT -5g >${LOG_BASE}_5g.log 2>&1 & +CUDA_VISIBLE_DEVICES=7 bash eval/sample.sh $CKPT -5h >${LOG_BASE}_5h.log 2>&1 & diff --git a/notebooks/inference.ipynb b/notebooks/inference.ipynb index 9a9a7d7..7ef8489 100644 --- a/notebooks/inference.ipynb +++ b/notebooks/inference.ipynb @@ -11,12 +11,12 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "Define global variables." + "Define global variables. You should change the following variables according to your setting." ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 10, "metadata": {}, "outputs": [], "source": [ @@ -24,7 +24,9 @@ "ROOT = \"..\"\n", "cfg_path = f\"{ROOT}/configs/opensora-v1-2/inference/sample.py\"\n", "ckpt_path = \"/home/lishenggui/projects/sora/Open-Sora-dev/outputs/207-STDiT3-XL-2/epoch0-global_step9000/\"\n", - "vae_path = f\"{ROOT}/pretrained_models/vae-pipeline\"" + "vae_path = f\"{ROOT}/pretrained_models/vae-pipeline\"\n", + "save_dir = f\"{ROOT}/samples/samples_notebook/\"\n", + "device = \"cuda:0\"" ] }, { @@ -36,9 +38,26 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/zhengzangwei/.conda/envs/opensora/lib/python3.10/site-packages/transformers/utils/generic.py:441: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n", + " _torch_pytree._register_pytree_node(\n", + "/home/zhengzangwei/.conda/envs/opensora/lib/python3.10/site-packages/colossalai/pipeline/schedule/_utils.py:19: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n", + " _register_pytree_node(OrderedDict, _odict_flatten, _odict_unflatten)\n", + "/home/zhengzangwei/.conda/envs/opensora/lib/python3.10/site-packages/torch/utils/_pytree.py:254: UserWarning: is already registered as pytree node. Overwriting the previous registration.\n", + " warnings.warn(\n", + "/home/zhengzangwei/.conda/envs/opensora/lib/python3.10/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n", + " _torch_pytree._register_pytree_node(\n", + "/home/zhengzangwei/.conda/envs/opensora/lib/python3.10/site-packages/transformers/utils/generic.py:309: UserWarning: torch.utils._pytree._register_pytree_node is deprecated. Please use torch.utils._pytree.register_pytree_node instead.\n", + " _torch_pytree._register_pytree_node(\n" + ] + } + ], "source": [ "import os\n", "from pprint import pformat\n", @@ -71,7 +90,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -80,7 +99,7 @@ "42" ] }, - "execution_count": 7, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -94,7 +113,6 @@ "cfg.vae.from_pretrained = vae_path\n", "\n", "# == device and dtype ==\n", - "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", "cfg_dtype = cfg.get(\"dtype\", \"fp32\")\n", "assert cfg_dtype in [\"fp16\", \"bf16\", \"fp32\"], f\"Unknown mixed precision {cfg_dtype}\"\n", "dtype = to_torch_dtype(cfg.get(\"dtype\", \"bf16\"))\n", @@ -106,13 +124,13 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { - "model_id": "29ca38c42a38453aa65784e1ee89a61a", + "model_id": "dbb26c0ba0824c69b1e1d293fbfe3d9a", "version_major": 2, "version_minor": 0 }, @@ -156,7 +174,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ @@ -181,9 +199,9 @@ " loop=cfg.get(\"loop\", 1),\n", " condition_frame_length=cfg.get(\"condition_frame_length\", 5),\n", " align=cfg.get(\"align\", None),\n", - " save_dir=os.path.join(ROOT, cfg.save_dir),\n", " sample_name=cfg.get(\"sample_name\", None),\n", " prompt_as_path=cfg.get(\"prompt_as_path\", False),\n", + " disable_progress=False,\n", "):\n", " global start_idx\n", " os.makedirs(save_dir, exist_ok=True)\n", @@ -209,12 +227,12 @@ " resolution is not None and aspect_ratio is not None\n", " ), \"resolution and aspect_ratio must be provided if image_size is not provided\"\n", " image_size = get_image_size(resolution, aspect_ratio)\n", - " num_frames = get_num_frames(cfg.num_frames)\n", + " num_frames = get_num_frames(num_frames)\n", " input_size = (num_frames, *image_size)\n", " latent_size = vae.get_latent_size(input_size)\n", "\n", " # == Iter over all samples ==\n", - " for i in tqdm(range(0, len(prompts), batch_size)):\n", + " for i in tqdm(range(0, len(prompts), batch_size), disable=disable_progress):\n", " # == prepare batch prompts ==\n", " batch_prompts = prompts[i : i + batch_size]\n", " ms = mask_strategy[i : i + batch_size]\n", @@ -296,18 +314,65 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "from IPython.display import Video, Image, display\n", "\n", + "\n", "def display_results(paths):\n", " for path in paths:\n", " if is_img(path):\n", " display(Image(path))\n", " else:\n", - " display(Video(path, embed=True))" + " display(Video(path, embed=True))\n", + "\n", + "\n", + "def reset_start_idx():\n", + " global start_idx\n", + " start_idx = 0\n", + "\n", + "\n", + "ALL_ASPECT_RATIO = [\"1:1\", \"16:9\", \"9:16\", \"3:4\", \"4:3\", \"1:2\", \"2:1\"]\n", + "\n", + "\n", + "def inference_all_aspects(prompts, resolution, num_frames, *args, **kwargs):\n", + " paths = []\n", + " for aspect_ratio in tqdm(ALL_ASPECT_RATIO):\n", + " paths.extend(\n", + " inference(\n", + " prompts,\n", + " resolution=resolution,\n", + " num_frames=num_frames,\n", + " aspect_ratio=aspect_ratio,\n", + " disable_progress=True,\n", + " *args,\n", + " **kwargs\n", + " )\n", + " )\n", + " return paths" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Inference for OpenSora" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sample code for inference for OpenSora." ] }, { @@ -326,6 +391,110 @@ ")\n", "display_results(paths)" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sample all aspect ratios." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "PROMPT = \"a boy.\"\n", + "paths = inference_all_aspects(\n", + " PROMPT,\n", + " resolution=\"240p\",\n", + " num_frames=\"1x\",\n", + " num_sampling_steps=30,\n", + " cfg_scale=7.0,\n", + ")\n", + "display_results(paths)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sample all resolution and length." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "PROMPT = \"a boy.\"\n", + "sample_cfg = {\n", + " \"144p\": [1, \"1x\", \"2x\", \"4x\", \"8x\"],\n", + " \"240p\": [1, \"1x\", \"2x\", \"4x\", \"8x\"],\n", + " \"360p\": [1, \"1x\", \"2x\", \"4x\"],\n", + " \"480p\": [1, \"1x\", \"2x\", \"4x\"],\n", + " \"720p\": [1, \"1x\", \"2x\"],\n", + "}\n", + "all_paths = []\n", + "for resolution, num_frames in sample_cfg.items():\n", + " for num_frame in num_frames:\n", + " print(f\"Resolution: {resolution}, Num Frames: {num_frame}\")\n", + " paths = inference(\n", + " PROMPT,\n", + " resolution=resolution,\n", + " num_frames=num_frame,\n", + " aspect_ratio=\"9:16\",\n", + " num_sampling_steps=30,\n", + " cfg_scale=7.0,\n", + " disable_progress=True,\n", + " )\n", + " display_results(paths)\n", + " all_paths.extend(paths)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Sample all resolution, length, and aspect ratios." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "PROMPT = \"a boy.\"\n", + "sample_cfg = {\n", + " \"144p\": [1, \"1x\", \"2x\", \"4x\", \"8x\"],\n", + " \"240p\": [1, \"1x\", \"2x\", \"4x\", \"8x\"],\n", + " \"360p\": [1, \"1x\", \"2x\", \"4x\"],\n", + " \"480p\": [1, \"1x\", \"2x\", \"4x\"],\n", + " \"720p\": [1, \"1x\", \"2x\"],\n", + "}\n", + "all_paths = []\n", + "for resolution, num_frames in sample_cfg.items():\n", + " for num_frame in num_frames:\n", + " paths = inference_all_aspects(\n", + " PROMPT,\n", + " resolution=resolution,\n", + " num_frames=num_frames,\n", + " num_sampling_steps=30,\n", + " cfg_scale=7.0,\n", + " )\n", + " display_results(paths)\n", + " all_paths.extend(paths)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {