mirror of
https://github.com/hpcaitech/Open-Sora.git
synced 2026-05-21 11:59:01 +02:00
Merge branch 'dev/v1.2' of https://github.com/hpcaitech/Open-Sora-dev into dev/v1.2
This commit is contained in:
Tom Young
commit
6b6e96e811
0
--llm-refine
Normal file
0
--llm-refine
Normal file
|
|
@ -5,17 +5,6 @@ fps = 24
|
|||
frame_interval = 1
|
||||
save_fps = 24
|
||||
|
||||
prompt_path = "./assets/texts/t2v_sora.txt"
|
||||
# == Uncomment the following line to use the prompt below ==
|
||||
# prompt = [
|
||||
# 'Drone view of waves crashing against the rugged cliffs along Big Sur\'s garay point beach. {"reference_path": "assets/images/condition/cliff.png", "mask_strategy": "0"}',
|
||||
# 'A breathtaking sunrise scene.{"reference_path": "assets/images/condition/sunset1.png","mask_strategy": "0"}',
|
||||
# 'A car driving on the ocean.{"reference_path": "https://cdn.openai.com/tmp/s/interp/d0.mp4","mask_strategy": "0,0,-8,0,8"}',
|
||||
# 'A snowy forest.{"reference_path": "https://cdn.pixabay.com/video/2021/04/25/72171-542991404_large.mp4","mask_strategy": "0,0,0,0,15,0.8"}',
|
||||
# 'A breathtaking sunrise scene.{"reference_path": "assets/images/condition/sunset1.png;assets/images/condition/sunset2.png","mask_strategy": "0;0,1,0,-1,1"}',
|
||||
# '|0|a white jeep equipped with a roof rack driving on a dirt road in a coniferous forest.|2|a white jeep equipped with a roof rack driving on a dirt road in the desert.|4|a white jeep equipped with a roof rack driving on a dirt road in a mountain.|6|A white jeep equipped with a roof rack driving on a dirt road in a city.|8|a white jeep equipped with a roof rack driving on a dirt road on the surface of a river.|10|a white jeep equipped with a roof rack driving on a dirt road under the lake.|12|a white jeep equipped with a roof rack flying into the sky.|14|a white jeep equipped with a roof rack driving in the universe. Earth is the background.{"reference_path": "https://cdn.openai.com/tmp/s/interp/d0.mp4", "mask_strategy": "0,0,0,0,15"}',
|
||||
# ]
|
||||
|
||||
save_dir = "./samples/samples/"
|
||||
seed = 42
|
||||
batch_size = 1
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ def parse_args(training=False):
|
|||
parser.add_argument("--prompt-path", default=None, type=str, help="path to prompt txt file")
|
||||
parser.add_argument("--prompt", default=None, type=str, nargs="+", help="prompt list")
|
||||
parser.add_argument("--llm-refine", default=None, type=str2bool, help="enable LLM refine")
|
||||
parser.add_argument("--prompt-generator", default=None, type=str, help="prompt generator")
|
||||
|
||||
# image/video
|
||||
parser.add_argument("--num-frames", default=None, type=str, help="number of frames")
|
||||
|
|
|
|||
|
|
@ -115,9 +115,10 @@ def extract_prompts_loop(prompts, num_loop):
|
|||
ret_prompts.append(prompt)
|
||||
return ret_prompts
|
||||
|
||||
|
||||
def split_prompt(prompt_text):
|
||||
if prompt_text.startswith("|0|"):
|
||||
# this is for prompts which look like
|
||||
# this is for prompts which look like
|
||||
# |0| a beautiful day |1| a sunny day |2| a rainy day
|
||||
# we want to parse it into a list of prompts with the loop index
|
||||
prompt_list = prompt_text.split("|")[1:]
|
||||
|
|
@ -131,7 +132,8 @@ def split_prompt(prompt_text):
|
|||
return text_list, loop_idx
|
||||
else:
|
||||
return [prompt_text], None
|
||||
|
||||
|
||||
|
||||
def merge_prompt(text_list, loop_idx_list=None):
|
||||
if loop_idx_list is None:
|
||||
return text_list[0]
|
||||
|
|
@ -285,9 +287,11 @@ def refine_prompt_by_openai(prompt):
|
|||
response = get_openai_response(REFINE_PROMPTS, prompt)
|
||||
return response
|
||||
|
||||
|
||||
def has_openai_key():
|
||||
return "OPENAI_API_KEY" in os.environ
|
||||
|
||||
|
||||
def refine_prompts_by_openai(prompts):
|
||||
new_prompts = []
|
||||
for prompt in prompts:
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ from opensora.utils.inference_utils import (
|
|||
prepare_multi_resolution_info,
|
||||
refine_prompts_by_openai,
|
||||
split_prompt,
|
||||
merge_prompt
|
||||
merge_prompt,
|
||||
)
|
||||
from opensora.utils.misc import all_exists, create_logger, is_distributed, is_main_process, to_torch_dtype
|
||||
|
||||
|
|
@ -115,8 +115,10 @@ def main():
|
|||
prompts = cfg.get("prompt", None)
|
||||
start_idx = cfg.get("start_index", 0)
|
||||
if prompts is None:
|
||||
assert cfg.get("prompt_path", None) is not None, "Prompt or prompt_path must be provided"
|
||||
prompts = load_prompts(cfg.prompt_path, start_idx, cfg.get("end_index", None))
|
||||
if cfg.get("prompt_path", None) is not None:
|
||||
prompts = load_prompts(cfg.prompt_path, start_idx, cfg.get("end_index", None))
|
||||
else:
|
||||
prompts = [cfg.get("prompt_generator", "")] * 1_000_000 # endless loop
|
||||
|
||||
# == prepare reference ==
|
||||
reference_path = cfg.get("reference_path", [""] * len(prompts))
|
||||
|
|
@ -180,19 +182,19 @@ def main():
|
|||
if prompt_as_path and all_exists(save_paths):
|
||||
continue
|
||||
|
||||
# == process prompts step by step ==
|
||||
# == process prompts step by step ==
|
||||
# 0. split prompt
|
||||
# each element in the list is [prompt_segment_list, loop_idx_list]
|
||||
batched_prompt_segment_list = []
|
||||
batched_loop_idx_list = []
|
||||
for prompt in batch_prompts:
|
||||
prompt_segment_list, loop_idx_list = split_prompt(prompt)
|
||||
prompt_segment_list, loop_idx_list = split_prompt(prompt)
|
||||
batched_prompt_segment_list.append(prompt_segment_list)
|
||||
batched_loop_idx_list.append(loop_idx_list)
|
||||
|
||||
|
||||
# 1. refine prompt by openai
|
||||
if cfg.get("llm_refine", False):
|
||||
# only call openai API when
|
||||
# only call openai API when
|
||||
# 1. seq parallel is not enabled
|
||||
# 2. seq parallel is enabled and the process is rank 0
|
||||
if not enable_sequence_parallelism or (enable_sequence_parallelism and is_main_process()):
|
||||
|
|
@ -202,10 +204,16 @@ def main():
|
|||
# sync the prompt if using seq parallel
|
||||
if enable_sequence_parallelism:
|
||||
coordinator.block_all()
|
||||
prompt_segment_length = [len(prompt_segment_list) for prompt_segment_list in batched_prompt_segment_list]
|
||||
prompt_segment_length = [
|
||||
len(prompt_segment_list) for prompt_segment_list in batched_prompt_segment_list
|
||||
]
|
||||
|
||||
# flatten the prompt segment list
|
||||
batched_prompt_segment_list = [prompt_segment for prompt_segment_list in batched_prompt_segment_list for prompt_segment in prompt_segment_list]
|
||||
batched_prompt_segment_list = [
|
||||
prompt_segment
|
||||
for prompt_segment_list in batched_prompt_segment_list
|
||||
for prompt_segment in prompt_segment_list
|
||||
]
|
||||
|
||||
# create a list of size equal to world size
|
||||
broadcast_obj_list = [batched_prompt_segment_list] * coordinator.world_size
|
||||
|
|
@ -216,9 +224,9 @@ def main():
|
|||
start_idx = 0
|
||||
all_prompts = broadcast_obj_list[0]
|
||||
for num_segment in prompt_segment_length:
|
||||
batched_prompt_segment_list.append(all_prompts[start_idx:start_idx+num_segment])
|
||||
batched_prompt_segment_list.append(all_prompts[start_idx : start_idx + num_segment])
|
||||
start_idx += num_segment
|
||||
|
||||
|
||||
# 2. append score
|
||||
for idx, prompt_segment_list in enumerate(batched_prompt_segment_list):
|
||||
batched_prompt_segment_list[idx] = append_score_to_prompts(
|
||||
|
|
@ -246,13 +254,7 @@ def main():
|
|||
# == add condition frames for loop ==
|
||||
if loop_i > 0:
|
||||
refs, ms = append_generated(
|
||||
vae,
|
||||
video_clips[-1],
|
||||
refs,
|
||||
ms,
|
||||
loop_i,
|
||||
condition_frame_length,
|
||||
condition_frame_edit
|
||||
vae, video_clips[-1], refs, ms, loop_i, condition_frame_length, condition_frame_edit
|
||||
)
|
||||
|
||||
# == sampling ==
|
||||
|
|
|
|||
Loading…
Reference in a new issue