diff --git a/configs/opensora/inference-long/16x256x256-extend.py b/configs/opensora/inference-long/16x256x256-extend.py new file mode 100644 index 0000000..cdf8c47 --- /dev/null +++ b/configs/opensora/inference-long/16x256x256-extend.py @@ -0,0 +1,55 @@ +# scripts/inference_long.py +num_frames = 16 +fps = 24 // 3 +image_size = (256, 256) + +# Define model +model = dict( + type="STDiT-XL/2", + space_scale=0.5, + time_scale=1.0, + enable_flashattn=True, + enable_layernorm_kernel=True, + from_pretrained=None, +) +vae = dict( + type="VideoAutoencoderKL", + from_pretrained="stabilityai/sd-vae-ft-ema", + micro_batch_size=4, +) +text_encoder = dict( + type="t5", + from_pretrained="DeepFloyd/t5-v1_1-xxl", + model_max_length=120, +) +scheduler = dict( + type="iddpm", + # type="dpm-solver", + num_sampling_steps=100, + cfg_scale=7.0, +) +dtype = "fp16" + +# Condition +prompt_path = None +prompt = [ + "Drone view of waves crashing against the rugged cliffs along Big Sur’s garay point beach. The crashing blue waters create white-tipped waves, while the golden light of the setting sun illuminates the rocky shore. A small island with a lighthouse sits in the distance, and green shrubbery covers the cliff’s edge. The steep drop from the road down to the beach is a dramatic feat, with the cliff's edges jutting out over the sea. This is a view that captures the raw beauty of the coast and the rugged landscape of the Pacific Coast Highway.", + "In an ornate, historical hall, a massive tidal wave peaks and begins to crash. Two surfers, seizing the moment, skillfully navigate the face of the wave.", +] + +loop = 10 +condition_frame_length = 4 +reference_path = [ + "assets/images/condition/cliff.png", + "assets/images/condition/wave.png", +] +mask_strategy = [ + "0,0,0,1,0", + "0,0,0,1,0", +] # valid when reference_path is not None +# (loop id, ref id, ref start, length, target start) + +# Others +batch_size = 2 +seed = 42 +save_dir = "./outputs/samples/"