mirror of
https://github.com/hpcaitech/Open-Sora.git
synced 2026-05-21 11:59:01 +02:00
Merge branch 'main' of https://github.com/hpcaitech/Open-Sora-dev into main
This commit is contained in:
Tom Young
commit
a9fe6dd9d5
|
|
@ -20,7 +20,7 @@ Open-Sora not only democratizes access to advanced video generation techniques,
|
|||
streamlined and user-friendly platform that simplifies the complexities of video generation.
|
||||
With Open-Sora, our goal is to foster innovation, creativity, and inclusivity within the field of content creation.
|
||||
|
||||
[[中文文档]](/docs/zh_CN/README.md) [[潞晨云部署视频教程]](https://www.bilibili.com/video/BV141421R7Ag)
|
||||
[[中文文档](/docs/zh_CN/README.md)] [[潞晨云](https://cloud.luchentech.com/)|[OpenSora镜像](https://cloud.luchentech.com/doc/docs/image/open-sora/)|[视频教程](https://www.bilibili.com/video/BV1ow4m1e7PX/?vd_source=c6b752764cd36ff0e535a768e35d98d2)]
|
||||
|
||||
## 📰 News
|
||||
|
||||
|
|
@ -38,8 +38,7 @@ With Open-Sora, our goal is to foster innovation, creativity, and inclusivity wi
|
|||
|
||||
## 🎥 Latest Demo
|
||||
|
||||
🔥 You can experience Open-Sora on our [🤗 Gradio application on Hugging Face](https://huggingface.co/spaces/hpcai-tech/open-sora). More samples are available in our [Gallery](https://hpcaitech.github.io/Open-Sora/).
|
||||
|
||||
🔥 You can experience Open-Sora on our [🤗 Gradio application on Hugging Face](https://huggingface.co/spaces/hpcai-tech/open-sora). More samples and corresponding prompts are available in our [Gallery](https://hpcaitech.github.io/Open-Sora/).
|
||||
|
||||
| **4s 720×1280** | **4s 720×1280** | **4s 720×1280** |
|
||||
| ---------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||
|
|
@ -47,7 +46,6 @@ With Open-Sora, our goal is to foster innovation, creativity, and inclusivity wi
|
|||
| [<img src="assets/demo/v1.2/sample_0052.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/644bf938-96ce-44aa-b797-b3c0b513d64c) | [<img src="assets/demo/v1.2/sample_1719.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/272d88ac-4b4a-484d-a665-8d07431671d0) | [<img src="assets/demo/v1.2/sample_0002.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/ebbac621-c34e-4bb4-9543-1c34f8989764) |
|
||||
| [<img src="assets/demo/v1.2/sample_0011.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/a1e3a1a3-4abd-45f5-8df2-6cced69da4ca) | [<img src="assets/demo/v1.2/sample_0004.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/d6ce9c13-28e1-4dff-9644-cc01f5f11926) | [<img src="assets/demo/v1.2/sample_0061.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/561978f8-f1b0-4f4d-ae7b-45bec9001b4a) |
|
||||
|
||||
|
||||
<details>
|
||||
<summary>OpenSora 1.1 Demo</summary>
|
||||
|
||||
|
|
|
|||
|
|
@ -89,18 +89,22 @@ def read_video_av(
|
|||
video_frames = np.zeros((total_frames, height, width, 3), dtype=np.uint8)
|
||||
|
||||
# == read ==
|
||||
# TODO: The reading has memory leak (4G for 8 workers 1 GPU)
|
||||
container = av.open(filename, metadata_errors="ignore")
|
||||
assert container.streams.video is not None
|
||||
video_frames = _read_from_stream(
|
||||
video_frames,
|
||||
container,
|
||||
start_pts,
|
||||
end_pts,
|
||||
pts_unit,
|
||||
container.streams.video[0],
|
||||
{"video": 0},
|
||||
)
|
||||
try:
|
||||
# TODO: The reading has memory leak (4G for 8 workers 1 GPU)
|
||||
container = av.open(filename, metadata_errors="ignore")
|
||||
assert container.streams.video is not None
|
||||
video_frames = _read_from_stream(
|
||||
video_frames,
|
||||
container,
|
||||
start_pts,
|
||||
end_pts,
|
||||
pts_unit,
|
||||
container.streams.video[0],
|
||||
{"video": 0},
|
||||
filename=filename,
|
||||
)
|
||||
except av.AVError as e:
|
||||
print(f"[Warning] Error while reading video {filename}: {e}")
|
||||
|
||||
vframes = torch.from_numpy(video_frames).clone()
|
||||
del video_frames
|
||||
|
|
@ -120,6 +124,7 @@ def _read_from_stream(
|
|||
pts_unit: str,
|
||||
stream: "av.stream.Stream",
|
||||
stream_name: Dict[str, Optional[Union[int, Tuple[int, ...], List[int]]]],
|
||||
filename: Optional[str] = None,
|
||||
) -> List["av.frame.Frame"]:
|
||||
|
||||
if pts_unit == "sec":
|
||||
|
|
@ -159,26 +164,28 @@ def _read_from_stream(
|
|||
try:
|
||||
# TODO check if stream needs to always be the video stream here or not
|
||||
container.seek(seek_offset, any_frame=False, backward=True, stream=stream)
|
||||
except av.AVError:
|
||||
# TODO add some warnings in this case
|
||||
# print("Corrupted file?", container.name)
|
||||
except av.AVError as e:
|
||||
print(f"[Warning] Error while seeking video {filename}: {e}")
|
||||
return []
|
||||
|
||||
# == main ==
|
||||
buffer_count = 0
|
||||
frames_pts = []
|
||||
cnt = 0
|
||||
for _idx, frame in enumerate(container.decode(**stream_name)):
|
||||
frames_pts.append(frame.pts)
|
||||
video_frames[cnt] = frame.to_rgb().to_ndarray()
|
||||
cnt += 1
|
||||
if cnt >= len(video_frames):
|
||||
break
|
||||
if frame.pts >= end_offset:
|
||||
if should_buffer and buffer_count < max_buffer_size:
|
||||
buffer_count += 1
|
||||
continue
|
||||
break
|
||||
try:
|
||||
for _idx, frame in enumerate(container.decode(**stream_name)):
|
||||
frames_pts.append(frame.pts)
|
||||
video_frames[cnt] = frame.to_rgb().to_ndarray()
|
||||
cnt += 1
|
||||
if cnt >= len(video_frames):
|
||||
break
|
||||
if frame.pts >= end_offset:
|
||||
if should_buffer and buffer_count < max_buffer_size:
|
||||
buffer_count += 1
|
||||
continue
|
||||
break
|
||||
except av.AVError as e:
|
||||
print(f"[Warning] Error while reading video {filename}: {e}")
|
||||
|
||||
# garbage collection for thread leakage
|
||||
container.close()
|
||||
|
|
|
|||
Loading…
Reference in a new issue