Merge branch 'main' of https://github.com/hpcaitech/Open-Sora-dev into main

README.md

@@ -20,7 +20,7 @@ Open-Sora not only democratizes access to advanced video generation techniques,
 streamlined and user-friendly platform that simplifies the complexities of video generation.
 With Open-Sora, our goal is to foster innovation, creativity, and inclusivity within the field of content creation.
 
-[[中文文档]](/docs/zh_CN/README.md) [[潞晨云部署视频教程]](https://www.bilibili.com/video/BV141421R7Ag)
+[[中文文档](/docs/zh_CN/README.md)] [[潞晨云](https://cloud.luchentech.com/)|[OpenSora镜像](https://cloud.luchentech.com/doc/docs/image/open-sora/)|[视频教程](https://www.bilibili.com/video/BV1ow4m1e7PX/?vd_source=c6b752764cd36ff0e535a768e35d98d2)]
 
 ## 📰 News
 
@@ -38,8 +38,7 @@ With Open-Sora, our goal is to foster innovation, creativity, and inclusivity wi
 
 ## 🎥 Latest Demo
 
-🔥 You can experience Open-Sora on our [🤗 Gradio application on Hugging Face](https://huggingface.co/spaces/hpcai-tech/open-sora). More samples are available in our [Gallery](https://hpcaitech.github.io/Open-Sora/).
+🔥 You can experience Open-Sora on our [🤗 Gradio application on Hugging Face](https://huggingface.co/spaces/hpcai-tech/open-sora). More samples and corresponding prompts are available in our [Gallery](https://hpcaitech.github.io/Open-Sora/).
-
 
 | **4s 720×1280**                                                                                                                                      | **4s 720×1280**                                                                                                                                      | **4s 720×1280**                                                                                                                                      |
 | ---------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- |
@@ -47,7 +46,6 @@ With Open-Sora, our goal is to foster innovation, creativity, and inclusivity wi
 | [<img src="assets/demo/v1.2/sample_0052.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/644bf938-96ce-44aa-b797-b3c0b513d64c) | [<img src="assets/demo/v1.2/sample_1719.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/272d88ac-4b4a-484d-a665-8d07431671d0) | [<img src="assets/demo/v1.2/sample_0002.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/ebbac621-c34e-4bb4-9543-1c34f8989764) |
 | [<img src="assets/demo/v1.2/sample_0011.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/a1e3a1a3-4abd-45f5-8df2-6cced69da4ca) | [<img src="assets/demo/v1.2/sample_0004.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/d6ce9c13-28e1-4dff-9644-cc01f5f11926) | [<img src="assets/demo/v1.2/sample_0061.gif" width="">](https://github.com/hpcaitech/Open-Sora/assets/99191637/561978f8-f1b0-4f4d-ae7b-45bec9001b4a) |
 
-
 <details>
 <summary>OpenSora 1.1 Demo</summary>
 

opensora/datasets/read_video.py

@@ -89,18 +89,22 @@ def read_video_av(
     video_frames = np.zeros((total_frames, height, width, 3), dtype=np.uint8)
 
     # == read ==
-    # TODO: The reading has memory leak (4G for 8 workers 1 GPU)
+    try:
-    container = av.open(filename, metadata_errors="ignore")
+        # TODO: The reading has memory leak (4G for 8 workers 1 GPU)
-    assert container.streams.video is not None
+        container = av.open(filename, metadata_errors="ignore")
-    video_frames = _read_from_stream(
+        assert container.streams.video is not None
-        video_frames,
+        video_frames = _read_from_stream(
-        container,
+            video_frames,
-        start_pts,
+            container,
-        end_pts,
+            start_pts,
-        pts_unit,
+            end_pts,
-        container.streams.video[0],
+            pts_unit,
-        {"video": 0},
+            container.streams.video[0],
-    )
+            {"video": 0},
+            filename=filename,
+        )
+    except av.AVError as e:
+        print(f"[Warning] Error while reading video {filename}: {e}")
 
     vframes = torch.from_numpy(video_frames).clone()
     del video_frames
@@ -120,6 +124,7 @@ def _read_from_stream(
     pts_unit: str,
     stream: "av.stream.Stream",
     stream_name: Dict[str, Optional[Union[int, Tuple[int, ...], List[int]]]],
+    filename: Optional[str] = None,
 ) -> List["av.frame.Frame"]:
 
     if pts_unit == "sec":
@@ -159,26 +164,28 @@ def _read_from_stream(
     try:
         # TODO check if stream needs to always be the video stream here or not
         container.seek(seek_offset, any_frame=False, backward=True, stream=stream)
-    except av.AVError:
+    except av.AVError as e:
-        # TODO add some warnings in this case
+        print(f"[Warning] Error while seeking video {filename}: {e}")
-        # print("Corrupted file?", container.name)
         return []
 
     # == main ==
     buffer_count = 0
     frames_pts = []
     cnt = 0
-    for _idx, frame in enumerate(container.decode(**stream_name)):
+    try:
-        frames_pts.append(frame.pts)
+        for _idx, frame in enumerate(container.decode(**stream_name)):
-        video_frames[cnt] = frame.to_rgb().to_ndarray()
+            frames_pts.append(frame.pts)
-        cnt += 1
+            video_frames[cnt] = frame.to_rgb().to_ndarray()
-        if cnt >= len(video_frames):
+            cnt += 1
-            break
+            if cnt >= len(video_frames):
-        if frame.pts >= end_offset:
+                break
-            if should_buffer and buffer_count < max_buffer_size:
+            if frame.pts >= end_offset:
-                buffer_count += 1
+                if should_buffer and buffer_count < max_buffer_size:
-                continue
+                    buffer_count += 1
-            break
+                    continue
+                break
+    except av.AVError as e:
+        print(f"[Warning] Error while reading video {filename}: {e}")
 
     # garbage collection for thread leakage
     container.close()