From e88185fb9f67f24949939e76ccbab25163cf9015 Mon Sep 17 00:00:00 2001 From: Zangwei Zheng Date: Sun, 14 Apr 2024 02:02:59 +0800 Subject: [PATCH] update notebook --- notebooks/data.ipynb | 555 ---------------- notebooks/launch.ipynb | 1353 ++++++++++++++++++++++++++++++++++++++++ scripts/train.py | 1 + 3 files changed, 1354 insertions(+), 555 deletions(-) delete mode 100644 notebooks/data.ipynb create mode 100644 notebooks/launch.ipynb diff --git a/notebooks/data.ipynb b/notebooks/data.ipynb deleted file mode 100644 index af89d5e..0000000 --- a/notebooks/data.ipynb +++ /dev/null @@ -1,555 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Data Process Pipeline" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "First, you should add hosts in your ~/.ssh/config file" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "import paramiko\n", - "import time\n", - "\n", - "HOSTS = [\"h800-80\", \"h800-81\", \"h800-82\", \"h800-83\", \"h800-84\", \"h800-85\", \"h800-86\", \"h800-170\", \"h800-171\"]\n", - "\n", - "# load from ~/.ssh/config\n", - "ssh_config = paramiko.SSHConfig()\n", - "user_config_file = os.path.expanduser(\"~/.ssh/config\")\n", - "if os.path.exists(user_config_file):\n", - " with open(user_config_file) as f:\n", - " ssh_config.parse(f)\n", - "\n", - "\n", - "def get_ssh_config(hostname):\n", - " # get the configuration for the host\n", - " user_config = ssh_config.lookup(hostname)\n", - " cfg = {\n", - " \"hostname\": user_config[\"hostname\"],\n", - " \"username\": user_config[\"user\"],\n", - " \"port\": int(user_config[\"port\"]),\n", - " \"key_filename\": user_config[\"identityfile\"],\n", - " }\n", - " return cfg\n", - "\n", - "\n", - "def connect(hostname):\n", - " cfg = get_ssh_config(hostname)\n", - " # connect\n", - " client = paramiko.SSHClient()\n", - " client.set_missing_host_key_policy(paramiko.AutoAddPolicy())\n", - " client.connect(**cfg)\n", - " return client\n", - "\n", - "\n", - "def run_command(command, hostname, nohup=False, log_file=None):\n", - " client = connect(hostname)\n", - " print(\"HOST:\", hostname)\n", - " command = f\"bash -ic '{command}'\"\n", - " if log_file:\n", - " command = f\"{command} >> {log_file} 2>&1\"\n", - " if nohup:\n", - " command = f\"nohup {command} &\"\n", - " print(\"COMMAND:\", command)\n", - " stdin, stdout, stderr = client.exec_command(command, get_pty=False)\n", - "\n", - " stdout_str = stdout.read().decode()\n", - " stderr_str = stderr.read().decode()\n", - " if stdout_str:\n", - " print(\"==== STDOUT ====\\n\", stdout_str)\n", - " if stderr_str:\n", - " print(\"==== STDERR ====\\n\", stderr_str)\n", - "\n", - " client.close()\n", - "\n", - "\n", - "def run_command_all_hosts(command, hosts=HOSTS):\n", - " for hostname in hosts:\n", - " run_command(command, hostname)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here are tools to examine machine's status." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [], - "source": [ - "def nvidia_smi(host):\n", - " if host:\n", - " run_command(\"nvidia-smi\", host)\n", - " else:\n", - " run_command_all_hosts(\"nvidia-smi\")\n", - "\n", - "\n", - "def nvitop(host=None):\n", - " if host:\n", - " run_command(f\"/home/zhaowangbo/.local/bin/nvitop -1\", host)\n", - " else:\n", - " run_command_all_hosts(\"/home/zhaowangbo/.local/bin/nvitop -1\")\n", - "\n", - "\n", - "def ps(host=None, interest=\"python|sleep|torchrun|colossal\"):\n", - " if host:\n", - " if interest is None:\n", - " run_command(\"ps ux | cat\", host)\n", - " else:\n", - " run_command(f'ps ux | cat | grep --color=never -E \"{interest}\"', host)\n", - " else:\n", - " if interest is None:\n", - " run_command_all_hosts(\"ps ux | cat\")\n", - " else:\n", - " run_command_all_hosts(f'ps ux | cat | grep --color=never -E \"{interest}\"')\n", - "\n", - "\n", - "def kill(pid, host):\n", - " run_command(f\"kill -KILL {pid}\", host)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Here we define different tasks." - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [], - "source": [ - "OPEN_SORA_HOME = \"/home/zhaowangbo/zangwei/opensora/\"\n", - "\n", - "\n", - "def convert_dataset_cmd(input_dir, output_file, datatype=\"video\"):\n", - " commands = []\n", - " commands.append(f'echo \"Converting {input_dir} to {output_file}\"')\n", - " output_dir = os.path.dirname(output_file)\n", - "\n", - " commands.append(f\"mkdir -p {output_dir}\")\n", - " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", - " commands.append(f\"python -m tools.datasets.convert {datatype} {input_dir} --output {output_file}\")\n", - " return \" && \".join(commands), output_file\n", - "\n", - "\n", - "def get_video_info(input_file):\n", - " commands = []\n", - " base, ext = os.path.splitext(input_file)\n", - " output_file = f\"{base}_info{ext}\"\n", - " output_format = ext[1:]\n", - "\n", - " commands.append(f'echo \"Getting info of {input_file} to {output_file}\"')\n", - " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", - " commands.append(\n", - " f\"python -m tools.datasets.datautil {input_file} --output {output_file} --format {output_format} --info --fmin 1\"\n", - " )\n", - " return \" && \".join(commands), output_file\n", - "\n", - "\n", - "def get_caption_llava7b_video(input_file):\n", - " commands = []\n", - " base, ext = os.path.splitext(input_file)\n", - " output_file = f\"{base}_info{ext}\"\n", - " output_format = ext[1:]\n", - "\n", - " commands.append(f'echo \"Getting info of {input_file} to {output_file}\"')\n", - " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", - " commands.append(\n", - " f\"torchrun --nproc_per_node 8 --standalone -m tools.caption.caption_llava {input_file} --dp-size 8 --tp-size 1 --model-path liuhaotian/llava-v1.6-mistral-7b --prompt video\"\n", - " )\n", - " commands.append(\n", - " f\"python -m tools.datasets.datautil {base}_part*{ext} --output {output_file} --format {output_format} --intersection {input_file} --clean-caption --refine-llm-caption --remove-empty-caption\"\n", - " )\n", - "\n", - "\n", - "def get_caption_load(input_file):\n", - " commands = []\n", - " base, ext = os.path.splitext(input_file)\n", - " output_file = f\"{base}_caption{ext}\"\n", - " output_format = ext[1:]\n", - "\n", - " commands.append(f'echo \"Getting caption of {input_file} to {output_file}\"')\n", - " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", - " commands.append(\n", - " f\"python -m tools.datasets.datautil {input_file} --output {output_file} --format {output_format} --load-caption json --remove-empty-caption --clean-caption\"\n", - " )\n", - " return \" && \".join(commands), output_file\n", - "\n", - "\n", - "def get_aesthetic_score(input_file):\n", - " commands = []\n", - " base, ext = os.path.splitext(input_file)\n", - " output_file = f\"{base}_aes{ext}\"\n", - " output_format = ext[1:]\n", - "\n", - " commands.append(f'echo \"Getting aesthetic score of {input_file} to {output_file}\"')\n", - " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", - " commands.append(f\"torchrun --standalone --nproc_per_node 8 -m tools.scoring.aesthetic.inference {input_file}\")\n", - " commands.append(\n", - " f\"python -m tools.datasets.datautil {base}_part*{ext} --output {output_file} --format {output_format} --sort aes\"\n", - " )\n", - " return \" && \".join(commands), output_file\n", - "\n", - "\n", - "def get_flow_score(input_file):\n", - " commands = []\n", - " base, ext = os.path.splitext(input_file)\n", - " output_file = f\"{base}_flow{ext}\"\n", - "\n", - " commands.append(f'echo \"Getting flow score of {input_file} to {output_file}\"')\n", - " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", - " commands.append(f\"torchrun --standalone --nproc_per_node 8 -m tools.scoring.optical_flow.inference {input_file}\")\n", - " return \" && \".join(commands), output_file\n", - "\n", - "\n", - "def get_match_score(input_file):\n", - " commands = []\n", - " base, ext = os.path.splitext(input_file)\n", - " output_file = f\"{base}_match{ext}\"\n", - "\n", - " commands.append(f'echo \"Getting match score of {input_file} to {output_file}\"')\n", - " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", - " commands.append(f\"torchrun --standalone --nproc_per_node 8 -m tools.scoring.matching.inference {input_file}\")\n", - " return \" && \".join(commands), output_file\n", - "\n", - "\n", - "def get_cmotion_score(input_file):\n", - " commands = []\n", - " base, ext = os.path.splitext(input_file)\n", - " output_file = f\"{base}_cmotion{ext}\"\n", - "\n", - " commands.append(f'echo \"Getting cmotion score of {input_file} to {output_file}\"')\n", - " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", - " commands.append(f\"python -m tools.caption.camera_motion_detect {input_file}\")\n", - " return \" && \".join(commands), output_file\n", - "\n", - "\n", - "def get_commands(job_list):\n", - " commands = []\n", - " output_file = None\n", - " for job in job_list:\n", - " cmd = job.pop(\"cmd\")\n", - " if output_file is None:\n", - " command, output_file = cmd(**job)\n", - " commands.append(command)\n", - " else:\n", - " job[\"input_file\"] = output_file\n", - " command, output_file = cmd(**job)\n", - " commands.append(command)\n", - " commands.append(f'echo \"All Done!\"')\n", - " return \" && \".join(commands), output_file" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "The following is the pipeline for panda." - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "echo \"Getting flow score of /mnt/hdd/data/panda70m_by/raw/meta/split-16/meta_info_caption.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-16/meta_info_caption_flow.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && torchrun --standalone --nproc_per_node 8 -m tools.scoring.optical_flow.inference /mnt/hdd/data/panda70m_by/raw/meta/split-16/meta_info_caption.csv && echo \"All Done!\"\n", - "/mnt/hdd/data/panda70m_by/raw/meta/split-16/meta_info_caption_flow.csv\n" - ] - } - ], - "source": [ - "host = \"h800-83\"\n", - "log_file = os.path.join(OPEN_SORA_HOME, \"logs/data-panda-16-split.log\")\n", - "input_dir = \"/mnt/disk1/data-panda/16\"\n", - "output_file = \"/mnt/hdd/data/panda70m_by/raw/meta/split-16/meta.csv\"\n", - "cmd, output_file = get_commands(\n", - " [\n", - " # {\n", - " # \"cmd\": convert_dataset_cmd,\n", - " # \"input_dir\": input_dir,\n", - " # \"output_file\": output_file,\n", - " # },\n", - " # {\n", - " # \"cmd\": get_video_info,\n", - " # },\n", - " # {\n", - " # \"cmd\": get_caption_load,\n", - " # },\n", - " # {\n", - " # \"cmd\": get_aesthetic_score,\n", - " # },\n", - " # {\n", - " # \"cmd\": get_flow_score,\n", - " # },\n", - " # {\n", - " # \"cmd\": get_match_score,\n", - " # },\n", - " # {\n", - " # \"cmd\": get_cmotion_score,\n", - " # },\n", - " ]\n", - ")\n", - "print(cmd)\n", - "print(output_file)" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-8/meta_loadjson_noempty_clean.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-8/meta_loadjson_noempty_clean_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-8/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-8/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1 && echo \"All Done!\"\n", - "/mnt/hdd/data/panda70m_by/raw/meta/split-8/meta_loadjson_noempty_clean_info.csv\n" - ] - } - ], - "source": [ - "host = \"h800-81\"\n", - "split = 8\n", - "log_file = os.path.join(OPEN_SORA_HOME, f\"logs/data-panda-{split}-split.log\")\n", - "cmd, output_file = get_commands(\n", - " [\n", - " {\n", - " \"cmd\": get_video_info,\n", - " \"input_file\": f\"/mnt/hdd/data/panda70m_by/raw/meta/split-{split}/meta_loadjson_noempty_clean.csv\",\n", - " },\n", - " ]\n", - ")\n", - "print(cmd)\n", - "print(output_file)" - ] - }, - { - "cell_type": "code", - "execution_count": 13, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "HOST: h800-83\n", - "COMMAND: nohup bash -ic 'echo \"Getting flow score of /mnt/hdd/data/panda70m_by/raw/meta/split-16/meta_info_caption.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-16/meta_info_caption_flow.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && torchrun --standalone --nproc_per_node 8 -m tools.scoring.optical_flow.inference /mnt/hdd/data/panda70m_by/raw/meta/split-16/meta_info_caption.csv && echo \"All Done!\"' >> /home/zhaowangbo/zangwei/opensora/logs/data-panda-16-split.log 2>&1 &\n", - "HOST: h800-83\n", - "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", - "==== STDOUT ====\n", - " zhaowan+ 891142 0.8 0.0 20886768 197296 pts/10 Sl+ 22:36 0:22 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/torchrun --standalone --nproc_per_node 1 -m tools.scoring.aesthetic.inference /mnt/hdd/data/panda70m_by/raw/meta/split-16/meta_info_caption.csv --num_frames 2 --num_workers 0\n", - "zhaowan+ 891294 3.0 0.0 74364928 1938304 ? Ssl 22:36 1:15 /home/zhaowangbo/.conda/envs/opensora/bin/python -u -m tools.scoring.aesthetic.inference /mnt/hdd/data/panda70m_by/raw/meta/split-16/meta_info_caption.csv --num_frames 2 --num_workers 0\n", - "zhaowan+ 2100459 2.7 0.0 14180 5292 ? S 23:17 0:00 bash -ic echo \"Getting flow score of /mnt/hdd/data/panda70m_by/raw/meta/split-16/meta_info_caption.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-16/meta_info_caption_flow.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && torchrun --standalone --nproc_per_node 8 -m tools.scoring.optical_flow.inference /mnt/hdd/data/panda70m_by/raw/meta/split-16/meta_info_caption.csv && echo \"All Done!\"\n", - "zhaowan+ 2100656 13.0 0.0 2803920 18924 ? I 23:17 0:00 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/torchrun --standalone --nproc_per_node 8 -m tools.scoring.optical_flow.inference /mnt/hdd/data/panda70m_by/raw/meta/split-16/meta_info_caption.csv\n", - "zhaowan+ 2100724 10.0 0.0 14176 5104 ? Ss 23:17 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", - "zhaowan+ 2100840 0.0 0.0 12124 728 ? S 23:17 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", - "\n", - "==== STDERR ====\n", - " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", - "bash: no job control in this shell\n", - "\n" - ] - } - ], - "source": [ - "run_command(cmd, host, log_file=log_file, nohup=True)\n", - "ps(host)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "Using following commands to monitor the status of the jobs." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "HOST: h800-80\n", - "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", - "==== STDOUT ====\n", - " zhaowan+ 3707972 0.1 0.0 8492 5240 ? S 21:07 0:00 bash -ic echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1 && echo \"All Done!\"\n", - "zhaowan+ 3708645 33.6 0.0 17792816 399760 ? Sl 21:07 0:21 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710337 0.4 0.0 18128668 386580 ? Sl 21:07 0:00 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710347 3.8 0.0 17590060 397448 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710349 4.2 0.0 17590060 397468 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710351 4.1 0.0 17590060 397460 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710353 4.8 0.0 17590060 397468 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710355 4.9 0.0 17590060 397456 ? R 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710358 4.6 0.0 17590060 397464 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710360 4.5 0.0 17590060 397456 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710362 5.4 0.0 17590060 397472 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710366 5.5 0.0 17590060 397500 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710368 5.3 0.0 17590060 397504 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710370 5.0 0.0 17590060 397496 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710377 3.6 0.0 17590060 397476 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710379 4.5 0.0 17590060 397472 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710381 4.5 0.0 17590060 397484 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710391 3.8 0.0 17590060 397468 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710401 5.2 0.0 17590060 397508 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710403 5.9 0.0 17590060 397500 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710409 4.6 0.0 17590060 397500 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710411 5.0 0.0 17590060 397472 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710425 4.0 0.0 17590060 397456 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710435 5.3 0.0 17590060 397500 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710448 5.2 0.0 17590060 397492 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710450 4.7 0.0 17590060 397512 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710459 5.1 0.0 17590060 397484 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710466 5.0 0.0 17590060 397516 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710468 4.5 0.0 17590060 397488 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710470 3.4 0.0 17590060 397492 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710472 4.6 0.0 17590060 397548 ? R 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710474 4.7 0.0 17590060 397520 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710476 4.5 0.0 17590060 397512 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710478 4.4 0.0 17590060 397524 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710480 5.1 0.0 17590060 397524 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710482 4.5 0.0 17590060 397524 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710483 5.6 0.0 17590060 397504 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710486 4.6 0.0 17590060 397532 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710489 3.5 0.0 17590060 397520 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710491 4.5 0.0 17590060 397528 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710493 5.5 0.0 17590060 397540 ? R 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710495 4.1 0.0 17590060 397532 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710497 4.1 0.0 17590060 397548 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710499 5.3 0.0 17590060 397536 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710501 4.1 0.0 17590060 397552 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710503 4.0 0.0 17590060 397540 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710505 4.9 0.0 17590060 397544 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710507 4.9 0.0 17590060 397544 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710509 4.6 0.0 17590060 397532 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710511 4.9 0.0 17590060 397588 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710513 4.4 0.0 17590060 397536 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710515 4.5 0.0 17590060 397536 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710517 5.5 0.0 17590060 397536 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710519 4.5 0.0 17590060 397552 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710521 5.3 0.0 17590060 397552 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710523 6.6 0.0 17590060 398028 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710525 5.7 0.0 17590060 397548 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710527 5.1 0.0 17590060 397544 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710529 6.8 0.0 17590060 397568 ? I 21:07 0:03 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710531 3.9 0.0 17590060 397560 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710533 4.5 0.0 17590060 397572 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710535 5.6 0.0 17590060 397548 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710537 6.1 0.0 17590060 398048 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710539 5.2 0.0 17590060 397552 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710541 4.4 0.0 17590060 397536 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710543 6.8 0.0 17590060 398020 ? R 21:07 0:03 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710545 4.4 0.0 17590060 397556 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710547 5.5 0.0 17590060 397552 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710549 4.3 0.0 17590060 397564 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710551 4.6 0.0 17590060 397564 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710553 5.1 0.0 17590060 397568 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710555 4.5 0.0 17590060 397564 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710556 5.9 0.0 17590060 397580 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710559 5.0 0.0 17590060 397568 ? D 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710561 6.4 0.0 17590060 397588 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710563 5.0 0.0 17590060 397644 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710565 5.1 0.0 17590060 397584 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710566 5.5 0.0 17590060 397584 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710568 5.1 0.0 17590060 397588 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710571 4.4 0.0 17590060 397600 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710573 5.1 0.0 17590060 397644 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710575 4.4 0.0 17590060 397652 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710577 4.6 0.0 17590060 397652 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710579 3.8 0.0 17590060 397596 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710581 5.0 0.0 17590060 397588 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710583 4.5 0.0 17590060 397596 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710585 4.4 0.0 17590060 397612 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710587 4.2 0.0 17590060 397596 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710589 5.7 0.0 17590060 397612 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710591 4.4 0.0 17590060 397596 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710593 5.3 0.0 17590060 397604 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710595 5.2 0.0 17590060 397604 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710597 4.5 0.0 17590060 397608 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710599 5.2 0.0 17590060 397608 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710601 5.2 0.0 17590060 397620 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710603 5.4 0.0 17590060 397600 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710605 3.5 0.0 17590060 397572 ? I 21:07 0:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710607 5.1 0.0 17590060 397612 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3710609 6.0 0.0 17590060 397628 ? I 21:07 0:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --format csv --info --fmin 1\n", - "zhaowan+ 3862481 5.0 0.0 8488 5328 ? Ss 21:08 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", - "zhaowan+ 3868112 0.0 0.0 6412 724 ? S 21:08 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", - "\n", - "==== STDERR ====\n", - " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", - "bash: no job control in this shell\n", - "\n" - ] - } - ], - "source": [ - "ps(host)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "nvitop(host)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "kill(, host)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.13" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/notebooks/launch.ipynb b/notebooks/launch.ipynb new file mode 100644 index 0000000..4d28123 --- /dev/null +++ b/notebooks/launch.ipynb @@ -0,0 +1,1353 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Data Process Pipeline" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First, you should add hosts in your ~/.ssh/config file" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import paramiko\n", + "import time\n", + "\n", + "HOSTS = [\"h800-80\", \"h800-81\", \"h800-82\", \"h800-83\", \"h800-84\", \"h800-85\", \"h800-86\", \"h800-170\", \"h800-171\"]\n", + "\n", + "# load from ~/.ssh/config\n", + "ssh_config = paramiko.SSHConfig()\n", + "user_config_file = os.path.expanduser(\"~/.ssh/config\")\n", + "if os.path.exists(user_config_file):\n", + " with open(user_config_file) as f:\n", + " ssh_config.parse(f)\n", + "\n", + "\n", + "def get_ssh_config(hostname):\n", + " # get the configuration for the host\n", + " user_config = ssh_config.lookup(hostname)\n", + " cfg = {\n", + " \"hostname\": user_config[\"hostname\"],\n", + " \"username\": user_config[\"user\"],\n", + " \"port\": int(user_config[\"port\"]),\n", + " \"key_filename\": user_config[\"identityfile\"],\n", + " }\n", + " return cfg\n", + "\n", + "\n", + "def connect(hostname):\n", + " cfg = get_ssh_config(hostname)\n", + " # connect\n", + " client = paramiko.SSHClient()\n", + " client.set_missing_host_key_policy(paramiko.AutoAddPolicy())\n", + " client.connect(**cfg)\n", + " return client\n", + "\n", + "\n", + "def run_command(command, hostname, nohup=False, log_file=None, sleep=None):\n", + " client = connect(hostname)\n", + " print(\"HOST:\", hostname)\n", + " if sleep:\n", + " command = f\"sleep {sleep}; {command}\"\n", + " command = f\"bash -ic '{command}'\"\n", + " if log_file:\n", + " command = f\"{command} >> {log_file} 2>&1\"\n", + " if nohup:\n", + " command = f\"nohup {command} &\"\n", + " print(\"COMMAND:\", command)\n", + " stdin, stdout, stderr = client.exec_command(command, get_pty=False)\n", + "\n", + " stdout_str = stdout.read().decode()\n", + " stderr_str = stderr.read().decode()\n", + " if stdout_str:\n", + " print(\"==== STDOUT ====\\n\", stdout_str)\n", + " if stderr_str:\n", + " print(\"==== STDERR ====\\n\", stderr_str)\n", + "\n", + " client.close()\n", + "\n", + "\n", + "def run_command_all_hosts(command, hosts=HOSTS):\n", + " for hostname in hosts:\n", + " run_command(command, hostname)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here are tools to examine machine's status." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def nvidia_smi(host):\n", + " if host:\n", + " run_command(\"nvidia-smi\", host)\n", + " else:\n", + " run_command_all_hosts(\"nvidia-smi\")\n", + "\n", + "\n", + "def nvitop(host=None):\n", + " if host:\n", + " run_command(f\"/home/zhaowangbo/.local/bin/nvitop -1\", host)\n", + " else:\n", + " run_command_all_hosts(\"/home/zhaowangbo/.local/bin/nvitop -1\")\n", + "\n", + "\n", + "def ps(host=None, interest=\"python|sleep|torchrun|colossal\"):\n", + " if host:\n", + " if interest is None:\n", + " run_command(\"ps ux | cat\", host)\n", + " else:\n", + " run_command(f'ps ux | cat | grep --color=never -E \"{interest}\"', host)\n", + " else:\n", + " if interest is None:\n", + " run_command_all_hosts(\"ps ux | cat\")\n", + " else:\n", + " run_command_all_hosts(f'ps ux | cat | grep --color=never -E \"{interest}\"')\n", + "\n", + "\n", + "def kill(pid, host):\n", + " run_command(f\"kill -KILL {pid}\", host)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here we define different tasks." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "OPEN_SORA_HOME = \"/home/zhaowangbo/zangwei/opensora/\"\n", + "\n", + "\n", + "def convert_dataset_cmd(input_dir, output_file, datatype=\"video\"):\n", + " commands = []\n", + " commands.append(f'echo \"Converting {input_dir} to {output_file}\"')\n", + " output_dir = os.path.dirname(output_file)\n", + "\n", + " commands.append(f\"mkdir -p {output_dir}\")\n", + " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", + " commands.append(f\"python -m tools.datasets.convert {datatype} {input_dir} --output {output_file}\")\n", + " return \" && \".join(commands), output_file\n", + "\n", + "\n", + "def get_video_info(input_file):\n", + " commands = []\n", + " base, ext = os.path.splitext(input_file)\n", + " output_file = f\"{base}_info{ext}\"\n", + " output_format = ext[1:]\n", + "\n", + " commands.append(f'echo \"Getting info of {input_file} to {output_file}\"')\n", + " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", + " commands.append(\n", + " f\"python -m tools.datasets.datautil {input_file} --output {output_file} --format {output_format} --info --fmin 1\"\n", + " )\n", + " return \" && \".join(commands), output_file\n", + "\n", + "def get_video_info_torchvision(input_file):\n", + " commands = []\n", + " base, ext = os.path.splitext(input_file)\n", + " output_file = f\"{base}_info{ext}\"\n", + " output_format = ext[1:]\n", + "\n", + " commands.append(f'echo \"Getting info of {input_file} to {output_file}\"')\n", + " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", + " commands.append(\n", + " f\"python -m tools.datasets.datautil {input_file} --output {output_file} --format {output_format} --video-info --fmin 1\"\n", + " )\n", + " return \" && \".join(commands), output_file\n", + "\n", + "\n", + "def get_caption_llava7b_video(input_file):\n", + " commands = []\n", + " base, ext = os.path.splitext(input_file)\n", + " output_file = f\"{base}_info{ext}\"\n", + " output_format = ext[1:]\n", + "\n", + " commands.append(f'echo \"Getting info of {input_file} to {output_file}\"')\n", + " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", + " commands.append(\n", + " f\"torchrun --nproc_per_node 8 --standalone -m tools.caption.caption_llava {input_file} --dp-size 8 --tp-size 1 --model-path liuhaotian/llava-v1.6-mistral-7b --prompt video\"\n", + " )\n", + " commands.append(\n", + " f\"python -m tools.datasets.datautil {base}_part*{ext} --output {output_file} --format {output_format} --intersection {input_file} --clean-caption --refine-llm-caption --remove-empty-caption\"\n", + " )\n", + "\n", + "\n", + "def get_caption_load(input_file):\n", + " commands = []\n", + " base, ext = os.path.splitext(input_file)\n", + " output_file = f\"{base}_caption{ext}\"\n", + " output_format = ext[1:]\n", + "\n", + " commands.append(f'echo \"Getting caption of {input_file} to {output_file}\"')\n", + " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", + " commands.append(\n", + " f\"python -m tools.datasets.datautil {input_file} --output {output_file} --format {output_format} --load-caption json --remove-empty-caption --clean-caption\"\n", + " )\n", + " return \" && \".join(commands), output_file\n", + "\n", + "\n", + "def get_aesthetic_score(input_file):\n", + " commands = []\n", + " base, ext = os.path.splitext(input_file)\n", + " output_file = f\"{base}_aes{ext}\"\n", + " output_format = ext[1:]\n", + "\n", + " commands.append(f'echo \"Getting aesthetic score of {input_file} to {output_file}\"')\n", + " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", + " commands.append(f\"torchrun --standalone --nproc_per_node 8 -m tools.scoring.aesthetic.inference {input_file}\")\n", + " commands.append(\n", + " f\"python -m tools.datasets.datautil {base}_part*{ext} --output {output_file} --format {output_format} --sort aes\"\n", + " )\n", + " return \" && \".join(commands), output_file\n", + "\n", + "\n", + "def get_flow_score(input_file):\n", + " commands = []\n", + " base, ext = os.path.splitext(input_file)\n", + " output_file = f\"{base}_flow{ext}\"\n", + "\n", + " commands.append(f'echo \"Getting flow score of {input_file} to {output_file}\"')\n", + " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", + " commands.append(f\"torchrun --standalone --nproc_per_node 8 -m tools.scoring.optical_flow.inference {input_file}\")\n", + " return \" && \".join(commands), output_file\n", + "\n", + "\n", + "def get_match_score(input_file):\n", + " commands = []\n", + " base, ext = os.path.splitext(input_file)\n", + " output_file = f\"{base}_match{ext}\"\n", + "\n", + " commands.append(f'echo \"Getting match score of {input_file} to {output_file}\"')\n", + " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", + " commands.append(f\"torchrun --standalone --nproc_per_node 8 -m tools.scoring.matching.inference {input_file}\")\n", + " return \" && \".join(commands), output_file\n", + "\n", + "\n", + "def get_cmotion_score(input_file):\n", + " commands = []\n", + " base, ext = os.path.splitext(input_file)\n", + " output_file = f\"{base}_cmotion{ext}\"\n", + "\n", + " commands.append(f'echo \"Getting cmotion score of {input_file} to {output_file}\"')\n", + " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", + " commands.append(f\"python -m tools.caption.camera_motion_detect {input_file}\")\n", + " return \" && \".join(commands), output_file\n", + "\n", + "\n", + "def get_commands(job_list):\n", + " commands = []\n", + " output_file = None\n", + " for job in job_list:\n", + " cmd = job.pop(\"cmd\")\n", + " if output_file is None:\n", + " command, output_file = cmd(**job)\n", + " commands.append(command)\n", + " else:\n", + " job[\"input_file\"] = output_file\n", + " command, output_file = cmd(**job)\n", + " commands.append(command)\n", + " commands.append(f'echo \"All Done!\"')\n", + " return \" && \".join(commands), output_file" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The following is the pipeline for panda." + ] + }, + { + "cell_type": "code", + "execution_count": 28, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "echo \"Converting /mnt/disk0/data-panda/24 to /mnt/hdd/data/panda70m_by/raw/meta/split-24/meta.csv\" && mkdir -p /mnt/hdd/data/panda70m_by/raw/meta/split-24 && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.convert video /mnt/disk0/data-panda/24 --output /mnt/hdd/data/panda70m_by/raw/meta/split-24/meta.csv && echo \"Getting caption of /mnt/hdd/data/panda70m_by/raw/meta/split-24/meta.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-24/meta_caption.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-24/meta.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-24/meta_caption.csv --format csv --load-caption json --remove-empty-caption --clean-caption && echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-24/meta_caption.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-24/meta_caption_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-24/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-24/meta_caption_info.csv --format csv --video-info --fmin 1 && echo \"All Done!\"\n", + "/mnt/hdd/data/panda70m_by/raw/meta/split-24/meta_caption_info.csv\n" + ] + } + ], + "source": [ + "host = \"h800-81\"\n", + "split = 24\n", + "input_dir = f\"/mnt/disk0/data-panda/{split}\"\n", + "log_file = os.path.join(OPEN_SORA_HOME, f\"logs/data-panda-{split}-split.log\")\n", + "output_file = f\"/mnt/hdd/data/panda70m_by/raw/meta/split-{split}/meta.csv\"\n", + "cmd, output_file = get_commands(\n", + " [\n", + " {\n", + " \"cmd\": convert_dataset_cmd,\n", + " \"input_dir\": input_dir,\n", + " \"output_file\": output_file,\n", + " },\n", + " {\n", + " \"cmd\": get_caption_load,\n", + " },\n", + " {\n", + " \"cmd\": get_video_info_torchvision,\n", + " },\n", + " # {\n", + " # \"cmd\": get_aesthetic_score,\n", + " # },\n", + " # {\n", + " # \"cmd\": get_flow_score,\n", + " # },\n", + " # {\n", + " # \"cmd\": get_match_score,\n", + " # },\n", + " # {\n", + " # \"cmd\": get_cmotion_score,\n", + " # },\n", + " ]\n", + ")\n", + "print(cmd)\n", + "print(output_file)" + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean_info.csv --format csv --video-info --fmin 1 && echo \"All Done!\"\n", + "/mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean_info.csv\n" + ] + } + ], + "source": [ + "host = \"h800-82\"\n", + "split = 7\n", + "log_file = os.path.join(OPEN_SORA_HOME, f\"logs/data-panda-{split}-split.log\")\n", + "cmd, output_file = get_commands(\n", + " [\n", + " {\n", + " \"cmd\": get_video_info_torchvision,\n", + " \"input_file\": f\"/mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean.csv\",\n", + " },\n", + " ]\n", + ")\n", + "print(cmd)\n", + "print(output_file)" + ] + }, + { + "cell_type": "code", + "execution_count": 46, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "HOST: h800-82\n", + "COMMAND: nohup bash -ic 'sleep 9h; echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean_info.csv --format csv --video-info --fmin 1 && echo \"All Done!\"' >> /home/zhaowangbo/zangwei/opensora/logs/data-panda-7-split.log 2>&1 &\n", + "HOST: h800-82\n", + "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", + "==== STDOUT ====\n", + " zhaowan+ 174693 0.0 0.0 8496 5148 ? S 01:59 0:00 bash -ic sleep 3h; echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-3/meta_remove_corrupted_loadjson_noempty_clean_info.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-3/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-3/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-3/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1 && echo \"All Done!\"\n", + "zhaowan+ 176000 0.0 0.0 5492 588 ? S 01:59 0:00 sleep 3h\n", + "zhaowan+ 203444 0.0 0.0 8496 5164 ? S 02:00 0:00 bash -ic sleep 6h; echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-5/meta_loadjson_noempty_clean_info.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-5/meta_loadjson_noempty_clean_info_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-5/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-5/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1 && echo \"All Done!\"\n", + "zhaowan+ 244363 0.0 0.0 5492 524 ? S 02:00 0:00 sleep 6h\n", + "zhaowan+ 262359 0.1 0.0 8496 5228 ? S 02:00 0:00 bash -ic sleep 9h; echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-7/meta_loadjson_noempty_clean_info.csv --format csv --video-info --fmin 1 && echo \"All Done!\"\n", + "zhaowan+ 319583 0.0 0.0 5492 592 ? S 02:01 0:00 sleep 9h\n", + "zhaowan+ 322729 10.0 0.0 8488 5056 ? Ss 02:01 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", + "zhaowan+ 323854 0.0 0.0 6412 660 ? S 02:01 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", + "zhaowan+ 2287475 0.0 0.0 8496 5140 ? S 01:28 0:00 bash -ic echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1 && echo \"All Done!\"\n", + "zhaowan+ 2287568 2.0 0.0 17858200 492768 ? Sl 01:28 0:41 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287884 3.5 0.0 24781812 445572 ? Sl 01:28 1:10 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287889 81.7 0.2 24651188 4575400 ? Rl 01:28 26:49 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287891 81.2 0.4 29534760 9090616 ? Rl 01:28 26:39 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287893 87.3 0.2 24911588 4835712 ? Rl 01:28 28:39 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287895 79.0 0.3 27911916 7467320 ? Il 01:28 25:57 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287897 82.4 0.3 28748088 8426420 ? Rl 01:28 27:03 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287899 81.0 0.2 26391472 5946128 ? Rl 01:28 26:35 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287901 81.2 0.4 28788064 8465952 ? Rl 01:28 26:40 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287903 81.3 0.1 23493264 3171844 ? Rl 01:28 26:42 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287905 82.2 0.3 27270164 6702972 ? Rl 01:28 26:59 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287907 83.1 0.1 24441824 4119772 ? Il 01:28 27:16 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287909 81.7 0.3 27606444 7407280 ? Rl 01:28 26:50 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287911 86.5 0.2 25645400 5324248 ? Rl 01:28 28:23 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287913 79.0 0.2 24647732 4571400 ? Rl 01:28 25:56 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287915 79.7 0.1 24367884 4136168 ? Il 01:28 26:10 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287917 78.3 0.5 32753232 12554632 ? Rl 01:28 25:42 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287919 98.5 0.2 26108244 6033108 ? Rl 01:28 32:20 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287921 82.0 0.3 29182656 7695120 ? Rl 01:28 26:54 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287923 78.2 0.2 25513264 5068400 ? Rl 01:28 25:40 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287925 77.3 0.4 29118336 8797036 ? Rl 01:28 25:23 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287927 102 0.2 26020336 5944484 ? Il 01:28 33:30 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287929 80.4 0.6 33263572 12909624 ? Rl 01:28 26:24 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287931 78.4 0.2 25538904 5462432 ? Rl 01:28 25:45 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287933 87.4 0.2 25253180 4809388 ? Rl 01:28 28:40 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287935 83.9 0.2 24572732 4496176 ? Rl 01:28 27:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287937 81.3 0.3 27060480 6738532 ? Rl 01:28 26:41 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287939 82.3 0.3 28575336 8253820 ? Rl 01:28 27:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287941 80.6 0.2 26125156 5893380 ? Rl 01:28 26:28 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287943 82.4 0.2 26480452 6248656 ? Il 01:28 27:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287945 81.1 0.2 25879780 5729980 ? Rl 01:28 26:37 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287947 86.3 0.4 30114300 9792536 ? Rl 01:28 28:21 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287949 80.7 0.4 29370340 9294764 ? Rl 01:28 26:29 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287951 82.2 0.2 26188556 5866276 ? Rl 01:28 26:58 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287953 79.7 0.2 25374756 5142956 ? Rl 01:28 26:09 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287955 81.0 0.3 28266836 8067336 ? Rl 01:28 26:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287957 83.5 0.2 25990340 5913772 ? Rl 01:28 27:24 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287959 79.6 0.2 24579252 4256972 ? Il 01:28 26:08 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287961 82.9 0.3 26600832 6369392 ? Rl 01:28 27:13 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287963 81.6 0.3 28275324 7830600 ? Il 01:28 26:48 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287965 79.2 0.1 23359892 3160384 ? Rl 01:28 26:00 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287967 80.9 0.3 27079216 7002960 ? Rl 01:28 26:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287969 78.4 0.1 23474836 3243416 ? Rl 01:28 25:44 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287971 79.2 0.3 28379420 8303124 ? Rl 01:28 25:59 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287973 82.0 0.2 24884224 4440028 ? Rl 01:28 26:55 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287975 83.1 0.2 25597020 5520652 ? Rl 01:28 27:16 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287977 81.0 0.3 28079348 7880896 ? Il 01:28 26:35 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287979 81.3 0.4 29262368 8771556 ? Rl 01:28 26:41 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287981 83.1 0.3 27351840 6907488 ? Rl 01:28 27:18 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287984 78.2 0.2 25304436 4982552 ? Rl 01:28 25:40 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287986 81.5 0.4 30241620 9920112 ? Rl 01:28 26:44 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287988 82.3 0.3 28461724 8386044 ? Rl 01:28 27:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287990 79.5 0.3 28202856 7880560 ? Rl 01:28 26:05 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287992 79.2 0.3 27899840 7577860 ? Rl 01:28 26:00 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287994 80.2 0.2 24834092 4511812 ? Il 01:28 26:20 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287996 77.5 0.3 31074444 7659556 ? Rl 01:28 25:27 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2287998 89.5 0.3 27467552 7007244 ? Sl 01:28 29:23 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288000 77.9 0.2 25211948 5135412 ? Rl 01:28 25:35 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288002 79.7 0.3 28495028 8263048 ? Il 01:28 26:10 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288004 81.7 0.3 26487876 6411312 ? Il 01:28 26:49 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288006 78.8 0.6 33437492 13164340 ? Sl 01:28 25:52 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288008 80.2 0.3 28591492 8393012 ? Rl 01:28 26:19 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288010 82.6 0.3 27772016 7572420 ? Rl 01:28 27:07 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288012 81.1 0.3 28174944 7730712 ? Rl 01:28 26:37 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288014 81.6 0.2 26763680 6319492 ? Rl 01:28 26:47 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288016 79.4 0.1 23597400 3521392 ? Rl 01:28 26:04 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288018 82.0 0.4 31685888 9411444 ? Rl 01:28 26:55 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288020 87.3 0.1 24291056 3970144 ? Rl 01:28 28:39 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288022 81.2 0.2 25739752 5295276 ? Il 01:28 26:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288024 81.4 0.2 25958552 5513476 ? Il 01:28 26:43 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288026 82.3 0.5 31563972 10996648 ? Rl 01:28 27:00 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288028 80.8 0.2 26221664 5776392 ? Rl 01:28 26:31 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288030 77.4 0.1 23602524 3280424 ? Rl 01:28 25:24 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288032 82.5 0.3 28330380 7852580 ? Rl 01:28 27:04 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288034 80.6 0.2 26390212 5945064 ? Rl 01:28 26:26 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288036 82.6 0.2 26109004 6033416 ? Rl 01:28 27:06 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288038 81.0 0.3 27059868 6737516 ? Rl 01:28 26:35 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288040 81.2 0.3 28404576 8082540 ? Rl 01:28 26:39 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288042 79.5 0.2 26106588 5784396 ? Rl 01:28 26:05 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288044 80.8 0.3 28032208 7587776 ? Rl 01:28 26:30 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288046 87.6 0.2 25301300 4857680 ? Il 01:28 28:45 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288048 80.1 0.1 23968632 3892940 ? Rl 01:28 26:17 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288050 80.0 0.3 28185112 8108204 ? Rl 01:28 26:15 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288052 81.6 0.3 26979508 6534224 ? Rl 01:28 26:47 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288054 79.4 0.2 26646384 6209700 ? Rl 01:28 26:03 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288056 81.2 0.2 25848948 5650180 ? Rl 01:28 26:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288058 78.9 0.2 25670332 5226216 ? Rl 01:28 25:52 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288060 77.1 0.2 25547472 5225088 ? Il 01:28 25:17 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288062 79.2 0.2 25660752 5462072 ? Il 01:28 25:59 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288064 77.7 0.4 29294668 8855704 ? Rl 01:28 25:30 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288066 81.7 0.2 25643340 5443816 ? Il 01:28 26:48 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288068 80.6 0.3 27484244 7161984 ? Rl 01:28 26:27 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288070 85.0 0.1 23549224 3472712 ? Rl 01:28 27:54 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288072 81.4 0.4 29936800 9860108 ? Rl 01:28 26:43 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288074 81.8 0.2 25994076 5672512 ? Rl 01:28 26:51 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288076 81.7 0.3 27228232 7152348 ? Rl 01:28 26:48 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288078 77.4 0.2 25083652 4761348 ? Rl 01:28 25:23 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2288080 81.0 0.4 31252608 8537844 ? Rl 01:28 26:35 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-0/meta_remove_corrupted_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "\n", + "==== STDERR ====\n", + " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", + "bash: no job control in this shell\n", + "\n" + ] + } + ], + "source": [ + "sleep = \"9h\"\n", + "run_command(cmd, host, log_file=log_file, nohup=True, sleep=sleep)\n", + "ps(host)" + ] + }, + { + "cell_type": "code", + "execution_count": 38, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "HOST: h800-84\n", + "COMMAND: bash -ic 'kill -KILL 1922317'\n", + "==== STDERR ====\n", + " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", + "bash: no job control in this shell\n", + "\n", + "HOST: h800-84\n", + "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", + "==== STDOUT ====\n", + " zhaowan+ 1262361 0.0 0.0 14180 5296 ? S 01:54 0:00 bash -ic sleep 3h; echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-2/meta_remove_corrupted_loadjson_noempty_clean.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-2/meta_remove_corrupted_loadjson_noempty_clean_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-2/meta_remove_corrupted_loadjson_noempty_clean.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-2/meta_remove_corrupted_loadjson_noempty_clean_info.csv --format csv --video-info --fmin 1 && echo \"All Done!\"\n", + "zhaowan+ 1288071 0.0 0.0 11168 524 ? S 01:54 0:00 sleep 3h\n", + "zhaowan+ 1354819 0.0 0.0 14180 5232 ? S 01:54 0:00 bash -ic sleep 6h; echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-4/meta_loadjson_noempty_clean_info.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-4/meta_loadjson_noempty_clean_info_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-4/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-4/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1 && echo \"All Done!\"\n", + "zhaowan+ 1359377 0.0 0.0 11168 592 ? S 01:54 0:00 sleep 6h\n", + "zhaowan+ 1522643 0.0 0.0 14180 5348 ? S 01:55 0:00 bash -ic sleep 9h; echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-6/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1 && echo \"All Done!\"\n", + "zhaowan+ 1526854 0.0 0.0 11168 592 ? S 01:55 0:00 sleep 9h\n", + "zhaowan+ 2067689 0.0 0.0 14180 5184 ? S 01:40 0:00 bash -ic sleep 1m; echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1 && echo \"All Done!\"\n", + "zhaowan+ 2068244 4.2 0.0 18185172 812836 ? Sl 01:41 0:43 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068405 6.1 0.0 25108784 763768 ? Sl 01:41 1:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068410 66.5 0.1 40081464 4002872 ? Rl 01:41 11:02 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068412 84.9 0.1 53370912 2699136 ? Rl 01:41 14:06 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068414 71.3 0.1 45830048 3300632 ? Rl 01:41 11:50 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068416 64.5 0.1 36159696 2181696 ? Rl 01:41 10:42 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068418 68.6 0.1 40646848 4147996 ? Rl 01:41 11:23 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068420 70.2 0.1 41765964 3046216 ? Il 01:41 11:39 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068422 88.3 0.2 64657824 4617116 ? Rl 01:41 14:40 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068424 71.8 0.1 44973076 3701892 ? Rl 01:41 11:55 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068426 72.4 0.1 52072040 2640628 ? Il 01:41 12:01 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068428 70.7 0.2 48912276 4344696 ? Il 01:41 11:44 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068430 79.8 0.1 47171896 3231428 ? Rl 01:41 13:15 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068432 70.1 0.1 48930576 4056728 ? Il 01:41 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068434 65.3 0.1 43735540 3238716 ? Rl 01:41 10:51 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068436 79.4 0.1 63527316 3907236 ? Rl 01:41 13:11 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068438 75.3 0.1 68238108 3895888 ? Il 01:41 12:30 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068440 62.8 0.4 29853176 9489236 ? Rl 01:41 10:26 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068442 65.7 0.1 39010496 3872812 ? Il 01:41 10:54 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068444 68.6 0.1 47677716 4146684 ? Il 01:41 11:24 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068446 69.9 0.1 46871244 3388288 ? Rl 01:41 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068448 68.4 0.1 55896468 3579808 ? Rl 01:41 11:21 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068450 74.9 0.1 51226320 4166040 ? Rl 01:41 12:26 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068452 64.0 0.1 23133248 2679492 ? Rl 01:41 10:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068454 71.6 0.1 48530008 3974596 ? Rl 01:41 11:53 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068456 65.4 0.1 41144232 2995604 ? Rl 01:41 10:52 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068458 64.3 0.1 38720868 3199228 ? Il 01:41 10:40 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068460 70.5 0.1 48263296 3592304 ? Rl 01:41 11:43 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068462 80.2 0.1 51951232 2916968 ? Rl 01:41 13:18 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068464 69.5 0.1 42342668 3400720 ? Sl 01:41 11:31 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068466 70.1 0.1 42982136 2988904 ? Rl 01:41 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068468 65.7 0.1 36393260 3596696 ? Rl 01:41 10:54 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068470 99.4 0.1 58307936 3987004 ? Il 01:41 16:30 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068472 67.0 0.1 43176860 2485856 ? Il 01:41 11:06 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068474 62.5 0.1 22925972 2840604 ? Rl 01:41 10:22 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068476 77.4 0.1 60499416 3971216 ? Rl 01:41 12:51 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068478 70.1 0.1 50691148 4164016 ? Rl 01:41 11:37 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068480 82.1 0.1 49989432 3522520 ? Il 01:41 13:37 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068482 67.5 0.1 43243484 3261156 ? Rl 01:41 11:12 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068484 72.4 0.1 54891888 4193444 ? Rl 01:41 12:00 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068486 63.4 0.2 63944812 4241336 ? Rl 01:41 10:31 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068488 66.1 0.1 47988808 2641972 ? Sl 01:41 10:58 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068490 69.9 0.2 55764732 5008460 ? Rl 01:41 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068492 71.3 0.1 58444280 3229160 ? Il 01:41 11:49 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068494 63.9 0.1 39768992 3428572 ? Il 01:41 10:35 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068496 80.7 0.1 59900088 3320368 ? Il 01:41 13:23 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068498 64.3 0.1 48576120 3376636 ? Rl 01:41 10:40 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068500 77.2 0.1 61380180 4054664 ? Rl 01:41 12:48 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068502 81.7 0.1 56815500 2594564 ? Rl 01:41 13:32 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068504 74.4 0.1 53311532 4140480 ? Sl 01:41 12:21 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068506 64.7 0.1 39537568 3379112 ? Rl 01:41 10:44 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068508 69.1 0.3 56005860 7179580 ? Il 01:41 11:27 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068510 68.5 0.1 47330876 3003556 ? Rl 01:41 11:22 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068512 68.4 0.3 49416332 7630192 ? Rl 01:41 11:21 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068514 70.3 0.1 55333380 3664032 ? Il 01:41 11:40 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068516 74.3 0.1 58503880 3761064 ? Rl 01:41 12:19 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068518 84.8 0.1 59470480 3827656 ? Rl 01:41 14:04 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068520 75.0 0.1 50593956 3391556 ? Rl 01:41 12:27 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068522 70.0 0.1 55683976 3501120 ? Rl 01:41 11:37 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068524 61.4 0.1 34296076 3344680 ? Il 01:41 10:11 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068526 63.7 0.1 43449312 3727660 ? Rl 01:41 10:34 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068528 73.5 0.1 47581304 2965100 ? Rl 01:41 12:11 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068530 77.1 0.1 53240452 2761268 ? Rl 01:41 12:48 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068532 65.1 0.1 46041240 3870092 ? Rl 01:41 10:47 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068534 69.6 0.1 52350100 3527760 ? Rl 01:41 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068536 71.8 0.2 50059812 4821092 ? Rl 01:41 11:54 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068538 61.1 0.1 23224144 2893448 ? Il 01:41 10:08 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068540 70.0 0.1 54990404 4030480 ? Rl 01:41 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068542 70.3 0.1 46581008 3377600 ? Rl 01:41 11:39 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068544 63.2 0.1 38957708 2809760 ? Il 01:41 10:29 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068546 68.3 0.1 48621748 4196412 ? Rl 01:41 11:19 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068548 74.4 0.1 48495760 3415204 ? Rl 01:41 12:21 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068550 73.5 0.1 47785884 3329964 ? Il 01:41 12:11 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068552 66.4 0.3 28119364 8001748 ? Rl 01:41 11:00 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068554 75.0 0.1 61825588 3855128 ? Il 01:41 12:26 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068556 78.7 0.2 51645644 4584920 ? Il 01:41 13:03 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068558 70.2 0.1 42158272 3365372 ? Rl 01:41 11:39 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068560 65.8 0.1 38237648 3444160 ? Rl 01:41 10:55 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068562 65.9 0.1 48142784 3449580 ? Rl 01:41 10:55 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068564 61.4 0.1 25398776 3072784 ? Il 01:41 10:11 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068566 68.0 0.1 23195296 2864428 ? Rl 01:41 11:17 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068568 70.2 0.1 45293812 3280116 ? Il 01:41 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068570 65.8 0.1 36230504 3279796 ? Rl 01:41 10:55 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068572 66.6 0.1 50080564 4036152 ? Rl 01:41 11:03 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068574 65.5 0.1 46070280 3177868 ? Rl 01:41 10:52 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068576 73.1 0.1 48377492 3479636 ? Rl 01:41 12:07 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068578 75.5 0.1 54496052 3668616 ? Il 01:41 12:31 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068580 68.8 0.1 45834848 3880500 ? Il 01:41 11:24 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068582 62.0 0.1 43981556 2293880 ? Il 01:41 10:17 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068584 110 0.1 85092848 4024708 ? Rl 01:41 18:17 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068586 66.8 0.1 34725264 2213928 ? Il 01:41 11:05 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068588 73.3 0.2 65600980 4258028 ? Rl 01:41 12:10 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068590 66.1 0.1 22870296 2785268 ? Il 01:41 10:58 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068592 70.9 0.1 42937228 3511772 ? Rl 01:41 11:45 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068594 88.7 0.1 60570636 4194400 ? Il 01:41 14:42 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068596 68.7 0.1 51889088 3185148 ? Il 01:41 11:24 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068598 64.8 0.1 34944796 2290740 ? Il 01:41 10:45 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2068600 63.9 0.4 46367296 9048748 ? Rl 01:41 10:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-1/meta_loadjson_noempty_clean_info_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2106235 0.0 0.0 11168 524 ? S 01:57 0:00 sleep 3h\n", + "zhaowan+ 2321296 11.0 0.0 14176 5432 ? Ss 01:58 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", + "zhaowan+ 2326357 0.0 0.0 12124 2668 ? S 01:58 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", + "\n", + "==== STDERR ====\n", + " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", + "bash: no job control in this shell\n", + "\n" + ] + } + ], + "source": [ + "kill(1922317, host)\n", + "ps(host)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using following commands to monitor the status of the jobs." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "HOST: h800-80\n", + "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", + "==== STDOUT ====\n", + " zhaowan+ 599938 0.0 0.0 5492 592 ? S 01:44 0:00 sleep 3h\n", + "zhaowan+ 820507 9.0 0.0 8488 5256 ? Ss 01:45 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", + "zhaowan+ 823896 0.0 0.0 6412 724 ? S 01:45 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", + "zhaowan+ 2900571 0.0 0.0 8496 5244 ? S 01:31 0:00 bash -ic echo \"Converting /mnt/disk0/data-panda/17 to /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta.csv\" && mkdir -p /mnt/hdd/data/panda70m_by/raw/meta/split-17 && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.convert video /mnt/disk0/data-panda/17 --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta.csv && echo \"Getting caption of /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --format csv --load-caption json --remove-empty-caption --clean-caption && echo \"Getting info of /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv to /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv\" && cd /home/zhaowangbo/zangwei/opensora/ && python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1 && echo \"All Done!\"\n", + "zhaowan+ 2912258 5.1 0.0 17862408 464236 ? Sl 01:33 0:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912747 7.5 0.0 24784996 447148 ? Sl 01:33 0:52 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912752 99.6 0.3 27611876 7062424 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912754 99.6 0.3 28230888 6993972 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912756 99.6 0.4 29655612 9474388 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912758 99.6 0.2 25897644 5593840 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912760 99.6 0.3 27507452 7204044 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912762 99.6 0.2 25852868 5426576 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912764 99.6 0.2 26248672 5576544 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912766 99.6 0.5 31998688 11054616 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912768 99.4 0.1 23769996 3466092 ? Rl 01:33 11:37 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912770 99.6 0.2 25804460 5623292 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912772 99.6 0.2 24610896 4307488 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912774 99.6 0.3 27691860 7510764 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912776 104 0.4 115059480 9288680 ? Rl 01:33 12:12 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912778 99.5 0.2 25696184 5146896 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912780 99.7 0.3 27301848 7120984 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912782 99.7 0.3 27260008 6923964 ? Rl 01:33 11:39 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912784 99.6 0.4 29350952 8924840 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912786 99.4 0.2 25553512 5373264 ? Rl 01:33 11:37 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912788 99.1 0.1 23088424 2661800 ? Rl 01:33 11:34 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912790 99.2 0.1 23146116 2965364 ? Rl 01:33 11:35 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912792 100 0.1 44587200 3248852 ? Rl 01:33 11:42 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912794 99.2 0.1 22993716 2689836 ? Rl 01:33 11:35 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912796 102 0.1 57999132 3382364 ? Rl 01:33 11:58 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912798 98.9 0.2 24554688 4304240 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912800 99.1 0.1 23664540 3115724 ? Rl 01:33 11:34 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912802 99.3 0.1 22983312 2803072 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912804 99.1 0.1 23469756 3043864 ? Rl 01:33 11:35 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912806 99.3 0.1 24009276 3308228 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912808 98.9 0.1 23317336 2890520 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912810 98.8 0.1 22871156 2690908 ? Rl 01:33 11:32 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912812 98.9 0.3 32512896 8406732 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912814 100 0.1 35345836 2421884 ? Rl 01:33 11:41 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912816 99.1 0.1 23286868 2738060 ? Rl 01:33 11:35 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912818 99.3 0.1 23903600 3477100 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912820 99.2 0.2 25462264 4667668 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912822 99.3 0.1 23917772 3737324 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912824 99.5 0.4 29858512 9555404 ? Rl 01:33 11:38 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912826 98.7 0.6 34394920 13197548 ? Rl 01:33 11:32 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912828 99.4 0.2 24821648 4272672 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912830 98.8 0.4 30811228 9875464 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912832 99.4 0.4 28741980 8528348 ? Rl 01:33 11:37 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912834 99.1 0.1 23763348 3214348 ? Rl 01:33 11:35 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912836 99.2 0.1 23704252 3277540 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912838 99.3 0.2 25013788 4588004 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912840 99.3 0.1 23692084 3266004 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912842 99.3 0.2 26366716 5652656 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912844 99.2 0.2 24760448 4456608 ? Rl 01:33 11:35 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912846 99.3 0.1 22863156 2649632 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912848 99.3 0.1 22895452 2591584 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912850 99.3 0.1 23153900 2605116 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912852 100 0.1 51176404 2929556 ? Rl 01:33 11:42 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912854 99.3 0.1 23310532 2760808 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912856 98.8 0.1 23056252 2506784 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912858 98.9 0.1 22985460 2682468 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912860 98.9 0.2 26316040 4768308 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912862 99.0 0.1 23179856 2718260 ? Rl 01:33 11:34 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912864 98.9 0.1 25261980 4164944 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912866 99.2 0.1 22534884 2354308 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912868 101 0.1 53333052 2747044 ? Rl 01:33 11:50 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912870 99.0 0.1 24556684 3682248 ? Rl 01:33 11:34 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912872 98.8 0.1 23405108 2962540 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912875 98.9 0.1 23305336 2752636 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912877 109 0.2 114214656 4640620 ? Rl 01:33 12:46 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912879 99.2 0.1 23878204 3329680 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912881 99.3 0.1 23966672 3294848 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912883 110 0.1 56521384 3958060 ? Rl 01:33 12:56 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912885 99.3 0.1 23013044 2709476 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912887 99.3 0.1 23359284 2810484 ? Rl 01:33 11:36 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912889 98.9 0.1 23708756 3405804 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912892 104 0.2 108417396 5409156 ? Rl 01:33 12:14 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912894 100 0.1 61041612 3965724 ? Rl 01:33 11:47 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912896 105 0.1 92753352 2605000 ? Rl 01:33 12:22 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912898 99.0 0.1 23132692 2706820 ? Rl 01:33 11:34 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912900 98.9 0.1 23596196 3034956 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912902 98.6 0.1 24354040 3164508 ? Rl 01:33 11:31 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912904 98.9 0.1 24636092 3941240 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912906 99.0 0.1 23538184 3112324 ? Rl 01:33 11:34 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912908 99.0 0.1 23851536 3302764 ? Rl 01:33 11:34 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912910 99.1 0.1 23453084 2792364 ? Rl 01:33 11:34 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912913 98.9 0.1 24442200 3028260 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912915 98.9 0.1 23943236 3271560 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912917 105 0.2 58809760 4691488 ? Rl 01:33 12:18 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912919 98.9 0.1 25037364 3758396 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912921 98.9 0.1 23558620 2887040 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912923 98.8 0.1 24671844 3433608 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912925 98.9 0.1 24297188 3273052 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912927 98.9 0.1 24152060 3133860 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912929 99.0 0.1 24114140 3688084 ? Rl 01:33 11:34 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912931 98.9 0.1 23496424 2996496 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912933 98.8 0.1 23595804 3136552 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912935 102 0.1 69551012 3396020 ? Rl 01:33 11:57 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912937 98.8 0.1 24049412 3376940 ? Rl 01:33 11:32 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912939 99.2 0.1 40622032 3484860 ? Rl 01:33 11:35 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912941 98.8 0.1 23434160 3098160 ? Rl 01:33 11:33 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912943 98.5 0.1 23565228 3016180 ? Rl 01:33 11:30 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "zhaowan+ 2912945 104 0.1 84339424 2933876 ? Sl 01:33 12:14 python -m tools.datasets.datautil /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption.csv --output /mnt/hdd/data/panda70m_by/raw/meta/split-17/meta_caption_info.csv --format csv --video-info --fmin 1\n", + "\n", + "==== STDERR ====\n", + " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", + "bash: no job control in this shell\n", + "\n" + ] + } + ], + "source": [ + "ps(host)" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "HOST: h800-80\n", + "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", + "==== STDOUT ====\n", + " zhaowan+ 2508488 0.9 0.0 62146500 335516 pts/62 Sl 00:23 0:23 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/colossalai run --nproc_per_node 8 --hostfile hostfile scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2509916 0.4 0.0 62277572 223088 pts/62 Sl 00:23 0:11 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/colossalai run --nproc_per_node 8 --hostfile hostfile scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2509917 0.4 0.0 62343108 225364 pts/62 Sl 00:23 0:10 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/colossalai run --nproc_per_node 8 --hostfile hostfile scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2509918 0.4 0.0 62343108 225360 pts/62 Sl 00:23 0:11 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/colossalai run --nproc_per_node 8 --hostfile hostfile scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2509919 0.4 0.0 62343108 225360 pts/62 Sl 00:23 0:10 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/colossalai run --nproc_per_node 8 --hostfile hostfile scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2509920 0.4 0.0 62343108 225348 pts/62 Sl 00:23 0:10 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/colossalai run --nproc_per_node 8 --hostfile hostfile scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2509921 0.4 0.0 62343108 225376 pts/62 Sl 00:23 0:10 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/colossalai run --nproc_per_node 8 --hostfile hostfile scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2509922 0.4 0.0 62343108 225376 pts/62 Sl 00:23 0:10 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/colossalai run --nproc_per_node 8 --hostfile hostfile scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2509923 0.4 0.0 62343108 225376 pts/62 Sl 00:23 0:10 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/colossalai run --nproc_per_node 8 --hostfile hostfile scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2509924 0.7 0.0 20806104 300932 pts/62 Sl 00:23 0:18 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/torchrun --nproc_per_node=8 --nnodes=8 --node_rank=0 --master_addr=10.20.1.80 --master_port=29500 scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2511006 134 0.5 124903600 11474180 ? Ssl 00:23 52:03 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2511007 134 0.6 124280820 13743976 ? Ssl 00:23 51:54 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2511008 135 0.6 105990928 12947264 ? Ssl 00:23 52:10 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2511009 135 0.6 116960388 13623620 ? Ssl 00:23 52:15 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2511010 92.3 0.6 118110784 13164180 ? Ssl 00:23 35:37 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2511011 92.5 0.5 95954420 11648152 ? Ssl 00:23 35:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2511012 92.2 0.5 113079028 12292900 ? Ssl 00:23 35:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2511013 92.3 0.6 119600720 13289524 ? Ssl 00:23 35:37 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2583466 2.8 0.3 71580272 7611528 ? Sl 00:37 0:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2583467 2.4 0.3 70955396 6808460 ? Sl 00:37 0:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2583483 3.4 0.3 71195056 6784112 ? Sl 00:37 0:51 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2583655 2.5 0.3 70840048 6749648 ? Sl 00:37 0:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2583656 2.5 0.3 70751460 6537264 ? Sl 00:37 0:39 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2583657 3.3 0.3 70811196 6767144 ? Sl 00:37 0:51 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2583844 3.5 0.3 71154156 6743652 ? Sl 00:37 0:53 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2583907 2.5 0.3 70736984 6537328 ? Sl 00:37 0:39 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2583970 3.4 0.3 71785552 7940056 ? Sl 00:37 0:52 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2583971 2.5 0.3 70818996 6366032 ? Sl 00:37 0:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2583972 3.2 0.2 70716648 6305084 ? Sl 00:37 0:49 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584159 2.7 0.3 71302656 6966060 ? Sl 00:37 0:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584160 2.0 0.3 71297060 6714336 ? Sl 00:37 0:30 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584272 3.3 0.3 71631184 6851248 ? Sl 00:37 0:51 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584350 3.3 0.3 70691172 6403012 ? Sl 00:37 0:50 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584413 2.3 0.2 70490588 6031100 ? Sl 00:37 0:35 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584445 2.5 0.3 71286984 6615472 ? Sl 00:37 0:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584477 2.2 0.2 69964336 5922200 ? Sl 00:37 0:33 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584478 3.4 0.3 71140640 6604820 ? Sl 00:37 0:52 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584665 3.2 0.3 71248152 7156392 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584666 2.6 0.3 71186464 6882132 ? Sl 00:37 0:40 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584791 3.1 0.3 71731476 7443404 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584834 2.2 0.2 70108896 5771660 ? Sl 00:37 0:34 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584982 2.9 0.3 71192304 7026364 ? Sl 00:37 0:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2584984 2.5 0.3 71353300 6830184 ? Sl 00:37 0:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2585210 3.1 0.3 71638320 7473928 ? Sl 00:37 0:47 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2585297 2.4 0.3 71085044 6903500 ? Sl 00:37 0:37 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2585389 2.4 0.3 70561784 6471144 ? Sl 00:37 0:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2585549 3.2 0.2 71061044 6277992 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2585613 3.2 0.3 70783148 6740760 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2585739 2.1 0.3 70666896 6378912 ? Sl 00:37 0:33 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2585865 2.2 0.3 71173748 6850892 ? Sl 00:37 0:33 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2737433 10.0 0.0 8488 5208 ? Ss 01:02 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", + "zhaowan+ 2737627 0.0 0.0 6412 728 ? S 01:02 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", + "\n", + "==== STDERR ====\n", + " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", + "bash: no job control in this shell\n", + "\n", + "HOST: h800-81\n", + "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", + "==== STDOUT ====\n", + " zhaowan+ 2329936 0.0 0.0 129732 45044 ? Sl 00:22 0:00 /usr/bin/python3 /home/zhaowangbo/.vscode-server/extensions/ms-python.black-formatter-2024.2.0/bundled/tool/lsp_server.py --stdio\n", + "zhaowan+ 2331293 0.8 0.0 20732364 192948 ? Ssl 00:23 0:19 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/torchrun --nproc_per_node=8 --nnodes=8 --node_rank=1 --master_addr=10.20.1.80 --master_port=29500 scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2331543 133 0.5 114118592 12511284 ? Ssl 00:23 51:50 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2331544 130 0.5 107818812 11814660 ? Ssl 00:23 50:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2331545 106 0.5 120764588 11656160 ? Ssl 00:23 41:13 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2331546 132 0.5 117471708 12554556 ? Ssl 00:23 51:18 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2331547 59.8 0.5 122785608 11958104 ? Ssl 00:23 23:09 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2331548 91.9 0.5 119456240 11732996 ? Ssl 00:23 35:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2331549 92.0 0.5 125015368 12445804 ? Ssl 00:23 35:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2331550 91.4 0.5 115179344 11822896 ? Ssl 00:23 35:24 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2340403 3.7 0.2 70014780 6009576 ? Sl 00:37 0:56 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2340404 5.4 0.2 70191928 6318252 ? Sl 00:37 1:23 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2340405 33.9 0.3 70937568 6651000 ? Sl 00:37 8:37 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2340406 29.0 0.3 70970928 6596332 ? Sl 00:37 7:22 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2340407 5.8 0.3 71005804 6723608 ? Sl 00:37 1:28 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2340408 37.1 0.3 70813276 6633304 ? Sl 00:37 9:26 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2340409 61.0 0.3 72827912 8238828 ? Rl 00:37 15:30 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2340410 3.1 0.3 70669776 6673380 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2340907 5.0 0.3 71219860 7214348 ? Sl 00:37 1:16 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2340908 5.9 0.3 70829120 6709804 ? Sl 00:37 1:30 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2340916 32.7 0.3 70821092 6442868 ? Sl 00:37 8:18 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2340964 41.3 0.3 70668284 6786052 ? Sl 00:37 10:30 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2341158 4.8 0.3 70886032 7004272 ? Sl 00:37 1:14 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2341160 29.2 0.3 70626436 6447132 ? Sl 00:37 7:25 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2341193 50.5 0.3 71921764 7265544 ? Rl 00:37 12:50 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2341348 26.0 0.3 71430944 6820720 ? Sl 00:37 6:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2341411 4.7 0.3 72209824 7712656 ? Sl 00:37 1:13 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2341444 4.7 0.3 71005156 6640220 ? Sl 00:37 1:12 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2341537 4.5 0.3 70656064 6650656 ? Sl 00:37 1:09 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2341538 59.1 0.3 71229044 6769008 ? Rl 00:37 15:01 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2341663 49.3 0.3 72228336 7803308 ? Sl 00:37 12:31 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2341670 57.4 0.3 70991832 6922300 ? Rl 00:37 14:34 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2341680 33.5 0.3 71163404 6879940 ? Sl 00:37 8:31 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2341706 3.9 0.3 70974740 6947024 ? Sl 00:37 1:00 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2341915 3.6 0.3 70616996 6456680 ? Sl 00:37 0:55 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2342041 2.5 0.3 71122920 6880208 ? Sl 00:37 0:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2342167 58.5 0.3 71648488 7716400 ? Rl 00:37 14:52 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2342231 4.6 0.3 70703504 6452484 ? Sl 00:37 1:11 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2342419 49.6 0.3 72371696 7586856 ? Rl 00:37 12:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2342542 58.1 0.3 71336672 7269116 ? Rl 00:37 14:46 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2342545 17.9 0.3 70285624 6413024 ? Sl 00:37 4:34 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2342784 2.7 0.3 70871828 6499244 ? Sl 00:37 0:41 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2367028 5.0 0.0 8492 5224 ? Ss 01:02 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", + "zhaowan+ 2367125 0.0 0.0 6412 724 ? S 01:02 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", + "zhaowan+ 3400520 0.3 0.0 1262616 464988 ? Sl Apr13 0:43 /home/zhaowangbo/.vscode-server/bin/1e790d77f81672c49be070e04474901747115651/node /home/zhaowangbo/.vscode-server/extensions/ms-python.vscode-pylance-2024.4.1/dist/server.bundle.js --cancellationReceive=file:98eaa38ca121a27bf469d3588850255a4a0b056b9b --node-ipc --clientProcessId=3420919\n", + "\n", + "==== STDERR ====\n", + " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", + "bash: no job control in this shell\n", + "\n", + "HOST: h800-82\n", + "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", + "==== STDOUT ====\n", + " zhaowan+ 2261487 0.8 0.0 20732364 301004 ? Ssl 00:23 0:19 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/torchrun --nproc_per_node=8 --nnodes=8 --node_rank=2 --master_addr=10.20.1.80 --master_port=29500 scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2261627 135 0.5 125636668 12298636 ? Ssl 00:23 52:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2261628 135 0.5 97522804 12558752 ? Ssl 00:23 52:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2261629 135 0.5 117170160 12182628 ? Ssl 00:23 52:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2261630 135 0.6 118773036 13098140 ? Ssl 00:23 52:37 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2261631 92.7 0.5 97113840 12158824 ? Ssl 00:23 36:01 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2261632 92.2 0.5 124314604 11472328 ? Ssl 00:23 35:49 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2261633 92.2 0.6 119122160 14474524 ? Ssl 00:23 35:50 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2261634 92.3 0.6 125297012 12860604 ? Ssl 00:23 35:50 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2263594 3.2 0.2 70341448 6338112 ? Sl 00:37 0:49 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2263652 2.8 0.3 71738852 7699056 ? Sl 00:37 0:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2263658 3.6 0.3 70807188 6814280 ? Sl 00:37 0:55 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2263659 2.3 0.3 70921548 6762796 ? Sl 00:37 0:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2263673 2.6 0.2 70460096 6310452 ? Sl 00:37 0:40 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2263719 3.2 0.3 71025924 7155592 ? Sl 00:37 0:49 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2263766 2.8 0.3 71600080 7438296 ? Sl 00:37 0:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264035 2.4 0.4 73416340 9167760 ? Sl 00:37 0:37 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264098 2.9 0.3 70518180 6638252 ? Sl 00:37 0:45 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264161 3.6 0.3 70744320 6628348 ? Sl 00:37 0:56 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264162 2.8 0.3 71348680 7231156 ? Sl 00:37 0:43 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264163 2.9 0.3 70657236 6532512 ? Sl 00:37 0:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264224 2.3 0.3 71322920 6715528 ? Sl 00:37 0:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264284 2.5 0.3 71762844 7813708 ? Sl 00:37 0:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264440 2.7 0.3 71089240 6715628 ? Sl 00:37 0:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264539 3.3 0.3 72287524 8285152 ? Sl 00:37 0:51 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264602 2.0 0.3 70945648 6584048 ? Sl 00:37 0:31 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264603 3.3 0.3 70744124 6671600 ? Sl 00:37 0:50 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264604 3.8 0.4 73111740 8863548 ? Sl 00:37 0:58 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264748 2.0 0.3 70882732 6642780 ? Sl 00:37 0:31 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264792 1.8 0.3 72141276 7738404 ? Sl 00:37 0:28 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264793 3.4 0.3 70957980 6719660 ? Sl 00:37 0:52 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2264980 3.3 0.3 70878052 6750228 ? Sl 00:37 0:50 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2265043 2.4 0.3 70911784 6662692 ? Sl 00:37 0:37 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2265169 2.7 0.3 71138548 6899732 ? Sl 00:37 0:41 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2265281 2.5 0.3 70589400 6426796 ? Sl 00:37 0:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2265421 3.1 0.3 71715340 7688820 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2265470 1.7 0.3 70834440 6954684 ? Sl 00:37 0:26 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2265474 2.6 0.3 70962692 6723956 ? Sl 00:37 0:40 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2265789 3.7 0.4 73269828 8899476 ? Sl 00:37 0:56 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2265862 2.7 0.3 70741748 6491056 ? Sl 00:37 0:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2265988 2.2 0.4 73111716 8862956 ? Sl 00:37 0:34 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2282497 10.0 0.0 8488 5228 ? Ss 01:02 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", + "zhaowan+ 2282590 0.0 0.0 6412 724 ? S 01:02 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", + "\n", + "==== STDERR ====\n", + " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", + "bash: no job control in this shell\n", + "\n", + "HOST: h800-83\n", + "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", + "==== STDOUT ====\n", + " zhaowan+ 3883209 0.8 0.0 20738040 300960 ? Ssl 00:23 0:18 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/torchrun --nproc_per_node=8 --nnodes=8 --node_rank=3 --master_addr=10.20.1.80 --master_port=29500 scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3883533 136 0.5 97322024 12419216 ? Ssl 00:23 53:00 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3883534 135 0.5 103578376 11581412 ? Ssl 00:23 52:51 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3883535 134 0.6 129268272 12838244 ? Ssl 00:23 52:30 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3883536 135 0.6 105399344 14686648 ? Ssl 00:23 52:43 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3883537 92.4 0.6 130387668 12921244 ? Ssl 00:23 36:02 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3883538 91.7 0.5 122906484 12533612 ? Ssl 00:23 35:45 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3883539 92.2 0.6 120092008 12806228 ? Ssl 00:23 35:57 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3883540 92.5 0.6 117633992 13342420 ? Ssl 00:23 36:03 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3901690 3.1 0.3 72669404 8144692 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3901691 2.1 0.3 71105612 6726912 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3901713 2.0 0.3 71090704 6713500 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3901779 2.9 0.3 71396432 7060160 ? Sl 00:37 0:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3901780 3.4 0.3 70843524 6553332 ? Sl 00:37 0:53 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3901792 3.1 0.3 71574896 7664740 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3901910 2.5 0.3 71466140 7179304 ? Sl 00:37 0:39 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902131 2.7 0.4 72812272 8812264 ? Sl 00:37 0:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902194 2.3 0.3 71347604 7101880 ? Sl 00:37 0:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902257 2.0 0.3 70620480 6609740 ? Sl 00:37 0:31 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902320 2.0 0.3 70541756 6496756 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902321 2.7 0.4 72649700 8494292 ? Sl 00:37 0:41 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902322 2.6 0.3 71322128 7035100 ? Sl 00:37 0:41 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902359 2.2 0.3 71019228 6930000 ? Sl 00:37 0:34 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902507 3.4 0.3 71077852 6822892 ? Sl 00:37 0:53 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902635 3.7 0.3 70678980 6679600 ? Sl 00:37 0:57 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902698 2.4 0.4 73113704 9114536 ? Sl 00:37 0:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902762 2.4 0.3 71115232 6983772 ? Sl 00:37 0:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902763 2.9 0.3 70375600 6487800 ? Sl 00:37 0:45 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902766 3.5 0.3 70953408 6661432 ? Sl 00:37 0:54 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902900 3.2 0.3 70697012 6909600 ? Sl 00:37 0:49 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902962 3.0 0.3 71092520 6757196 ? Sl 00:37 0:46 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3902973 2.5 0.3 70571668 6563076 ? Sl 00:37 0:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3903088 2.8 0.2 70228696 6073656 ? Sl 00:37 0:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3903269 3.4 0.4 73075816 9076828 ? Sl 00:37 0:52 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3903270 3.1 0.3 70836216 6705740 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3903272 2.4 0.3 70553204 6629732 ? Sl 00:37 0:37 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3903501 3.1 0.3 71464840 7330812 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3903509 2.6 0.3 70816120 6782524 ? Sl 00:37 0:41 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3903761 1.6 0.3 71174536 6961708 ? Sl 00:37 0:26 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3903775 1.7 0.3 70962784 6461724 ? Sl 00:37 0:26 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3904091 2.9 0.4 72738176 8828524 ? Sl 00:37 0:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 3950799 5.0 0.0 14176 5296 ? Ss 01:02 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", + "zhaowan+ 3950958 0.0 0.0 12124 728 ? S 01:02 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", + "\n", + "==== STDERR ====\n", + " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", + "bash: no job control in this shell\n", + "\n", + "HOST: h800-84\n", + "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", + "==== STDOUT ====\n", + " zhaowan+ 2042167 0.8 0.0 20738048 300952 ? Ssl 00:23 0:19 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/torchrun --nproc_per_node=8 --nnodes=8 --node_rank=4 --master_addr=10.20.1.80 --master_port=29500 scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2042309 136 0.6 119907372 13867396 ? Ssl 00:23 53:22 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2042310 134 0.6 116424732 12839468 ? Ssl 00:23 52:39 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2042311 136 0.5 117042012 12298480 ? Ssl 00:23 53:14 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2042312 136 0.6 102916536 12799780 ? Ssl 00:23 53:19 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2042313 92.8 0.6 113140144 13205952 ? Ssl 00:23 36:17 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2042314 92.5 0.6 122741964 13727800 ? Ssl 00:23 36:12 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2042315 92.6 0.5 116893500 12217156 ? Ssl 00:23 36:13 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2042316 92.6 0.5 116934464 12281904 ? Ssl 00:23 36:14 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044205 2.2 0.4 72562584 8653704 ? Sl 00:37 0:35 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044206 2.7 0.3 71185412 6776120 ? Sl 00:37 0:41 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044217 2.3 0.3 71864968 7574592 ? Sl 00:37 0:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044394 2.0 0.3 71166516 6798452 ? Sl 00:37 0:31 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044395 3.2 0.3 70970900 6594732 ? Sl 00:37 0:49 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044396 2.8 0.4 74106200 9738676 ? Sl 00:37 0:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044583 3.2 0.3 72335048 8179880 ? Sl 00:37 0:51 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044646 2.6 0.3 70550976 6604116 ? Sl 00:37 0:41 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044709 2.7 0.3 70788792 6913120 ? Sl 00:37 0:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044710 2.9 0.3 71591332 7581588 ? Sl 00:37 0:45 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044835 1.8 0.3 71719548 7518464 ? Sl 00:37 0:28 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044836 2.7 0.3 72443920 7953036 ? Sl 00:37 0:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044849 3.1 0.3 71516756 7640188 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2044900 2.8 0.3 71094612 6716348 ? Sl 00:37 0:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2045087 2.8 0.3 70480092 6481620 ? Sl 00:37 0:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2045088 1.2 0.3 72291748 8134568 ? Sl 00:37 0:19 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2045089 3.3 0.3 71712636 7212664 ? Sl 00:37 0:51 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2045231 2.0 0.3 70713008 6521708 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2045339 2.1 0.3 71427468 7259316 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2045402 3.7 0.3 70961684 6839220 ? Sl 00:37 0:57 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2045403 3.2 0.3 70793392 6793808 ? Sl 00:37 0:50 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2045490 2.7 0.3 70850584 6605424 ? Sl 00:37 0:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2045555 3.0 0.3 71408924 7523044 ? Sl 00:37 0:46 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2045586 2.2 0.3 70756944 6501652 ? Sl 00:37 0:35 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2045843 2.5 0.3 72483832 8295392 ? Sl 00:37 0:39 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2045901 2.7 0.3 71489524 7444780 ? Sl 00:37 0:41 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2046032 2.7 0.3 71068828 6754752 ? Sl 00:37 0:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2046095 2.6 0.4 72952376 8705452 ? Sl 00:37 0:40 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2046131 1.2 0.3 70900696 6900048 ? Sl 00:37 0:18 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2046347 2.0 0.3 71612600 7358824 ? Sl 00:37 0:31 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2046474 1.1 0.3 71835572 7892288 ? Sl 00:37 0:17 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2046555 2.5 0.3 70480672 6535452 ? Sl 00:37 0:39 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 2060955 5.0 0.0 14176 5348 ? Ss 01:03 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", + "zhaowan+ 2061050 0.0 0.0 12124 724 ? S 01:03 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", + "\n", + "==== STDERR ====\n", + " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", + "bash: no job control in this shell\n", + "\n", + "HOST: h800-85\n", + "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", + "==== STDOUT ====\n", + " zhaowan+ 2201547 0.0 0.0 8700 3260 pts/2 T Apr13 0:00 bash -ic nohup sleep 3m\n", + "zhaowan+ 2485709 5.5 0.0 8496 5252 ? Ss 01:03 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", + "zhaowan+ 2485805 0.0 0.0 6444 728 ? S 01:03 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", + "\n", + "==== STDERR ====\n", + " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", + "bash: no job control in this shell\n", + "\n", + "HOST: h800-86\n", + "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", + "==== STDOUT ====\n", + " zhaowan+ 1701098 0.7 0.0 20738048 300780 ? Ssl 00:23 0:18 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/torchrun --nproc_per_node=8 --nnodes=8 --node_rank=5 --master_addr=10.20.1.80 --master_port=29500 scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1701250 137 0.5 97169600 12251616 ? Ssl 00:23 53:56 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1701251 136 0.5 119211040 12573788 ? Ssl 00:23 53:39 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1701252 136 0.6 123724256 12693136 ? Ssl 00:23 53:51 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1701253 136 0.5 123278436 12578084 ? Ssl 00:23 53:46 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1701254 93.1 0.6 105758068 13141228 ? Ssl 00:23 36:39 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1701255 92.7 0.5 117909252 11354576 ? Ssl 00:23 36:31 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1701256 92.7 0.5 112548740 12552620 ? Ssl 00:23 36:31 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1701257 92.9 0.5 121790896 12056016 ? Ssl 00:23 36:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1704359 3.0 0.3 71001880 6625632 ? Sl 00:37 0:47 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1704360 2.4 0.3 71205008 6582356 ? Sl 00:37 0:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1704361 2.6 0.3 70589584 6511852 ? Sl 00:37 0:41 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1704486 2.5 0.3 71355272 7099212 ? Sl 00:37 0:39 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1704502 2.9 0.3 70463768 6342168 ? Sl 00:37 0:46 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1704526 3.6 0.3 71122656 6754892 ? Sl 00:37 0:57 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1704540 3.2 0.3 72564476 8196524 ? Sl 00:37 0:50 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1704795 3.0 0.3 70747312 6706560 ? Sl 00:37 0:47 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1704863 1.5 0.2 70303788 6294792 ? Sl 00:37 0:24 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1704864 1.7 0.2 70148852 5772932 ? Sl 00:37 0:27 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1704989 2.0 0.3 71002104 6801560 ? Sl 00:37 0:31 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705027 3.9 0.4 73332296 8808840 ? Sl 00:37 1:02 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705049 3.0 0.3 70879612 6757240 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705069 3.2 0.4 73648488 9269320 ? Sl 00:37 0:51 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705241 2.1 0.3 71074576 6952588 ? Sl 00:37 0:34 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705304 2.0 0.3 71403400 6994132 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705308 2.4 0.3 71449044 7318308 ? Sl 00:37 0:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705323 2.1 0.2 70080976 6039608 ? Sl 00:37 0:33 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705493 3.3 0.4 73378928 8932600 ? Sl 00:37 0:52 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705556 1.9 0.3 70270472 6394804 ? Sl 00:37 0:30 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705557 2.8 0.3 71366656 6998808 ? Sl 00:37 0:43 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705682 2.0 0.3 70760304 6749072 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705745 1.6 0.2 70366452 6080096 ? Sl 00:37 0:25 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705756 2.0 0.3 72137084 7851180 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1705788 2.9 0.3 71073684 6704892 ? Sl 00:37 0:46 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1706060 2.6 0.3 71914792 7628360 ? Sl 00:37 0:41 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1706061 1.3 0.2 70260316 5936928 ? Sl 00:37 0:21 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1706249 2.5 0.3 70983060 6860340 ? Sl 00:37 0:40 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1706375 2.7 0.3 71504624 7103860 ? Sl 00:37 0:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1706501 1.9 0.3 71187492 6807780 ? Sl 00:37 0:29 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1706627 2.9 0.3 71823976 7577928 ? Sl 00:37 0:45 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1706753 2.8 0.3 70605848 6720044 ? Sl 00:37 0:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1724007 10.5 0.0 14600 5876 ? Ss 01:03 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", + "zhaowan+ 1724193 0.0 0.0 12148 724 ? S 01:03 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", + "\n", + "==== STDERR ====\n", + " bash: 无法设定终端进程组(-1): 对设备不适当的 ioctl 操作\n", + "bash: 此 shell 中无任务控制\n", + "\n", + "HOST: h800-170\n", + "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", + "==== STDOUT ====\n", + " zhaowan+ 1388437 0.7 0.0 20738040 257544 ? Ssl 00:23 0:18 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/torchrun --nproc_per_node=8 --nnodes=8 --node_rank=6 --master_addr=10.20.1.80 --master_port=29500 scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1388598 136 0.5 114737464 12232144 ? Ssl 00:23 53:53 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1388599 136 0.5 108267628 11968012 ? Ssl 00:23 53:53 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1388600 136 0.6 117886840 14709356 ? Ssl 00:23 53:58 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1388601 135 0.5 117232300 12517548 ? Ssl 00:23 53:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1388602 92.3 0.5 123662544 11328164 ? Ssl 00:23 36:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1388603 92.5 0.5 107140492 12635096 ? Ssl 00:23 36:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1388604 92.1 0.5 121753644 11977416 ? Ssl 00:23 36:27 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1388605 91.6 0.5 125459324 12507636 ? Ssl 00:23 36:16 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1391732 2.5 0.4 72852840 8760696 ? Sl 00:37 0:40 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1391733 2.4 0.3 72519872 7971964 ? Sl 00:37 0:38 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1391858 2.6 0.3 70689808 6803944 ? Sl 00:37 0:42 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1391859 3.1 0.3 71788448 8098352 ? Sl 00:37 0:50 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1391860 3.6 0.3 71641832 7583552 ? Sl 00:37 0:57 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1391861 3.7 0.3 70440688 6432868 ? Sl 00:37 0:59 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1391883 2.3 0.3 71341788 7271604 ? Sl 00:37 0:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392173 2.0 0.4 73532344 9229140 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392236 2.1 0.3 70703376 6822492 ? Sl 00:37 0:34 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392237 3.3 0.3 72561776 8382148 ? Sl 00:37 0:52 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392362 2.2 0.3 71277532 6899896 ? Sl 00:37 0:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392425 2.3 0.3 70530584 6645580 ? Sl 00:37 0:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392426 2.9 0.3 70577764 6610464 ? Sl 00:37 0:46 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392427 3.0 0.3 70772324 7082640 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392614 3.2 0.3 70633748 6807920 ? Sl 00:37 0:51 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392677 2.2 0.4 72919044 9107532 ? Sl 00:37 0:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392713 3.4 0.3 71292540 7325300 ? Sl 00:37 0:54 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392742 2.6 0.3 71071000 6731668 ? Sl 00:37 0:41 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392842 1.9 0.3 72074248 7942224 ? Sl 00:37 0:30 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392930 2.7 0.3 71084208 6902948 ? Sl 00:37 0:43 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392931 3.6 0.3 70683024 6624708 ? Sl 00:37 0:57 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1392932 3.0 0.3 71070224 6815316 ? Sl 00:37 0:47 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1393119 3.5 0.4 74594924 10133240 ? Sl 00:37 0:56 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1393166 1.6 0.2 70142384 5961684 ? Sl 00:37 0:25 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1393324 1.8 0.3 71550540 7294576 ? Sl 00:37 0:29 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1393377 2.3 0.3 70717836 6770356 ? Sl 00:37 0:36 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1393506 2.9 0.4 73173296 8870636 ? Sl 00:37 0:46 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1393623 2.5 0.2 70143284 6297576 ? Sl 00:37 0:40 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1393749 2.2 0.3 71642508 7109104 ? Sl 00:37 0:35 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1393833 2.8 0.3 70749876 6937276 ? Sl 00:37 0:45 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1394001 2.0 0.3 70452068 6750208 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1394127 2.4 0.4 73323364 8863940 ? Sl 00:37 0:37 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 1412474 5.0 0.0 14176 5228 ? Ss 01:03 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", + "zhaowan+ 1412587 0.0 0.0 12124 724 ? S 01:03 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", + "\n", + "==== STDERR ====\n", + " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", + "bash: no job control in this shell\n", + "\n", + "HOST: h800-171\n", + "COMMAND: bash -ic 'ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"'\n", + "==== STDOUT ====\n", + " zhaowan+ 120244 0.7 0.0 20732364 256532 ? Ssl 00:23 0:18 /home/zhaowangbo/.conda/envs/opensora/bin/python /home/zhaowangbo/.conda/envs/opensora/bin/torchrun --nproc_per_node=8 --nnodes=8 --node_rank=7 --master_addr=10.20.1.80 --master_port=29500 scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 120384 137 0.5 119935896 11595724 ? Ssl 00:23 54:28 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 120385 137 0.5 107571912 11274228 ? Ssl 00:23 54:28 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 120386 136 0.5 113510612 12494568 ? Ssl 00:23 54:09 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 120387 137 0.6 106352948 12776980 ? Ssl 00:23 54:35 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 120388 92.8 0.5 119958188 12475232 ? Ssl 00:23 36:51 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 120389 92.7 0.6 116996276 14392176 ? Ssl 00:23 36:49 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 120390 92.6 0.6 107300164 13560148 ? Ssl 00:23 36:46 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 120391 92.9 0.5 108523264 12508980 ? Ssl 00:23 36:54 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122194 2.5 0.2 70196928 5860308 ? Sl 00:37 0:40 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122195 2.0 0.3 70691132 6399516 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122196 2.7 0.3 70276004 6573280 ? Sl 00:37 0:43 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122197 2.3 0.3 70801020 6709944 ? Sl 00:37 0:37 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122443 3.3 0.3 70527872 6484788 ? Sl 00:37 0:53 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122447 3.5 0.3 72106556 7645904 ? Sl 00:37 0:56 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122451 3.5 0.3 71078596 6954384 ? Sl 00:37 0:55 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122635 2.4 0.3 72706452 8418800 ? Sl 00:37 0:39 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122661 2.8 0.3 72191496 7977696 ? Sl 00:37 0:45 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122762 2.0 0.3 70892388 6847728 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122825 2.0 0.3 71025408 6810188 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122861 3.0 0.3 71405828 7367040 ? Sl 00:37 0:47 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122951 3.1 0.3 70822592 6657832 ? Sl 00:37 0:49 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122952 2.8 0.3 70959848 6868352 ? Sl 00:37 0:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 122953 4.2 0.3 70955480 6863580 ? Sl 00:37 1:06 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 123140 2.9 0.3 71693200 7601852 ? Sl 00:37 0:47 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 123203 2.8 0.3 71441156 7240460 ? Sl 00:37 0:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 123266 3.7 0.3 72158476 7945096 ? Sl 00:37 0:58 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 123267 2.0 0.3 70479600 6564124 ? Sl 00:37 0:32 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 123392 3.4 0.3 72158780 8116516 ? Sl 00:37 0:55 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 123455 2.1 0.3 72409968 8072876 ? Sl 00:37 0:33 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 123546 2.8 0.3 70490828 6522300 ? Sl 00:37 0:44 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 123563 1.7 0.3 72276652 7863008 ? Sl 00:37 0:28 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 123583 2.1 0.3 70431412 6462832 ? Sl 00:37 0:34 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 123757 3.0 0.3 71458756 7293788 ? Sl 00:37 0:47 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 123896 1.9 0.3 70803708 6641372 ? Sl 00:37 0:30 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 123897 1.6 0.3 70586316 6740684 ? Sl 00:37 0:26 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 124148 2.2 0.3 71501648 7459620 ? Sl 00:37 0:35 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 124149 3.0 0.3 72552488 8387624 ? Sl 00:37 0:48 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 124207 1.8 0.3 70308448 6462020 ? Sl 00:37 0:28 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 124216 2.3 0.3 71672824 7303504 ? Sl 00:37 0:37 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 124590 2.7 0.3 72167568 8125976 ? Sl 00:37 0:43 /home/zhaowangbo/.conda/envs/opensora/bin/python -u scripts/train.py configs/opensora-v1-1/train/video.py --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --wandb True --load outputs/764-STDiT2-XL-2/epoch1-global_step6000\n", + "zhaowan+ 141814 5.0 0.0 8496 5120 ? Ss 01:03 0:00 bash -ic ps ux | cat | grep --color=never -E \"python|sleep|torchrun|colossal\"\n", + "zhaowan+ 141954 0.0 0.0 6444 724 ? S 01:03 0:00 grep --color=auto --color=never -E python|sleep|torchrun|colossal\n", + "\n", + "==== STDERR ====\n", + " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", + "bash: no job control in this shell\n", + "\n" + ] + } + ], + "source": [ + "ps()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "nvitop(host)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "kill(, host)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Training" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [], + "source": [ + "def colossal_run(data_path, load_path=None):\n", + " commands = []\n", + " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", + " command = f\"colossalai run --nproc_per_node 8 --hostfile hostfile scripts/train.py configs/opensora-v1-1/train/video.py --wandb True --data-path {data_path}\"\n", + " if load_path:\n", + " command = f\"{command} --load-path {load_path}\"\n", + " commands.append(command)\n", + " cmd = \" && \".join(commands)\n", + " return cmd\n", + "\n", + "\n", + "def kill_all():\n", + " commands = []\n", + " commands.append(f\"cd {OPEN_SORA_HOME}\")\n", + " commands.append('cat hostfile | xargs -I \"{}\" ssh \"{}\" pkill -9 python')\n", + " cmd = \" && \".join(commands)\n", + " return cmd" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "cd /home/zhaowangbo/zangwei/opensora/ && colossalai run --nproc_per_node 8 --hostfile hostfile scripts/train.py configs/opensora-v1-1/train/video.py --wandb True --data-path /home/zhaowangbo/data/csv/video_image_test_2.csv --load-path outputs/764-STDiT2-XL-2/epoch1-global_step6000\n" + ] + } + ], + "source": [ + "host = \"h800-80\"\n", + "log_file = os.path.join(OPEN_SORA_HOME, \"logs/train_02.log\")\n", + "data_path = \"/home/zhaowangbo/data/csv/video_image_test_2.csv\"\n", + "ckpt_path = \"outputs/764-STDiT2-XL-2/epoch1-global_step6000\"\n", + "cmd = colossal_run(data_path, ckpt_path)\n", + "print(cmd)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "run_command(cmd, host, log_file=log_file, nohup=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 27, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "HOST: h800-80\n", + "COMMAND: bash -ic 'cd /home/zhaowangbo/zangwei/opensora/ && cat hostfile | xargs -I \"{}\" ssh \"{}\" pkill -9 python'\n", + "==== STDERR ====\n", + " bash: cannot set terminal process group (-1): Inappropriate ioctl for device\n", + "bash: no job control in this shell\n", + "pkill: killing pid 382879 failed: Operation not permitted\n", + "\n" + ] + } + ], + "source": [ + "cmd = kill_all()\n", + "run_command(cmd, host)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/scripts/train.py b/scripts/train.py index 8ddf9c6..d25c4a4 100644 --- a/scripts/train.py +++ b/scripts/train.py @@ -304,6 +304,7 @@ def main(): dataloader.sampler.set_start_index(0) if cfg.dataset.type == "VariableVideoTextDataset": dataloader.batch_sampler.set_epoch(epoch + 1) + print("Epoch done, recomputing batch sampler") start_step = 0