nicolabs/Open-Sora
1
0
Fork 0
mirror of https://github.com/hpcaitech/Open-Sora.git synced 2026-04-17 14:25:07 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

[fix] merge csv

Browse source
This commit is contained in:
Zangwei Zheng 2024-04-09 14:01:10 +08:00
parent e00f93d111
commit aced1fb80f
1 changed files with 3 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
tools/datasets/csvutil.py
View file

@ -451,7 +451,9 @@ def main(args):
if args.intersection is not None:
data_new = pd.read_csv(args.intersection)
print(f"Intersection csv contains {len(data_new)} samples.")
data = pd.merge(data, data_new, on="path", how="inner")
cols_to_use = data_new.columns.difference(data.columns)
cols_to_use = cols_to_use.insert(0, "path")
data = pd.merge(data, data_new[cols_to_use], on="path", how="inner")
print(f"Intersection number of samples: {len(data)}.")
# get output path