AI/gpt4all
1
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all.git synced 2024-10-01 01:06:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix: don't rewrite clean file if exists!

Browse Source
This commit is contained in:
Zach Nussbaum 2023-03-26 18:15:22 +00:00
parent eac7734cbf
commit dfee6963a1
1 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
clean.py
View File

@ -6,8 +6,10 @@ import jsonlines
import pandas as pd
prompt_generation_dir = "prompts-reponses"
prompt_generation_dir = "raw_data_sanity_cleaned_delobotomized"
for file in glob.glob(os.path.join(prompt_generation_dir, "*.jsonl")):
if "clean" in file:
continue
data = []
print(file)
with open(file) as f: