Git-Mirrors/PushshiftDumps
1
0
Fork 0
mirror of https://github.com/Watchful1/PushshiftDumps.git synced 2025-07-23 14:50:35 -04:00
Code Issues Projects Releases Packages Wiki Activity

Clean up

Browse source
This commit is contained in:
Watchful1 2021-09-09 22:24:14 -07:00
parent bd7378ff91
commit dd12687141
4 changed files with 40 additions and 29 deletions
Download patch file Download diff file Expand all files Collapse all files

4
scripts/single_file.py
View file

@ -1,3 +1,5 @@
# this is an example of loading and iterating over a single file
import zstandard
import os
import json
@ -29,7 +31,7 @@ def read_lines_zst(file_name):
if __name__ == "__main__":
file_path = r"\\MYCLOUDPR4100\Public\reddit\submissions\RS_2013-03.zst"
file_path = sys.argv[1]
file_size = os.stat(file_path).st_size
file_lines = 0
file_bytes_processed = 0