mirror of
https://github.com/Watchful1/PushshiftDumps.git
synced 2025-07-25 15:45:19 -04:00
Some cleanup, optimize multiprocess
This commit is contained in:
Watchful1
parent
461028b401
commit
1a99630073
5 changed files with 246 additions and 237 deletions
2
Pipfile
2
Pipfile
|
|
@ -13,4 +13,4 @@ scipy = "*"
|
|||
[dev-packages]
|
||||
|
||||
[requires]
|
||||
python_version = "3.8"
|
||||
python_version = "3.9"
|
||||
|
|
|
|||
409
Pipfile.lock
generated
409
Pipfile.lock
generated
|
|
@ -1,11 +1,11 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "7b82a2e74212a0cbf95cc089a5f0752f547e89f374d95e3d84ccf22b1ecbd6bd"
|
||||
"sha256": "a51fa71c4c074b077d533e5468b6ea282e5d0bef9d17ddaf122e43488868d7f1"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
"python_version": "3.8"
|
||||
"python_version": "3.9"
|
||||
},
|
||||
"sources": [
|
||||
{
|
||||
|
|
@ -18,18 +18,19 @@
|
|||
"default": {
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:78884e7c1d4b00ce3cea67b44566851c4343c120abd683433ce934a68ea58872",
|
||||
"sha256:d62a0163eb4c2344ac042ab2bdf75399a71a2d8c7d47eac2e2ee91b9d6339569"
|
||||
"sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d",
|
||||
"sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"
|
||||
],
|
||||
"version": "==2021.10.8"
|
||||
"markers": "python_full_version >= '3.6.0'",
|
||||
"version": "==2022.6.15"
|
||||
},
|
||||
"charset-normalizer": {
|
||||
"hashes": [
|
||||
"sha256:1eecaa09422db5be9e29d7fc65664e6c33bd06f9ced7838578ba40d58bdf3721",
|
||||
"sha256:b0b883e8e874edfdece9c28f314e3dd5badf067342e42fb162203335ae61aa2c"
|
||||
"sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5",
|
||||
"sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"
|
||||
],
|
||||
"markers": "python_version >= '3'",
|
||||
"version": "==2.0.9"
|
||||
"markers": "python_full_version >= '3.6.0'",
|
||||
"version": "==2.1.0"
|
||||
},
|
||||
"discord-logging": {
|
||||
"editable": true,
|
||||
|
|
@ -38,247 +39,233 @@
|
|||
},
|
||||
"dnspython": {
|
||||
"hashes": [
|
||||
"sha256:95d12f6ef0317118d2a1a6fc49aac65ffec7eb8087474158f42f26a639135216",
|
||||
"sha256:e4a87f0b573201a0f3727fa18a516b055fd1107e0e5477cded4a2de497df1dd4"
|
||||
"sha256:0f7569a4a6ff151958b64304071d370daa3243d15941a7beedf0c9fe5105603e",
|
||||
"sha256:a851e51367fb93e9e1361732c1d60dab63eff98712e503ea7d92e6eccb109b4f"
|
||||
],
|
||||
"version": "==2.1.0"
|
||||
"version": "==2.2.1"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
|
||||
"sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
|
||||
],
|
||||
"markers": "python_version >= '3'",
|
||||
"markers": "python_version >= '3.5'",
|
||||
"version": "==3.3"
|
||||
},
|
||||
"numpy": {
|
||||
"hashes": [
|
||||
"sha256:0b78ecfa070460104934e2caf51694ccd00f37d5e5dbe76f021b1b0b0d221823",
|
||||
"sha256:1247ef28387b7bb7f21caf2dbe4767f4f4175df44d30604d42ad9bd701ebb31f",
|
||||
"sha256:1403b4e2181fc72664737d848b60e65150f272fe5a1c1cbc16145ed43884065a",
|
||||
"sha256:170b2a0805c6891ca78c1d96ee72e4c3ed1ae0a992c75444b6ab20ff038ba2cd",
|
||||
"sha256:2e4ed57f45f0aa38beca2a03b6532e70e548faf2debbeb3291cfc9b315d9be8f",
|
||||
"sha256:32fe5b12061f6446adcbb32cf4060a14741f9c21e15aaee59a207b6ce6423469",
|
||||
"sha256:34f3456f530ae8b44231c63082c8899fe9c983fd9b108c997c4b1c8c2d435333",
|
||||
"sha256:4c9c23158b87ed0e70d9a50c67e5c0b3f75bcf2581a8e34668d4e9d7474d76c6",
|
||||
"sha256:5d95668e727c75b3f5088ec7700e260f90ec83f488e4c0aaccb941148b2cd377",
|
||||
"sha256:615d4e328af7204c13ae3d4df7615a13ff60a49cb0d9106fde07f541207883ca",
|
||||
"sha256:69077388c5a4b997442b843dbdc3a85b420fb693ec8e33020bb24d647c164fa5",
|
||||
"sha256:74b85a17528ca60cf98381a5e779fc0264b4a88b46025e6bcbe9621f46bb3e63",
|
||||
"sha256:81225e58ef5fce7f1d80399575576fc5febec79a8a2742e8ef86d7b03beef49f",
|
||||
"sha256:8890b3360f345e8360133bc078d2dacc2843b6ee6059b568781b15b97acbe39f",
|
||||
"sha256:92aafa03da8658609f59f18722b88f0a73a249101169e28415b4fa148caf7e41",
|
||||
"sha256:9864424631775b0c052f3bd98bc2712d131b3e2cd95d1c0c68b91709170890b0",
|
||||
"sha256:9e6f5f50d1eff2f2f752b3089a118aee1ea0da63d56c44f3865681009b0af162",
|
||||
"sha256:a3deb31bc84f2b42584b8c4001c85d1934dbfb4030827110bc36bfd11509b7bf",
|
||||
"sha256:ad010846cdffe7ec27e3f933397f8a8d6c801a48634f419e3d075db27acf5880",
|
||||
"sha256:b1e2312f5b8843a3e4e8224b2b48fe16119617b8fc0a54df8f50098721b5bed2",
|
||||
"sha256:bc988afcea53e6156546e5b2885b7efab089570783d9d82caf1cfd323b0bb3dd",
|
||||
"sha256:c449eb870616a7b62e097982c622d2577b3dbc800aaf8689254ec6e0197cbf1e",
|
||||
"sha256:c74c699b122918a6c4611285cc2cad4a3aafdb135c22a16ec483340ef97d573c",
|
||||
"sha256:c885bfc07f77e8fee3dc879152ba993732601f1f11de248d4f357f0ffea6a6d4",
|
||||
"sha256:e3c3e990274444031482a31280bf48674441e0a5b55ddb168f3a6db3e0c38ec8",
|
||||
"sha256:e4799be6a2d7d3c33699a6f77201836ac975b2e1b98c2a07f66a38f499cb50ce",
|
||||
"sha256:e6c76a87633aa3fa16614b61ccedfae45b91df2767cf097aa9c933932a7ed1e0",
|
||||
"sha256:e89717274b41ebd568cd7943fc9418eeb49b1785b66031bc8a7f6300463c5898",
|
||||
"sha256:f5162ec777ba7138906c9c274353ece5603646c6965570d82905546579573f73",
|
||||
"sha256:fde96af889262e85aa033f8ee1d3241e32bf36228318a61f1ace579df4e8170d"
|
||||
"sha256:1408c3527a74a0209c781ac82bde2182b0f0bf54dea6e6a363fe0cc4488a7ce7",
|
||||
"sha256:173f28921b15d341afadf6c3898a34f20a0569e4ad5435297ba262ee8941e77b",
|
||||
"sha256:1865fdf51446839ca3fffaab172461f2b781163f6f395f1aed256b1ddc253622",
|
||||
"sha256:3119daed207e9410eaf57dcf9591fdc68045f60483d94956bee0bfdcba790953",
|
||||
"sha256:35590b9c33c0f1c9732b3231bb6a72d1e4f77872390c47d50a615686ae7ed3fd",
|
||||
"sha256:37e5ebebb0eb54c5b4a9b04e6f3018e16b8ef257d26c8945925ba8105008e645",
|
||||
"sha256:37ece2bd095e9781a7156852e43d18044fd0d742934833335599c583618181b9",
|
||||
"sha256:3ab67966c8d45d55a2bdf40701536af6443763907086c0a6d1232688e27e5447",
|
||||
"sha256:47f10ab202fe4d8495ff484b5561c65dd59177949ca07975663f4494f7269e3e",
|
||||
"sha256:55df0f7483b822855af67e38fb3a526e787adf189383b4934305565d71c4b148",
|
||||
"sha256:5d732d17b8a9061540a10fda5bfeabca5785700ab5469a5e9b93aca5e2d3a5fb",
|
||||
"sha256:68b69f52e6545af010b76516f5daaef6173e73353e3295c5cb9f96c35d755641",
|
||||
"sha256:7e8229f3687cdadba2c4faef39204feb51ef7c1a9b669247d49a24f3e2e1617c",
|
||||
"sha256:8002574a6b46ac3b5739a003b5233376aeac5163e5dcd43dd7ad062f3e186129",
|
||||
"sha256:876f60de09734fbcb4e27a97c9a286b51284df1326b1ac5f1bf0ad3678236b22",
|
||||
"sha256:9ce242162015b7e88092dccd0e854548c0926b75c7924a3495e02c6067aba1f5",
|
||||
"sha256:a35c4e64dfca659fe4d0f1421fc0f05b8ed1ca8c46fb73d9e5a7f175f85696bb",
|
||||
"sha256:aeba539285dcf0a1ba755945865ec61240ede5432df41d6e29fab305f4384db2",
|
||||
"sha256:b15c3f1ed08df4980e02cc79ee058b788a3d0bef2fb3c9ca90bb8cbd5b8a3a04",
|
||||
"sha256:c2f91f88230042a130ceb1b496932aa717dcbd665350beb821534c5c7e15881c",
|
||||
"sha256:d748ef349bfef2e1194b59da37ed5a29c19ea8d7e6342019921ba2ba4fd8b624",
|
||||
"sha256:e0d7447679ae9a7124385ccf0ea990bb85bb869cef217e2ea6c844b6a6855073"
|
||||
],
|
||||
"markers": "python_version < '3.11' and python_version >= '3.7'",
|
||||
"version": "==1.21.4"
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==1.23.1"
|
||||
},
|
||||
"pymongo": {
|
||||
"extras": [
|
||||
"srv"
|
||||
],
|
||||
"hashes": [
|
||||
"sha256:0238e53b452ab699b5e2e3f8af2557844c80ab0d0c7a0e066226882838e72756",
|
||||
"sha256:0271bbba36bb130202e011171c1883c4c193036ad0b1e02ecfbea6837790b7de",
|
||||
"sha256:069d49b193f94bb1d748cfd8faf697060a2299f40d86bf5b6d41dd3cedf0fd48",
|
||||
"sha256:06af6e6374ee2bb70f724e09ddf9402907a3d6714828b908737948cd83e5685c",
|
||||
"sha256:0c77cd3dbe0dd9e7cdf8c93dc24e5a4fcb56e115ffb259d4f399e4aaf3f3c62d",
|
||||
"sha256:0e9a2628bcd896368ede456bcfe189d9ca65b18fb0dd91974cb734baf2e24af9",
|
||||
"sha256:12d336bdbe60982de55651be397b5e49d7eadd2aa144f11da353002cd52502ed",
|
||||
"sha256:132cc67b909835d7c230888387b4cc9596d4559a3ce90d947e03bc0b0ffe420b",
|
||||
"sha256:13d0624c13a91da71fa0d960205d93b3d98344481be865ee7cc238c972d41d73",
|
||||
"sha256:1498f388181ae5592c7b60549faaefaffc62d6e3754097576611cb642d21d37b",
|
||||
"sha256:1617fd52da7b208fe5ea176d251dd7cf1b5309e5a4272754b9599edfdf7e64e5",
|
||||
"sha256:177ed1b14aa4f84f00ebef1b0f785680fbaa610361942b23eb54f562fe4c6b34",
|
||||
"sha256:186b2ff4518c1c169fcef5047deb0e6c13a2354d143859587e745fd9f2cf68e9",
|
||||
"sha256:1ba8eb426d56556fffec53d600a8f2572589c19d50b30f61daa8f4d72ab92fbe",
|
||||
"sha256:1c153274699424e8f89f2097d5113f8cbe7898a8d62afaad0270a0f0bd0af53b",
|
||||
"sha256:1fd71b4d7070b01c7f66edc44c1ec2f8bcace2761c3a6ecc10449a40e474d2fa",
|
||||
"sha256:28afb00423e521f4b04fb8f75da7c0215e46631e821e27abf5a7176f9b671f47",
|
||||
"sha256:349c8e522e0b785f442fc9d7fc01c59f7f13f1abe9395310d0d817cff03ec034",
|
||||
"sha256:35a5843546bcbe0422f30b4b2bd5e0b630b04cc4006492c70e8168a921d94b9e",
|
||||
"sha256:38b21eddd021a943b1978b0a3d42e974956a338e3dbb88d56aeb8b8799abd6e8",
|
||||
"sha256:3a4eb0a4db8a2d960bdd5354f05e2e57530e83d333cb644fb2b7120a7a954a69",
|
||||
"sha256:40269fe6bb79fe00c8ba7c2f2d542a82711eb234c3dedb90b7e489386120e9d1",
|
||||
"sha256:426584e99af31ad2398e617c3eb0f1ebcda37f0ffb2d3e56087cdaf23a2f1689",
|
||||
"sha256:47a58f15fc70198cf95982f9699e17fec12287b90f30e90c5e2b7c1c1bc07914",
|
||||
"sha256:512059a902ea2cbcd0afac370af580e67ccd4c7e41ecaff0f0fbd03653b25ca2",
|
||||
"sha256:51664dac8d9b138259876f324adca5ab31d991acf88d1d0ffcc94f423ff2e31b",
|
||||
"sha256:59a4a5fe5379e4fa93380fd0b55bccbdbeb8d04fcfbbad8b42bd31610d5ed3ad",
|
||||
"sha256:5cbfa85a12cfe3dca21951cd432051c505ac461bd9f4a635207d982dd9df2373",
|
||||
"sha256:5fea4207fec8909e155a7948c987eac61949dbbe97fd0c388e587d06ba9bc78d",
|
||||
"sha256:6183476860511cb553a7e4c40936221b6985af7852029c84df898370ec8a028c",
|
||||
"sha256:62459b91a513a7b441cfd70ea7fd15c50b858877ca823915d32bab08fe173edb",
|
||||
"sha256:633ca2001f80900142068bab907feca99554b557ac105c74a9ed157ed38ca5d6",
|
||||
"sha256:65f8a93816dcb2202710839907759aca9eece94d9f13215686f224fcc8966f9e",
|
||||
"sha256:686c40344f7f82c4deaa4e17aa46ad97df51263be1434aeedd2d6b6f38c7f44a",
|
||||
"sha256:6cd7a4321e718cb98a7c7c475b0757e77fdaf1cdb013d7d2e781ba45219e1144",
|
||||
"sha256:6f0605b1146bc24c720aac0e806492144aea9d5a4dc956589e0544301862756a",
|
||||
"sha256:716499113650aacfe1b94d37e0a863f1e84b8d47737c74a2f44f8dfccad46952",
|
||||
"sha256:71810eade75ae1c466adc158d1fa8141040f75427b76240316d97f3c89edd72f",
|
||||
"sha256:72a0c06b76b254bdec18af9add3b8d35796dda51e64a5e0e48d40bff7b41ab13",
|
||||
"sha256:7450b25a803b0f57dae4c3fbd0df742f7f3344c3c9cabb86e4180083c3ebd893",
|
||||
"sha256:75e449ab068af63b7729195343315bc63d242166d88467314be182cc54ce235d",
|
||||
"sha256:7629abba158610cb5db6c22041b287f9398555d72bf9468d44d2efc03d837b81",
|
||||
"sha256:774b9f48bdc385af6654def31e7a7617e01b99cc8aaca1ab3ef6ea0492205e57",
|
||||
"sha256:7a091050bb8d54a5200193b4998e0cf763d083f93d97c7780963c09996f85a38",
|
||||
"sha256:7bdb66340e246b5dcddfcfe79a63ac2ec3808dc394853476f49fc785425040f4",
|
||||
"sha256:812650a2e8a08b812d6a3c937f482bd2c9355e90574964fa283b4d8ef4ae665e",
|
||||
"sha256:84eec41ed982f21ceb58689e16a630a70301eb14499c929388a5bf6464518d9d",
|
||||
"sha256:86d0e28dd5867153d9d9963a4eb17764854a925758fc2db0a814260f82fd4319",
|
||||
"sha256:87dce7c85387ca033cf76cce773ace7675550dcffc456db32a34403439e53e45",
|
||||
"sha256:8869feff59f08cd63979df26aa12343a85bdc7fbd1b79fda8ae39f31a310fa62",
|
||||
"sha256:8baf23d6a0a08b697854e5bcdf82afb91da732cf575fd47ee93945c3654132d8",
|
||||
"sha256:8da525765dbcc1b7abf1bba623f9f701d8759a8fb19594cd71a13b7b0c2c56bd",
|
||||
"sha256:9043bfb816ed50d831acc8d06469dcc41597b4f50c30e62227a93f9f9e37d6c7",
|
||||
"sha256:91c049104b51321e4e18d41edc6850d9f0890ac609b3cb3b8db86dc51666de17",
|
||||
"sha256:93c25fbb5dbc436edbb74101f4da49a42bd3af534513fdf8e75fc72ef035d5e0",
|
||||
"sha256:953129b6b952a9d22042ac23050053444624f630e1928f5f590788660905fa9c",
|
||||
"sha256:9ff0dbec451a2c6226bbd6f2bbbde438bc263e002f3f97d151c8708732ba5197",
|
||||
"sha256:a47f4b24b1360da172cae07ce90e9bd425b6db0052d76142c7fef47173a27283",
|
||||
"sha256:a57e271a0647002b5683dd0c7c2fd7f5fb939357c44396d85298e51a3561b9e3",
|
||||
"sha256:b0606d14892ae2a2b1450e37c8924381e9b64683386a9853e4467f02fd5b44b6",
|
||||
"sha256:b73ff8582964f52ab1bf1a9fdddc1454143172a0b8a9d7d6e3972dd1134f7982",
|
||||
"sha256:bf6047dea1bc8ae19fc14e01b5cb70b3810f91b100d9a535751dd3eadcd3016c",
|
||||
"sha256:c0efc5ab7d9b9e64726496bf650dbc7f1754124a48d076e5292cc5306e61a530",
|
||||
"sha256:c86a0614eda95db036fae01a89f3917d7abdc657c806bac2a32eec74724d9330",
|
||||
"sha256:c878286b1464f462616a47f315d14f02f03512c6b81cb568e996c3f1f79bff8a",
|
||||
"sha256:cd4cde3dfdd347d638171eca53ee6e787d4b1247c6e182f8616039b1df6278d5",
|
||||
"sha256:ceb9a4986f56595e73fffeef3ec037280eda938ed5fe6e4e0961656669d89b32",
|
||||
"sha256:d419e2dbc4943ad6df7ee05e707d7b2c2b512b92407bb6ff643bccbdea399c3a",
|
||||
"sha256:d66462f740dcea496bd779775688a0f805860f0b01998bb59ca22566b098ee26",
|
||||
"sha256:d7514231a03e95072b32d9b335b96253799802ab94647ce83585d5010749380a",
|
||||
"sha256:d9f61b08b60909d936c1f3a4e12c163ca71fd1a4665fc6e078afc6f54f886977",
|
||||
"sha256:da576e59f5f8a642ee26d027479325a45be45defe075b6fa7c84506dabc76883",
|
||||
"sha256:ddaf391ba74eef47eb5afbc40d0b6ddcdbdb417ec8edc8ae95352d25485076db",
|
||||
"sha256:e2b6a323ca545bcb4286d14c0bd75d9a1f5bce2fa1d7fa3621e5f71fd9b8d196",
|
||||
"sha256:e3f6faea65a73ed54111f209b4a411fe012c68f04e8bde96dd7af89b13cac92b",
|
||||
"sha256:e4e36810c541bd1976cd05452e797860b775886cf32c3e8136b9fe48c2c8ba95",
|
||||
"sha256:e5441f4c8142a250695e249e432637c14f79d856a2b60e0974da082e006c53e2",
|
||||
"sha256:e7aedefc87cb46544a3865a19c1d5ca7ddf5ec5ed7dfe162d9538d7543aef499",
|
||||
"sha256:ee2c1fd5bd57fd0092dfa31c1f9f166cf2850f191311603ce343cadcc8608d60",
|
||||
"sha256:f2b6e12f98cce588525f3db802c88f9795d294549ebfe7c2c9bb81333f533ecd",
|
||||
"sha256:f333c0d71dd892683e608f8d1731785a0aa67b1ec012b0d9fc863e8d7224f64e",
|
||||
"sha256:f3e20eb096deea92350f7198a4287d45883a62fe4459d027ce789e72ceba12ee",
|
||||
"sha256:f785375ca2b4e2192786f1e0d2a94c66900d12e780ebae1eccbbab85eb9a7054"
|
||||
"sha256:019a4c13ef1d9accd08de70247068671b116a0383adcd684f6365219f29f41cd",
|
||||
"sha256:07f50a3b8a3afb086089abcd9ab562fb2a27b63fd7017ca13dfe7b663c8f3762",
|
||||
"sha256:08a619c92769bd7346434dfc331a3aa8dc63bee80ed0be250bb0e878c69a6f3e",
|
||||
"sha256:0a3474e6a0df0077a44573727341df6627042df5ca61ea5373c157bb6512ccc7",
|
||||
"sha256:0b8a1c766de29173ddbd316dbd75a97b19a4cf9ac45a39ad4f53426e5df1483b",
|
||||
"sha256:0f7e3872fb7b61ec574b7e04302ea03928b670df583f8691cb1df6e54cd42b19",
|
||||
"sha256:17df40753085ccba38a0e150001f757910d66440d9b5deced30ed4cc8b45b6f3",
|
||||
"sha256:298908478d07871dbe17e9ccd37a10a27ad3f37cc1faaf0cc4d205da3c3e8539",
|
||||
"sha256:302ac0f4825501ab0900b8f1a2bb2dc7d28f69c7f15fbc799fb26f9b9ebb1ecb",
|
||||
"sha256:303d1b3da2461586379d98b344b529598c8156857285ba5bd156dab1c875d1f6",
|
||||
"sha256:306336dab4537b2343e52ec34017c3051c3aee5a961fff4915ab27f7e6d9b1e9",
|
||||
"sha256:30d35a8855f328a85e5002f0908b24e500efdf8f5f78b73098995ce111baa2a9",
|
||||
"sha256:3139c9ddee379c22a9109a0b3bf4cdb64597db2bbd3909f7a2825b47226977a4",
|
||||
"sha256:32e785c37f6a0e844788c6085ea2c9c0c528348c22cebe91896705a92f2b1b26",
|
||||
"sha256:33a5693e8d1fbb7743b7e867d43c1095652a0c6fedddab6cefe6020bee2ca393",
|
||||
"sha256:35d02603c2318676fca5049cdc722bb2e7a378eaccf139ad767365e0eb3bcdbe",
|
||||
"sha256:4516a5ce2beaebddc74d6e304ed520324dda99573c310ef4078284b026f81e93",
|
||||
"sha256:49bb36986f11da2da190a2e777a411c0a28eeb8623850091ea8099b84e3860c7",
|
||||
"sha256:4aa4800530782f7d38aeb169476a5bc692aacc394686f0ca3866e4bb85c9aa3f",
|
||||
"sha256:4d1cdece06156542c18b691511a01fe78a694b9fa287ffd8e15680dbf2beeed5",
|
||||
"sha256:4e4d2babb8737d650250d0fa940ffa1b88aa92b8eb399af093734950a1eeca45",
|
||||
"sha256:4fd5c4f25d8d488ee5701c3ec786f52907dca653b47ce8709bcc2bfb0f5506ae",
|
||||
"sha256:52c8b7bffd2140818ade2aa28c24cfe47935a7273a3bb976d1d8fb17e716536f",
|
||||
"sha256:56b856a459762a3c052987e28ed2bd4b874f0be6671d2cc4f74c4891f47f997a",
|
||||
"sha256:571a3e1ef4abeb4ac719ac381f5aada664627b4ee048d9995e93b4bcd0f70601",
|
||||
"sha256:5cae9c935cdc53e4729920543b7d990615a115d85f32144773bc4b2b05144628",
|
||||
"sha256:5d6ef3fa41f3e3be93483a77f81dea8c7ce5ed4411382a31af2b09b9ec5d9585",
|
||||
"sha256:6396f0db060db9d8751167ea08f3a77a41a71cd39236fade4409394e57b377e8",
|
||||
"sha256:69beffb048de19f7c18617b90e38cbddfac20077b1826c27c3fe2e3ef8ac5a43",
|
||||
"sha256:7507439cd799295893b5602f438f8b6a0f483efb00720df1aa33a39102b41bcf",
|
||||
"sha256:7aa40509dd9f75c256f0a7533d5e2ccef711dbbf0d91c13ac937d21d76d71656",
|
||||
"sha256:7d69a3d980ecbf7238ab37b9027c87ad3b278bb3742a150fc33b5a8a9d990431",
|
||||
"sha256:7dae2cf84a09329617b08731b95ad1fc98d50a9b40c2007e351438bd119a2f7a",
|
||||
"sha256:7f36eacc70849d40ce86c85042ecfcbeab810691b1a3b08062ede32a2d6521ac",
|
||||
"sha256:7f55a602d55e8f0feafde533c69dfd29bf0e54645ab0996b605613cda6894a85",
|
||||
"sha256:8357aa727094798f1d831339ecfd8b3e388c01db6015a3cbd51790cb75e39994",
|
||||
"sha256:84dc6bfeaeba98fe93fc837b12f9af4842694cdbde18083f150e80aec3de88f9",
|
||||
"sha256:86b18420f00d5977bda477369ac85e04185ef94046a04ae0d85f5a807d1a8eb4",
|
||||
"sha256:89f32d8450e15b0c11efdc81e2704d68c502c889d48415a50add9fa031144f75",
|
||||
"sha256:8a1de8931cdad8cd12724e12a6167eef8cb478cc3ee5d2c9f4670c934f2975e1",
|
||||
"sha256:8f106468062ac7ff03e3522a66cb7b36c662326d8eb7af1be0f30563740ff002",
|
||||
"sha256:9a4ea87a0401c06b687db29e2ae836b2b58480ab118cb6eea8ac2ef45a4345f8",
|
||||
"sha256:9ee1b019a4640bf39c0705ab65e934cfe6b89f1a8dc26f389fae3d7c62358d6f",
|
||||
"sha256:a0d7c6d6fbca62508ea525abd869fca78ecf68cd3bcf6ae67ec478aa37cf39c0",
|
||||
"sha256:a1417cb339a367a5dfd0e50193a1c0e87e31325547a0e7624ee4ff414c0b53b3",
|
||||
"sha256:a35f1937b0560587d478fd2259a6d4f66cf511c9d28e90b52b183745eaa77d95",
|
||||
"sha256:a4a35e83abfdac7095430e1c1476e0871e4b234e936f4a7a7631531b09a4f198",
|
||||
"sha256:a7d1c8830a7bc10420ceb60a256d25ab5b032a6dad12a46af6ab2e470cee9124",
|
||||
"sha256:a938d4d5b530f8ea988afb80817209eabc150c53b8c7af79d40080313a35e470",
|
||||
"sha256:a9a2c377106fe01a57bad0f703653de286d56ee5285ed36c6953535cfa11f928",
|
||||
"sha256:baf7546afd27be4f96f23307d7c295497fb512875167743b14a7457b95761294",
|
||||
"sha256:bb21e2f35d6f09aa4a6df0c716f41e036cfcf05a98323b50294f93085ad775e9",
|
||||
"sha256:bc62ba37bcb42e4146b853940b65a2de31c2962d2b6da9bc3ce28270d13b5c4e",
|
||||
"sha256:be3ba736aabf856195199208ed37459408c932940cbccd2dc9f6ff2e800b0261",
|
||||
"sha256:c03eb43d15c8af58159e7561076634d565530aaacaf48cf4e070c3501e88a372",
|
||||
"sha256:c1349331fa743eed4042f9652200e60596f8beb957554acbcbb42aad4272c606",
|
||||
"sha256:c3637cfce519560e2a2579d05eb81e912d109283b8ddc8de46f57ec20d273d92",
|
||||
"sha256:c481cd1af2a77f58f495f7f87c2d715c6f1179d07c1ec927cca1f7977a2d99aa",
|
||||
"sha256:c575f9499e5f540e034ff87bef894f031ae613a98b0d1d3afcc1f482527d5f1c",
|
||||
"sha256:c604831daf2e7e5979ecd97a90cb8c4a7bae208ff45bc792e32eae09c3281afb",
|
||||
"sha256:c759e1e0333664831d8d1d6b26cf59f23f3707758f696c71f506504b33130f81",
|
||||
"sha256:c8a2743dd50629c0222f26c5f55975e45841d985b4b1c7a54b3f03b53de3427d",
|
||||
"sha256:cbcac9263f500da94405cc9fc7e7a42a3ba6c2fe88b2cd7039737cba44c66889",
|
||||
"sha256:cce1b7a680653e31ff2b252f19a39f1ded578a35a96c419ddb9632c62d2af7d8",
|
||||
"sha256:cf96799b3e5e2e2f6dbca015f72b28e7ae415ce8147472f89a3704a035d6336d",
|
||||
"sha256:d06ed18917dbc7a938c4231cbbec52a7e474be270b2ef9208abb4d5a34f5ceb9",
|
||||
"sha256:d4ba5b4f1a0334dbe673f767f28775744e793fcb9ea57a1d72bc622c9f90e6b4",
|
||||
"sha256:d7b8f25c9b0043cbaf77b8b895814e33e7a3c807a097377c07e1bd49946030d5",
|
||||
"sha256:d86511ef8217822fb8716460aaa1ece31fe9e8a48900e541cb35acb7c35e9e2e",
|
||||
"sha256:db8a9cbe965c7343feab2e2bf9a3771f303f8a7ca401dececb6ef28e06b3b18c",
|
||||
"sha256:dbe92a8808cefb284e235b8f82933d7d2e24ff929fe5d53f1fd3ca55fced4b58",
|
||||
"sha256:deb83cc9f639045e2febcc8d4306d4b83893af8d895f2ed70aa342a3430b534c",
|
||||
"sha256:df9084e06efb3d59608a6a443faa9861828585579f0ae8e95f5a4dab70f1a00f",
|
||||
"sha256:dfb89e92746e4a1e0d091cba73d6cc1e16b4094ebdbb14c2e96a80320feb1ad7",
|
||||
"sha256:e13ddfe2ead9540e8773cae098f54c5206d6fcef64846a3e5042db47fc3a41ed",
|
||||
"sha256:e4956384340eec7b526149ac126c8aa11d32441cb3ce77a690cb4821d0d0635c",
|
||||
"sha256:e6eecd027b6ba5617ea6af3e12e20d578d8f4ad1bf51a9abe69c6fd4835ea532",
|
||||
"sha256:eff9818b7671a55f1ce781398607e0d8c304cd430c0581fbe15b868a7a371c27",
|
||||
"sha256:f0aea377b9dfc166c8fa05bb158c30ee3d53d73f0ed2fc05ba6c638d9563422f",
|
||||
"sha256:f1fba193ab2f25849e24caa4570611aa2f80bc1c1ba791851523734b4ed69e43",
|
||||
"sha256:f6db4f00d3baad615e99a865539391243d12b113fb628ebda1d7794ce02d5a10",
|
||||
"sha256:f9405c02af86850e0a8a8ba777b7e7609e0d07bff46adc4f78892cc2d5456018",
|
||||
"sha256:fb4445e3721720c5ca14c0650f35c263b3430e6e16df9d2504618df914b3fb99"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.0.1"
|
||||
"version": "==4.1.1"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:6c1246513ecd5ecd4528a0906f910e8f0f9c6b8ec72030dc9fd154dc1a6efd24",
|
||||
"sha256:b8aa58f8cf793ffd8782d3d8cb19e66ef36f7aba4353eec859e74678b01b07a7"
|
||||
"sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983",
|
||||
"sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.26.0"
|
||||
"version": "==2.28.1"
|
||||
},
|
||||
"scipy": {
|
||||
"hashes": [
|
||||
"sha256:033ce76ed4e9f62923e1f8124f7e2b0800db533828c853b402c7eec6e9465d80",
|
||||
"sha256:173308efba2270dcd61cd45a30dfded6ec0085b4b6eb33b5eb11ab443005e088",
|
||||
"sha256:21b66200cf44b1c3e86495e3a436fc7a26608f92b8d43d344457c54f1c024cbc",
|
||||
"sha256:2c56b820d304dffcadbbb6cbfbc2e2c79ee46ea291db17e288e73cd3c64fefa9",
|
||||
"sha256:304dfaa7146cffdb75fbf6bb7c190fd7688795389ad060b970269c8576d038e9",
|
||||
"sha256:3f78181a153fa21c018d346f595edd648344751d7f03ab94b398be2ad083ed3e",
|
||||
"sha256:4d242d13206ca4302d83d8a6388c9dfce49fc48fdd3c20efad89ba12f785bf9e",
|
||||
"sha256:5d1cc2c19afe3b5a546ede7e6a44ce1ff52e443d12b231823268019f608b9b12",
|
||||
"sha256:5f2cfc359379c56b3a41b17ebd024109b2049f878badc1e454f31418c3a18436",
|
||||
"sha256:65bd52bf55f9a1071398557394203d881384d27b9c2cad7df9a027170aeaef93",
|
||||
"sha256:7edd9a311299a61e9919ea4192dd477395b50c014cdc1a1ac572d7c27e2207fa",
|
||||
"sha256:8499d9dd1459dc0d0fe68db0832c3d5fc1361ae8e13d05e6849b358dc3f2c279",
|
||||
"sha256:866ada14a95b083dd727a845a764cf95dd13ba3dc69a16b99038001b05439709",
|
||||
"sha256:87069cf875f0262a6e3187ab0f419f5b4280d3dcf4811ef9613c605f6e4dca95",
|
||||
"sha256:93378f3d14fff07572392ce6a6a2ceb3a1f237733bd6dcb9eb6a2b29b0d19085",
|
||||
"sha256:95c2d250074cfa76715d58830579c64dff7354484b284c2b8b87e5a38321672c",
|
||||
"sha256:ab5875facfdef77e0a47d5fd39ea178b58e60e454a4c85aa1e52fcb80db7babf",
|
||||
"sha256:b0e0aeb061a1d7dcd2ed59ea57ee56c9b23dd60100825f98238c06ee5cc4467e",
|
||||
"sha256:b78a35c5c74d336f42f44106174b9851c783184a85a3fe3e68857259b37b9ffb",
|
||||
"sha256:c9e04d7e9b03a8a6ac2045f7c5ef741be86727d8f49c45db45f244bdd2bcff17",
|
||||
"sha256:ca36e7d9430f7481fc7d11e015ae16fbd5575615a8e9060538104778be84addf",
|
||||
"sha256:ceebc3c4f6a109777c0053dfa0282fddb8893eddfb0d598574acfb734a926168",
|
||||
"sha256:e2c036492e673aad1b7b0d0ccdc0cb30a968353d2c4bf92ac8e73509e1bf212c",
|
||||
"sha256:eb326658f9b73c07081300daba90a8746543b5ea177184daed26528273157294",
|
||||
"sha256:eb7ae2c4dbdb3c9247e07acc532f91077ae6dbc40ad5bd5dca0bb5a176ee9bda",
|
||||
"sha256:edad1cf5b2ce1912c4d8ddad20e11d333165552aba262c882e28c78bbc09dbf6",
|
||||
"sha256:eef93a446114ac0193a7b714ce67659db80caf940f3232bad63f4c7a81bc18df",
|
||||
"sha256:f7eaea089345a35130bc9a39b89ec1ff69c208efa97b3f8b25ea5d4c41d88094",
|
||||
"sha256:f99d206db1f1ae735a8192ab93bd6028f3a42f6fa08467d37a14eb96c9dd34a3"
|
||||
"sha256:02b567e722d62bddd4ac253dafb01ce7ed8742cf8031aea030a41414b86c1125",
|
||||
"sha256:1166514aa3bbf04cb5941027c6e294a000bba0cf00f5cdac6c77f2dad479b434",
|
||||
"sha256:1da52b45ce1a24a4a22db6c157c38b39885a990a566748fc904ec9f03ed8c6ba",
|
||||
"sha256:23b22fbeef3807966ea42d8163322366dd89da9bebdc075da7034cee3a1441ca",
|
||||
"sha256:28d2cab0c6ac5aa131cc5071a3a1d8e1366dad82288d9ec2ca44df78fb50e649",
|
||||
"sha256:2ef0fbc8bcf102c1998c1f16f15befe7cffba90895d6e84861cd6c6a33fb54f6",
|
||||
"sha256:3b69b90c9419884efeffaac2c38376d6ef566e6e730a231e15722b0ab58f0328",
|
||||
"sha256:4b93ec6f4c3c4d041b26b5f179a6aab8f5045423117ae7a45ba9710301d7e462",
|
||||
"sha256:4e53a55f6a4f22de01ffe1d2f016e30adedb67a699a310cdcac312806807ca81",
|
||||
"sha256:6311e3ae9cc75f77c33076cb2794fb0606f14c8f1b1c9ff8ce6005ba2c283621",
|
||||
"sha256:65b77f20202599c51eb2771d11a6b899b97989159b7975e9b5259594f1d35ef4",
|
||||
"sha256:6cc6b33139eb63f30725d5f7fa175763dc2df6a8f38ddf8df971f7c345b652dc",
|
||||
"sha256:70de2f11bf64ca9921fda018864c78af7147025e467ce9f4a11bc877266900a6",
|
||||
"sha256:70ebc84134cf0c504ce6a5f12d6db92cb2a8a53a49437a6bb4edca0bc101f11c",
|
||||
"sha256:83606129247e7610b58d0e1e93d2c5133959e9cf93555d3c27e536892f1ba1f2",
|
||||
"sha256:93d07494a8900d55492401917a119948ed330b8c3f1d700e0b904a578f10ead4",
|
||||
"sha256:9c4e3ae8a716c8b3151e16c05edb1daf4cb4d866caa385e861556aff41300c14",
|
||||
"sha256:9dd4012ac599a1e7eb63c114d1eee1bcfc6dc75a29b589ff0ad0bb3d9412034f",
|
||||
"sha256:9e3fb1b0e896f14a85aa9a28d5f755daaeeb54c897b746df7a55ccb02b340f33",
|
||||
"sha256:a0aa8220b89b2e3748a2836fbfa116194378910f1a6e78e4675a095bcd2c762d",
|
||||
"sha256:d3b3c8924252caaffc54d4a99f1360aeec001e61267595561089f8b5900821bb",
|
||||
"sha256:e013aed00ed776d790be4cb32826adb72799c61e318676172495383ba4570aa4",
|
||||
"sha256:f3e7a8867f307e3359cc0ed2c63b61a1e33a19080f92fe377bc7d49f646f2ec1"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.7.3"
|
||||
"version": "==1.8.1"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece",
|
||||
"sha256:c4fdf4019605b6e5423637e01bc9fe4daef873709a7973e195ceba0a62bbc844"
|
||||
"sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec",
|
||||
"sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
|
||||
"version": "==1.26.7"
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
|
||||
"version": "==1.26.10"
|
||||
},
|
||||
"zstandard": {
|
||||
"hashes": [
|
||||
"sha256:066488e721ec882485a500c216302b443f2eaef39356f7c65130e76c671e3ce2",
|
||||
"sha256:08a728715858f1477239887ba3c692bc462b2c86e7a8e467dc5affa7bba9093f",
|
||||
"sha256:11216b47c62e9fc71a25f4b42f525a81da268071bdb434bc1e642ffc38a24a02",
|
||||
"sha256:127c4c93f578d9b509732c74ed9b44b23e94041ba11b13827be0a7d2e3869b39",
|
||||
"sha256:12dddee2574b00c262270cfb46bd0c048e92208b95fdd39ad2a9eac1cef30498",
|
||||
"sha256:1bdda52224043e13ed20f847e3b308de1c9372d1563824fad776b1cf1f847ef0",
|
||||
"sha256:2e31680d1bcf85e7a58a45df7365af894402ae77a9868c751dc991dd13099a5f",
|
||||
"sha256:42992e89b250fe6878c175119af529775d4be7967cd9de86990145d615d6a444",
|
||||
"sha256:453e42af96923582ddbf3acf843f55d2dc534a3f7b345003852dd522aa51eae6",
|
||||
"sha256:4d8a296dab7f8f5d53acc693a6785751f43ca39b51c8eabc672f978306fb40e6",
|
||||
"sha256:5251ac352d8350869c404a0ca94457da018b726f692f6456ec82bbf907fbc956",
|
||||
"sha256:57a6cfc34d906d514358769ed6d510b312be1cf033aafb5db44865a6717579bd",
|
||||
"sha256:6ed51162e270b9b8097dcae6f2c239ada05ec112194633193ec3241498988924",
|
||||
"sha256:74cbea966462afed5a89eb99e4577538d10d425e05bf6240a75c086d59ccaf89",
|
||||
"sha256:87bea44ad24c15cd872263c0d5f912186a4be3db361eab3b25f1a61dcb5ca014",
|
||||
"sha256:8a745862ed525eee4e28bdbd58bf3ea952bf9da3c31bb4e4ce11ef15aea5c625",
|
||||
"sha256:8b760fc8118b1a0aa1d8f4e2012622e8f5f178d4b8cb94f8c6d2948b6a49a485",
|
||||
"sha256:8c8c0e813b67de1c9d7f2760768c4ae53f011c75ace18d5cff4fb40d2173763f",
|
||||
"sha256:8d5fe983e23b05f0e924fe8d0dd3935f0c9fd3266e4c6ff8621c12c350da299d",
|
||||
"sha256:8f5785c0b9b71d49d789240ae16a636728596631cf100f32b963a6f9857af5a4",
|
||||
"sha256:91efd5ea5fb3c347e7ebb6d5622bfa37d72594a2dec37c5dde70b691edb6cc03",
|
||||
"sha256:92e6c1a656390176d51125847f2f422f9d8ed468c24b63958f6ee50d9aa98c83",
|
||||
"sha256:9bcbfe1ec89789239f63daeea8778488cb5ba9034a374d7753815935f83dad65",
|
||||
"sha256:a92aa26789f17ca3b1f45cc7e728597165e2b166b99d1204bb397a672edee761",
|
||||
"sha256:a9ec6de2c058e611e9dfe88d9809a5676bc1d2a53543c1273a90a60e41b8f43c",
|
||||
"sha256:ac5d97f9dece91a1162f651da79b735c5cde4d5863477785962aad648b592446",
|
||||
"sha256:ae19628886d994ac1f3d2fc7f9ed5bb551d81000f7b4e0c57a0e88301aea2766",
|
||||
"sha256:b2ea1937eff0ed5621876dc377933fe76624abfb2ab5b418995f43af6bac50de",
|
||||
"sha256:b46220bef7bf9271a2a05512e86acbabc86cca08bebde8447bdbb4acb3179447",
|
||||
"sha256:b61586b0ff55c4137e512f1e9df4e4d7a6e1e9df782b4b87652df27737c90cc1",
|
||||
"sha256:be68fbac1e88f0dbe033a2d2e3aaaf9c8307730b905f3cd3c698ca4b904f0702",
|
||||
"sha256:c75557d53bb2d064521ff20cce9b8a51ee8301e031b1d6bcedb6458dda3bc85d",
|
||||
"sha256:c7e6b6ad58ae6f77872da9376ef0ecbf8c1ae7a0c8fc29a2473abc90f79a9a1b",
|
||||
"sha256:c8828f4e78774a6c0b8d21e59677f8f48d2e17fe2ef72793c94c10abc032c41c",
|
||||
"sha256:cae9bfcb9148152f8bfb9163b4b779326ca39fe9889e45e0572c56d25d5021be",
|
||||
"sha256:ce61492764d0442ca1e81d38d7bf7847d7df5003bce28089bab64c0519749351",
|
||||
"sha256:d40447f4a44b442fa6715779ff49a1e319729d829198279927d18bca0d7ac32d",
|
||||
"sha256:d9946cfe54bf3365f14a5aa233eb2425de3b77eac6a4c7d03dda7dbb6acd3267",
|
||||
"sha256:dd5a2287893e52204e4ce9d0e1bcea6240661dbb412efb53d5446b881d3c10a2",
|
||||
"sha256:e9456492eb13249841e53221e742bef93f4868122bfc26bafa12a07677619732",
|
||||
"sha256:eaae2d3e8fdf8bfe269628385087e4b648beef85bb0c187644e7df4fb0fe9046",
|
||||
"sha256:eba125d3899f2003debf97019cd6f46f841a405df067da23d11443ad17952a40",
|
||||
"sha256:ef759c1dfe78aa5a01747d3465d2585de14e08fc2b0195ce3f31f45477fc5a72",
|
||||
"sha256:ffe1d24c5e11e98e4c5f96f846cdd19619d8c7e5e8e5082bed62d39baa30cecb"
|
||||
"sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02",
|
||||
"sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f",
|
||||
"sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8",
|
||||
"sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab",
|
||||
"sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0",
|
||||
"sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9",
|
||||
"sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b",
|
||||
"sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8",
|
||||
"sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac",
|
||||
"sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887",
|
||||
"sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1",
|
||||
"sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805",
|
||||
"sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936",
|
||||
"sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e",
|
||||
"sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec",
|
||||
"sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67",
|
||||
"sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244",
|
||||
"sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e",
|
||||
"sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5",
|
||||
"sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435",
|
||||
"sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6",
|
||||
"sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784",
|
||||
"sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990",
|
||||
"sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224",
|
||||
"sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c",
|
||||
"sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2",
|
||||
"sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722",
|
||||
"sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b",
|
||||
"sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d",
|
||||
"sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d",
|
||||
"sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab",
|
||||
"sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a",
|
||||
"sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32",
|
||||
"sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e",
|
||||
"sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63",
|
||||
"sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25",
|
||||
"sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb",
|
||||
"sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c",
|
||||
"sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2",
|
||||
"sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0",
|
||||
"sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885",
|
||||
"sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed",
|
||||
"sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a",
|
||||
"sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.16.0"
|
||||
"version": "==0.18.0"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
|
|
|
|||
|
|
@ -115,7 +115,12 @@ def read_lines_zst(file_name):
|
|||
buffer = ''
|
||||
reader = zstandard.ZstdDecompressor(max_window_size=2**31).stream_reader(file_handle)
|
||||
while True:
|
||||
chunk = reader.read(2**27).decode()
|
||||
data_chunk = reader.read(2**27)
|
||||
try:
|
||||
chunk = data_chunk.decode()
|
||||
except UnicodeDecodeError:
|
||||
data_chunk += reader.read(2**29)
|
||||
chunk = data_chunk.decode()
|
||||
if not chunk:
|
||||
break
|
||||
lines = (buffer + chunk).split("\n")
|
||||
|
|
@ -227,7 +232,8 @@ if __name__ == '__main__':
|
|||
total_lines_errored = 0
|
||||
files_to_process = []
|
||||
# calculate the total file size for progress reports, build a list of incomplete files to process
|
||||
for file in input_files:
|
||||
# do this largest to smallest by file size so that we aren't processing a few really big files with only a few threads at the end
|
||||
for file in sorted(input_files, key=lambda item: item.file_size, reverse=True):
|
||||
total_bytes += file.file_size
|
||||
if file.complete:
|
||||
files_processed += 1
|
||||
|
|
@ -254,13 +260,13 @@ if __name__ == '__main__':
|
|||
file_update = queue.get()
|
||||
if file_update.error_message is not None:
|
||||
log.warning(f"File failed {file_update.input_path}: {file_update.error_message}")
|
||||
continue
|
||||
# I'm going to assume that the list of files is short enough that it's no
|
||||
# big deal to just iterate each time since that saves a bunch of work
|
||||
total_lines_processed = 0
|
||||
total_bytes_processed = 0
|
||||
total_lines_errored = 0
|
||||
files_processed = 0
|
||||
files_errored = 0
|
||||
i = 0
|
||||
for file in input_files:
|
||||
if file.input_path == file_update.input_path:
|
||||
|
|
@ -269,9 +275,9 @@ if __name__ == '__main__':
|
|||
total_lines_processed += file.lines_processed
|
||||
total_bytes_processed += file.bytes_processed
|
||||
total_lines_errored += file.error_lines
|
||||
files_processed += 1 if file.complete else 0
|
||||
files_processed += 1 if file.complete or file.error_message is not None else 0
|
||||
i += 1
|
||||
if file_update.complete:
|
||||
if file_update.complete or file_update.error_message is not None:
|
||||
save_file_list(input_files, args.output, args.name)
|
||||
current_time = time.time()
|
||||
progress_queue.put([current_time, total_lines_processed, total_bytes_processed])
|
||||
|
|
@ -287,7 +293,7 @@ if __name__ == '__main__':
|
|||
log.info(
|
||||
f"{total_lines_processed:,} lines at {(total_lines_processed - first_lines)/(current_time - first_time):,.0f}/s, {total_lines_errored:,} errored : "
|
||||
f"{(total_bytes_processed / (2**30)):.2f} gb at {(bytes_per_second / (2**20)):,.0f} mb/s, {(total_bytes_processed / total_bytes) * 100:.0f}% : "
|
||||
f"{files_processed}/{len(input_files)} files : "
|
||||
f"{files_processed}({files_errored})/{len(input_files)} files : "
|
||||
f"{(str(days_left) + 'd ' if days_left > 0 else '')}{hours_left - (days_left * 24)}:{minutes_left - (hours_left * 60):02}:{seconds_left - (minutes_left * 60):02} remaining")
|
||||
|
||||
log.info(f"{total_lines_processed:,}, {total_lines_errored} errored : {(total_bytes_processed / (2**30)):.2f} gb, {(total_bytes_processed / total_bytes) * 100:.0f}% : {files_processed}/{len(input_files)}")
|
||||
|
|
@ -297,7 +303,10 @@ if __name__ == '__main__':
|
|||
# build a list of output files to combine
|
||||
for file in input_files:
|
||||
if not file.complete:
|
||||
log.info(f"File {file.input_path} is not marked as complete")
|
||||
if file.error_message is not None:
|
||||
log.info(f"File {file.input_path} errored {file.error_message}")
|
||||
else:
|
||||
log.info(f"File {file.input_path} is not marked as complete")
|
||||
count_incomplete += 1
|
||||
else:
|
||||
if file.error_lines > file.lines_processed * (args.error_rate * 0.01):
|
||||
|
|
|
|||
|
|
@ -14,11 +14,21 @@ log.addHandler(logging.StreamHandler())
|
|||
|
||||
|
||||
def read_lines_zst(file_name):
|
||||
skip_to = 3566550750
|
||||
bytes_read = 0
|
||||
with open(file_name, 'rb') as file_handle:
|
||||
buffer = ''
|
||||
reader = zstandard.ZstdDecompressor(max_window_size=2**31).stream_reader(file_handle)
|
||||
#reader.read(40000000000)
|
||||
while True:
|
||||
chunk = reader.read(2**27).decode()
|
||||
data_chunk = reader.read(2**27)
|
||||
try:
|
||||
chunk = data_chunk.decode()
|
||||
except UnicodeDecodeError:
|
||||
log.info("Decoding error, reading a second chunk")
|
||||
data_chunk += reader.read(2**29)
|
||||
chunk = data_chunk.decode()
|
||||
|
||||
if not chunk:
|
||||
break
|
||||
lines = (buffer + chunk).split("\n")
|
||||
|
|
@ -27,6 +37,7 @@ def read_lines_zst(file_name):
|
|||
yield line, file_handle.tell()
|
||||
|
||||
buffer = lines[-1]
|
||||
|
||||
reader.close()
|
||||
|
||||
|
||||
|
|
@ -39,19 +50,20 @@ if __name__ == "__main__":
|
|||
field = "subreddit"
|
||||
value = "wallstreetbets"
|
||||
bad_lines = 0
|
||||
try:
|
||||
for line, file_bytes_processed in read_lines_zst(file_path):
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
created = datetime.utcfromtimestamp(int(obj['created_utc']))
|
||||
temp = obj[field] == value
|
||||
except (KeyError, json.JSONDecodeError) as err:
|
||||
bad_lines += 1
|
||||
file_lines += 1
|
||||
if file_lines % 100000 == 0:
|
||||
log.info(f"{created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines:,} : {bad_lines:,} : {(file_bytes_processed / file_size) * 100:.0f}%")
|
||||
except Exception as err:
|
||||
log.info(err)
|
||||
# try:
|
||||
for line, file_bytes_processed in read_lines_zst(file_path):
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
created = datetime.utcfromtimestamp(int(obj['created_utc']))
|
||||
temp = obj[field] == value
|
||||
except (KeyError, json.JSONDecodeError) as err:
|
||||
bad_lines += 1
|
||||
file_lines += 1
|
||||
if file_lines % 100000 == 0:
|
||||
log.info(f"{created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines:,} : {bad_lines:,} : {file_bytes_processed:,}:{(file_bytes_processed / file_size) * 100:.0f}%")
|
||||
|
||||
# except Exception as err:
|
||||
# log.info(err)
|
||||
|
||||
log.info(f"Complete : {file_lines:,} : {bad_lines:,}")
|
||||
|
||||
|
|
|
|||
|
|
@ -56,10 +56,11 @@ if __name__ == "__main__":
|
|||
for line, file_bytes_processed in read_lines_zst(input_file_path):
|
||||
try:
|
||||
obj = json.loads(line)
|
||||
output_obj = []
|
||||
for field in fields:
|
||||
output_obj.append(obj[field].encode("utf-8", errors='replace').decode())
|
||||
writer.writerow(output_obj)
|
||||
if "social dilemma" in obj['body'].lower():
|
||||
output_obj = []
|
||||
for field in fields:
|
||||
output_obj.append(str(obj[field]).encode("utf-8", errors='replace').decode())
|
||||
writer.writerow(output_obj)
|
||||
|
||||
created = datetime.utcfromtimestamp(int(obj['created_utc']))
|
||||
except json.JSONDecodeError as err:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue