mirror of
https://github.com/Watchful1/PushshiftDumps.git
synced 2025-07-25 15:45:19 -04:00
Re-arrange a bit
This commit is contained in:
Watchful1
parent
de209b338a
commit
4f56c141fd
4 changed files with 378 additions and 340 deletions
515
Pipfile.lock
generated
515
Pipfile.lock
generated
|
|
@ -1,7 +1,7 @@
|
|||
{
|
||||
"_meta": {
|
||||
"hash": {
|
||||
"sha256": "a51fa71c4c074b077d533e5468b6ea282e5d0bef9d17ddaf122e43488868d7f1"
|
||||
"sha256": "da2ec602c80b176c90ccdba3624777867aabb098ec6d45f850a2d09e4c7cf22f"
|
||||
},
|
||||
"pipfile-spec": 6,
|
||||
"requires": {
|
||||
|
|
@ -18,254 +18,369 @@
|
|||
"default": {
|
||||
"certifi": {
|
||||
"hashes": [
|
||||
"sha256:84c85a9078b11105f04f3036a9482ae10e4621616db313fe045dd24743a0820d",
|
||||
"sha256:fe86415d55e84719d75f8b69414f6438ac3547d2078ab91b67e779ef69378412"
|
||||
"sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
|
||||
"sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
|
||||
],
|
||||
"markers": "python_full_version >= '3.6.0'",
|
||||
"version": "==2022.6.15"
|
||||
"markers": "python_version >= '3.6'",
|
||||
"version": "==2023.7.22"
|
||||
},
|
||||
"charset-normalizer": {
|
||||
"hashes": [
|
||||
"sha256:5189b6f22b01957427f35b6a08d9a0bc45b46d3788ef5a92e978433c7a35f8a5",
|
||||
"sha256:575e708016ff3a5e3681541cb9d79312c416835686d054a23accb873b254f413"
|
||||
"sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96",
|
||||
"sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c",
|
||||
"sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710",
|
||||
"sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706",
|
||||
"sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020",
|
||||
"sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252",
|
||||
"sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad",
|
||||
"sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329",
|
||||
"sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a",
|
||||
"sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f",
|
||||
"sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6",
|
||||
"sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4",
|
||||
"sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a",
|
||||
"sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46",
|
||||
"sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2",
|
||||
"sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23",
|
||||
"sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace",
|
||||
"sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd",
|
||||
"sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982",
|
||||
"sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10",
|
||||
"sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2",
|
||||
"sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea",
|
||||
"sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09",
|
||||
"sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5",
|
||||
"sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149",
|
||||
"sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489",
|
||||
"sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9",
|
||||
"sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80",
|
||||
"sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592",
|
||||
"sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3",
|
||||
"sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6",
|
||||
"sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed",
|
||||
"sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c",
|
||||
"sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200",
|
||||
"sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a",
|
||||
"sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e",
|
||||
"sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d",
|
||||
"sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6",
|
||||
"sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623",
|
||||
"sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669",
|
||||
"sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3",
|
||||
"sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa",
|
||||
"sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9",
|
||||
"sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2",
|
||||
"sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f",
|
||||
"sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1",
|
||||
"sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4",
|
||||
"sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a",
|
||||
"sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8",
|
||||
"sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3",
|
||||
"sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029",
|
||||
"sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f",
|
||||
"sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959",
|
||||
"sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22",
|
||||
"sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7",
|
||||
"sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952",
|
||||
"sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346",
|
||||
"sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e",
|
||||
"sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d",
|
||||
"sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299",
|
||||
"sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd",
|
||||
"sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a",
|
||||
"sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3",
|
||||
"sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037",
|
||||
"sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94",
|
||||
"sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c",
|
||||
"sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858",
|
||||
"sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a",
|
||||
"sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449",
|
||||
"sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c",
|
||||
"sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918",
|
||||
"sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1",
|
||||
"sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c",
|
||||
"sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac",
|
||||
"sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"
|
||||
],
|
||||
"markers": "python_full_version >= '3.6.0'",
|
||||
"version": "==2.1.0"
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==3.2.0"
|
||||
},
|
||||
"discord-logging": {
|
||||
"editable": true,
|
||||
"git": "https://github.com/Watchful1/DiscordLogging.git",
|
||||
"ref": "9e543194d3612dde92eae1203d0ea143a7963f6e"
|
||||
"ref": "b3861f20c73ff8925a0aefefc7ed01896d340849"
|
||||
},
|
||||
"dnspython": {
|
||||
"hashes": [
|
||||
"sha256:0f7569a4a6ff151958b64304071d370daa3243d15941a7beedf0c9fe5105603e",
|
||||
"sha256:a851e51367fb93e9e1361732c1d60dab63eff98712e503ea7d92e6eccb109b4f"
|
||||
"sha256:57c6fbaaeaaf39c891292012060beb141791735dbb4004798328fc2c467402d8",
|
||||
"sha256:8dcfae8c7460a2f84b4072e26f1c9f4101ca20c071649cb7c34e8b6a93d58984"
|
||||
],
|
||||
"version": "==2.2.1"
|
||||
"markers": "python_version >= '3.8' and python_version < '4.0'",
|
||||
"version": "==2.4.2"
|
||||
},
|
||||
"idna": {
|
||||
"hashes": [
|
||||
"sha256:84d9dd047ffa80596e0f246e2eab0b391788b0503584e8945f2368256d2735ff",
|
||||
"sha256:9d643ff0a55b762d5cdb124b8eaa99c66322e2157b69160bc32796e824360e6d"
|
||||
"sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
|
||||
"sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
|
||||
],
|
||||
"markers": "python_version >= '3.5'",
|
||||
"version": "==3.3"
|
||||
"version": "==3.4"
|
||||
},
|
||||
"numpy": {
|
||||
"hashes": [
|
||||
"sha256:1408c3527a74a0209c781ac82bde2182b0f0bf54dea6e6a363fe0cc4488a7ce7",
|
||||
"sha256:173f28921b15d341afadf6c3898a34f20a0569e4ad5435297ba262ee8941e77b",
|
||||
"sha256:1865fdf51446839ca3fffaab172461f2b781163f6f395f1aed256b1ddc253622",
|
||||
"sha256:3119daed207e9410eaf57dcf9591fdc68045f60483d94956bee0bfdcba790953",
|
||||
"sha256:35590b9c33c0f1c9732b3231bb6a72d1e4f77872390c47d50a615686ae7ed3fd",
|
||||
"sha256:37e5ebebb0eb54c5b4a9b04e6f3018e16b8ef257d26c8945925ba8105008e645",
|
||||
"sha256:37ece2bd095e9781a7156852e43d18044fd0d742934833335599c583618181b9",
|
||||
"sha256:3ab67966c8d45d55a2bdf40701536af6443763907086c0a6d1232688e27e5447",
|
||||
"sha256:47f10ab202fe4d8495ff484b5561c65dd59177949ca07975663f4494f7269e3e",
|
||||
"sha256:55df0f7483b822855af67e38fb3a526e787adf189383b4934305565d71c4b148",
|
||||
"sha256:5d732d17b8a9061540a10fda5bfeabca5785700ab5469a5e9b93aca5e2d3a5fb",
|
||||
"sha256:68b69f52e6545af010b76516f5daaef6173e73353e3295c5cb9f96c35d755641",
|
||||
"sha256:7e8229f3687cdadba2c4faef39204feb51ef7c1a9b669247d49a24f3e2e1617c",
|
||||
"sha256:8002574a6b46ac3b5739a003b5233376aeac5163e5dcd43dd7ad062f3e186129",
|
||||
"sha256:876f60de09734fbcb4e27a97c9a286b51284df1326b1ac5f1bf0ad3678236b22",
|
||||
"sha256:9ce242162015b7e88092dccd0e854548c0926b75c7924a3495e02c6067aba1f5",
|
||||
"sha256:a35c4e64dfca659fe4d0f1421fc0f05b8ed1ca8c46fb73d9e5a7f175f85696bb",
|
||||
"sha256:aeba539285dcf0a1ba755945865ec61240ede5432df41d6e29fab305f4384db2",
|
||||
"sha256:b15c3f1ed08df4980e02cc79ee058b788a3d0bef2fb3c9ca90bb8cbd5b8a3a04",
|
||||
"sha256:c2f91f88230042a130ceb1b496932aa717dcbd665350beb821534c5c7e15881c",
|
||||
"sha256:d748ef349bfef2e1194b59da37ed5a29c19ea8d7e6342019921ba2ba4fd8b624",
|
||||
"sha256:e0d7447679ae9a7124385ccf0ea990bb85bb869cef217e2ea6c844b6a6855073"
|
||||
"sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2",
|
||||
"sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55",
|
||||
"sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf",
|
||||
"sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01",
|
||||
"sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca",
|
||||
"sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901",
|
||||
"sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d",
|
||||
"sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4",
|
||||
"sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf",
|
||||
"sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380",
|
||||
"sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044",
|
||||
"sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545",
|
||||
"sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f",
|
||||
"sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f",
|
||||
"sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3",
|
||||
"sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364",
|
||||
"sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9",
|
||||
"sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418",
|
||||
"sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f",
|
||||
"sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295",
|
||||
"sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3",
|
||||
"sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187",
|
||||
"sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926",
|
||||
"sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357",
|
||||
"sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==1.23.1"
|
||||
"markers": "python_version >= '3.9'",
|
||||
"version": "==1.25.2"
|
||||
},
|
||||
"praw": {
|
||||
"hashes": [
|
||||
"sha256:9ec5dc943db00c175bc6a53f4e089ce625f3fdfb27305564b616747b767d38ef",
|
||||
"sha256:f1d7eef414cafe28080dda12ed09253a095a69933d5c8132eca11d4dc8a070bf"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==7.7.1"
|
||||
},
|
||||
"prawcore": {
|
||||
"hashes": [
|
||||
"sha256:48c17db447fa06a13ca3e722201f443031437708daa736c05a1df895fbcceff5",
|
||||
"sha256:daf1ccd4b7a80dc4e6567d48336d782e94a9a6dad83770fc2edf76dc9a84f56d"
|
||||
],
|
||||
"markers": "python_version ~= '3.6'",
|
||||
"version": "==2.3.0"
|
||||
},
|
||||
"pymongo": {
|
||||
"extras": [
|
||||
"srv"
|
||||
],
|
||||
"hashes": [
|
||||
"sha256:019a4c13ef1d9accd08de70247068671b116a0383adcd684f6365219f29f41cd",
|
||||
"sha256:07f50a3b8a3afb086089abcd9ab562fb2a27b63fd7017ca13dfe7b663c8f3762",
|
||||
"sha256:08a619c92769bd7346434dfc331a3aa8dc63bee80ed0be250bb0e878c69a6f3e",
|
||||
"sha256:0a3474e6a0df0077a44573727341df6627042df5ca61ea5373c157bb6512ccc7",
|
||||
"sha256:0b8a1c766de29173ddbd316dbd75a97b19a4cf9ac45a39ad4f53426e5df1483b",
|
||||
"sha256:0f7e3872fb7b61ec574b7e04302ea03928b670df583f8691cb1df6e54cd42b19",
|
||||
"sha256:17df40753085ccba38a0e150001f757910d66440d9b5deced30ed4cc8b45b6f3",
|
||||
"sha256:298908478d07871dbe17e9ccd37a10a27ad3f37cc1faaf0cc4d205da3c3e8539",
|
||||
"sha256:302ac0f4825501ab0900b8f1a2bb2dc7d28f69c7f15fbc799fb26f9b9ebb1ecb",
|
||||
"sha256:303d1b3da2461586379d98b344b529598c8156857285ba5bd156dab1c875d1f6",
|
||||
"sha256:306336dab4537b2343e52ec34017c3051c3aee5a961fff4915ab27f7e6d9b1e9",
|
||||
"sha256:30d35a8855f328a85e5002f0908b24e500efdf8f5f78b73098995ce111baa2a9",
|
||||
"sha256:3139c9ddee379c22a9109a0b3bf4cdb64597db2bbd3909f7a2825b47226977a4",
|
||||
"sha256:32e785c37f6a0e844788c6085ea2c9c0c528348c22cebe91896705a92f2b1b26",
|
||||
"sha256:33a5693e8d1fbb7743b7e867d43c1095652a0c6fedddab6cefe6020bee2ca393",
|
||||
"sha256:35d02603c2318676fca5049cdc722bb2e7a378eaccf139ad767365e0eb3bcdbe",
|
||||
"sha256:4516a5ce2beaebddc74d6e304ed520324dda99573c310ef4078284b026f81e93",
|
||||
"sha256:49bb36986f11da2da190a2e777a411c0a28eeb8623850091ea8099b84e3860c7",
|
||||
"sha256:4aa4800530782f7d38aeb169476a5bc692aacc394686f0ca3866e4bb85c9aa3f",
|
||||
"sha256:4d1cdece06156542c18b691511a01fe78a694b9fa287ffd8e15680dbf2beeed5",
|
||||
"sha256:4e4d2babb8737d650250d0fa940ffa1b88aa92b8eb399af093734950a1eeca45",
|
||||
"sha256:4fd5c4f25d8d488ee5701c3ec786f52907dca653b47ce8709bcc2bfb0f5506ae",
|
||||
"sha256:52c8b7bffd2140818ade2aa28c24cfe47935a7273a3bb976d1d8fb17e716536f",
|
||||
"sha256:56b856a459762a3c052987e28ed2bd4b874f0be6671d2cc4f74c4891f47f997a",
|
||||
"sha256:571a3e1ef4abeb4ac719ac381f5aada664627b4ee048d9995e93b4bcd0f70601",
|
||||
"sha256:5cae9c935cdc53e4729920543b7d990615a115d85f32144773bc4b2b05144628",
|
||||
"sha256:5d6ef3fa41f3e3be93483a77f81dea8c7ce5ed4411382a31af2b09b9ec5d9585",
|
||||
"sha256:6396f0db060db9d8751167ea08f3a77a41a71cd39236fade4409394e57b377e8",
|
||||
"sha256:69beffb048de19f7c18617b90e38cbddfac20077b1826c27c3fe2e3ef8ac5a43",
|
||||
"sha256:7507439cd799295893b5602f438f8b6a0f483efb00720df1aa33a39102b41bcf",
|
||||
"sha256:7aa40509dd9f75c256f0a7533d5e2ccef711dbbf0d91c13ac937d21d76d71656",
|
||||
"sha256:7d69a3d980ecbf7238ab37b9027c87ad3b278bb3742a150fc33b5a8a9d990431",
|
||||
"sha256:7dae2cf84a09329617b08731b95ad1fc98d50a9b40c2007e351438bd119a2f7a",
|
||||
"sha256:7f36eacc70849d40ce86c85042ecfcbeab810691b1a3b08062ede32a2d6521ac",
|
||||
"sha256:7f55a602d55e8f0feafde533c69dfd29bf0e54645ab0996b605613cda6894a85",
|
||||
"sha256:8357aa727094798f1d831339ecfd8b3e388c01db6015a3cbd51790cb75e39994",
|
||||
"sha256:84dc6bfeaeba98fe93fc837b12f9af4842694cdbde18083f150e80aec3de88f9",
|
||||
"sha256:86b18420f00d5977bda477369ac85e04185ef94046a04ae0d85f5a807d1a8eb4",
|
||||
"sha256:89f32d8450e15b0c11efdc81e2704d68c502c889d48415a50add9fa031144f75",
|
||||
"sha256:8a1de8931cdad8cd12724e12a6167eef8cb478cc3ee5d2c9f4670c934f2975e1",
|
||||
"sha256:8f106468062ac7ff03e3522a66cb7b36c662326d8eb7af1be0f30563740ff002",
|
||||
"sha256:9a4ea87a0401c06b687db29e2ae836b2b58480ab118cb6eea8ac2ef45a4345f8",
|
||||
"sha256:9ee1b019a4640bf39c0705ab65e934cfe6b89f1a8dc26f389fae3d7c62358d6f",
|
||||
"sha256:a0d7c6d6fbca62508ea525abd869fca78ecf68cd3bcf6ae67ec478aa37cf39c0",
|
||||
"sha256:a1417cb339a367a5dfd0e50193a1c0e87e31325547a0e7624ee4ff414c0b53b3",
|
||||
"sha256:a35f1937b0560587d478fd2259a6d4f66cf511c9d28e90b52b183745eaa77d95",
|
||||
"sha256:a4a35e83abfdac7095430e1c1476e0871e4b234e936f4a7a7631531b09a4f198",
|
||||
"sha256:a7d1c8830a7bc10420ceb60a256d25ab5b032a6dad12a46af6ab2e470cee9124",
|
||||
"sha256:a938d4d5b530f8ea988afb80817209eabc150c53b8c7af79d40080313a35e470",
|
||||
"sha256:a9a2c377106fe01a57bad0f703653de286d56ee5285ed36c6953535cfa11f928",
|
||||
"sha256:baf7546afd27be4f96f23307d7c295497fb512875167743b14a7457b95761294",
|
||||
"sha256:bb21e2f35d6f09aa4a6df0c716f41e036cfcf05a98323b50294f93085ad775e9",
|
||||
"sha256:bc62ba37bcb42e4146b853940b65a2de31c2962d2b6da9bc3ce28270d13b5c4e",
|
||||
"sha256:be3ba736aabf856195199208ed37459408c932940cbccd2dc9f6ff2e800b0261",
|
||||
"sha256:c03eb43d15c8af58159e7561076634d565530aaacaf48cf4e070c3501e88a372",
|
||||
"sha256:c1349331fa743eed4042f9652200e60596f8beb957554acbcbb42aad4272c606",
|
||||
"sha256:c3637cfce519560e2a2579d05eb81e912d109283b8ddc8de46f57ec20d273d92",
|
||||
"sha256:c481cd1af2a77f58f495f7f87c2d715c6f1179d07c1ec927cca1f7977a2d99aa",
|
||||
"sha256:c575f9499e5f540e034ff87bef894f031ae613a98b0d1d3afcc1f482527d5f1c",
|
||||
"sha256:c604831daf2e7e5979ecd97a90cb8c4a7bae208ff45bc792e32eae09c3281afb",
|
||||
"sha256:c759e1e0333664831d8d1d6b26cf59f23f3707758f696c71f506504b33130f81",
|
||||
"sha256:c8a2743dd50629c0222f26c5f55975e45841d985b4b1c7a54b3f03b53de3427d",
|
||||
"sha256:cbcac9263f500da94405cc9fc7e7a42a3ba6c2fe88b2cd7039737cba44c66889",
|
||||
"sha256:cce1b7a680653e31ff2b252f19a39f1ded578a35a96c419ddb9632c62d2af7d8",
|
||||
"sha256:cf96799b3e5e2e2f6dbca015f72b28e7ae415ce8147472f89a3704a035d6336d",
|
||||
"sha256:d06ed18917dbc7a938c4231cbbec52a7e474be270b2ef9208abb4d5a34f5ceb9",
|
||||
"sha256:d4ba5b4f1a0334dbe673f767f28775744e793fcb9ea57a1d72bc622c9f90e6b4",
|
||||
"sha256:d7b8f25c9b0043cbaf77b8b895814e33e7a3c807a097377c07e1bd49946030d5",
|
||||
"sha256:d86511ef8217822fb8716460aaa1ece31fe9e8a48900e541cb35acb7c35e9e2e",
|
||||
"sha256:db8a9cbe965c7343feab2e2bf9a3771f303f8a7ca401dececb6ef28e06b3b18c",
|
||||
"sha256:dbe92a8808cefb284e235b8f82933d7d2e24ff929fe5d53f1fd3ca55fced4b58",
|
||||
"sha256:deb83cc9f639045e2febcc8d4306d4b83893af8d895f2ed70aa342a3430b534c",
|
||||
"sha256:df9084e06efb3d59608a6a443faa9861828585579f0ae8e95f5a4dab70f1a00f",
|
||||
"sha256:dfb89e92746e4a1e0d091cba73d6cc1e16b4094ebdbb14c2e96a80320feb1ad7",
|
||||
"sha256:e13ddfe2ead9540e8773cae098f54c5206d6fcef64846a3e5042db47fc3a41ed",
|
||||
"sha256:e4956384340eec7b526149ac126c8aa11d32441cb3ce77a690cb4821d0d0635c",
|
||||
"sha256:e6eecd027b6ba5617ea6af3e12e20d578d8f4ad1bf51a9abe69c6fd4835ea532",
|
||||
"sha256:eff9818b7671a55f1ce781398607e0d8c304cd430c0581fbe15b868a7a371c27",
|
||||
"sha256:f0aea377b9dfc166c8fa05bb158c30ee3d53d73f0ed2fc05ba6c638d9563422f",
|
||||
"sha256:f1fba193ab2f25849e24caa4570611aa2f80bc1c1ba791851523734b4ed69e43",
|
||||
"sha256:f6db4f00d3baad615e99a865539391243d12b113fb628ebda1d7794ce02d5a10",
|
||||
"sha256:f9405c02af86850e0a8a8ba777b7e7609e0d07bff46adc4f78892cc2d5456018",
|
||||
"sha256:fb4445e3721720c5ca14c0650f35c263b3430e6e16df9d2504618df914b3fb99"
|
||||
"sha256:076afa0a4a96ca9f77fec0e4a0d241200b3b3a1766f8d7be9a905ecf59a7416b",
|
||||
"sha256:08819da7864f9b8d4a95729b2bea5fffed08b63d3b9c15b4fea47de655766cf5",
|
||||
"sha256:0a1f26bc1f5ce774d99725773901820dfdfd24e875028da4a0252a5b48dcab5c",
|
||||
"sha256:0f4b125b46fe377984fbaecf2af40ed48b05a4b7676a2ff98999f2016d66b3ec",
|
||||
"sha256:1240edc1a448d4ada4bf1a0e55550b6292420915292408e59159fd8bbdaf8f63",
|
||||
"sha256:152259f0f1a60f560323aacf463a3642a65a25557683f49cfa08c8f1ecb2395a",
|
||||
"sha256:168172ef7856e20ec024fe2a746bfa895c88b32720138e6438fd765ebd2b62dd",
|
||||
"sha256:1b1d7d9aabd8629a31d63cd106d56cca0e6420f38e50563278b520f385c0d86e",
|
||||
"sha256:1d40ad09d9f5e719bc6f729cc6b17f31c0b055029719406bd31dde2f72fca7e7",
|
||||
"sha256:21b953da14549ff62ea4ae20889c71564328958cbdf880c64a92a48dda4c9c53",
|
||||
"sha256:23cc6d7eb009c688d70da186b8f362d61d5dd1a2c14a45b890bd1e91e9c451f2",
|
||||
"sha256:2988ef5e6b360b3ff1c6d55c53515499de5f48df31afd9f785d788cdacfbe2d3",
|
||||
"sha256:2a0aade2b11dc0c326ccd429ee4134d2d47459ff68d449c6d7e01e74651bd255",
|
||||
"sha256:2b0176f9233a5927084c79ff80b51bd70bfd57e4f3d564f50f80238e797f0c8a",
|
||||
"sha256:2d4fa1b01fa7e5b7bb8d312e3542e211b320eb7a4e3d8dc884327039d93cb9e0",
|
||||
"sha256:3236cf89d69679eaeb9119c840f5c7eb388a2110b57af6bb6baf01a1da387c18",
|
||||
"sha256:33faa786cc907de63f745f587e9879429b46033d7d97a7b84b37f4f8f47b9b32",
|
||||
"sha256:37df8f6006286a5896d1cbc3efb8471ced42e3568d38e6cb00857277047b0d63",
|
||||
"sha256:3a7166d57dc74d679caa7743b8ecf7dc3a1235a9fd178654dddb2b2a627ae229",
|
||||
"sha256:3d79ae3bb1ff041c0db56f138c88ce1dfb0209f3546d8d6e7c3f74944ecd2439",
|
||||
"sha256:3e33064f1984db412b34d51496f4ea785a9cff621c67de58e09fb28da6468a52",
|
||||
"sha256:3fa3648e4f1e63ddfe53563ee111079ea3ab35c3b09cd25bc22dadc8269a495f",
|
||||
"sha256:40d5f6e853ece9bfc01e9129b228df446f49316a4252bb1fbfae5c3c9dedebad",
|
||||
"sha256:41771b22dd2822540f79a877c391283d4e6368125999a5ec8beee1ce566f3f82",
|
||||
"sha256:435228d3c16a375274ac8ab9c4f9aef40c5e57ddb8296e20ecec9e2461da1017",
|
||||
"sha256:44ee985194c426ddf781fa784f31ffa29cb59657b2dba09250a4245431847d73",
|
||||
"sha256:465fd5b040206f8bce7016b01d7e7f79d2fcd7c2b8e41791be9632a9df1b4999",
|
||||
"sha256:496c9cbcb4951183d4503a9d7d2c1e3694aab1304262f831d5e1917e60386036",
|
||||
"sha256:49dce6957598975d8b8d506329d2a3a6c4aee911fa4bbcf5e52ffc6897122950",
|
||||
"sha256:4c42748ccc451dfcd9cef6c5447a7ab727351fd9747ad431db5ebb18a9b78a4d",
|
||||
"sha256:505f8519c4c782a61d94a17b0da50be639ec462128fbd10ab0a34889218fdee3",
|
||||
"sha256:53f2dda54d76a98b43a410498bd12f6034b2a14b6844ca08513733b2b20b7ad8",
|
||||
"sha256:56320c401f544d762fc35766936178fbceb1d9261cd7b24fbfbc8fb6f67aa8a5",
|
||||
"sha256:58a63a26a1e3dc481dd3a18d6d9f8bd1d576cd1ffe0d479ba7dd38b0aeb20066",
|
||||
"sha256:5caee7bd08c3d36ec54617832b44985bd70c4cbd77c5b313de6f7fce0bb34f93",
|
||||
"sha256:631492573a1bef2f74f9ac0f9d84e0ce422c251644cd81207530af4aa2ee1980",
|
||||
"sha256:63d8019eee119df308a075b8a7bdb06d4720bf791e2b73d5ab0e7473c115d79c",
|
||||
"sha256:6422b6763b016f2ef2beedded0e546d6aa6ba87910f9244d86e0ac7690f75c96",
|
||||
"sha256:681f252e43b3ef054ca9161635f81b730f4d8cadd28b3f2b2004f5a72f853982",
|
||||
"sha256:6d64878d1659d2a5bdfd0f0a4d79bafe68653c573681495e424ab40d7b6d6d41",
|
||||
"sha256:74c0da07c04d0781490b2915e7514b1adb265ef22af039a947988c331ee7455b",
|
||||
"sha256:7591a3beea6a9a4fa3080d27d193b41f631130e3ffa76b88c9ccea123f26dc59",
|
||||
"sha256:76a262c41c1a7cbb84a3b11976578a7eb8e788c4b7bfbd15c005fb6ca88e6e50",
|
||||
"sha256:77cfff95c1fafd09e940b3fdcb7b65f11442662fad611d0e69b4dd5d17a81c60",
|
||||
"sha256:8027c9063579083746147cf401a7072a9fb6829678076cd3deff28bb0e0f50c8",
|
||||
"sha256:80a167081c75cf66b32f30e2f1eaee9365af935a86dbd76788169911bed9b5d5",
|
||||
"sha256:840eaf30ccac122df260b6005f9dfae4ac287c498ee91e3e90c56781614ca238",
|
||||
"sha256:8543253adfaa0b802bfa88386db1009c6ebb7d5684d093ee4edc725007553d21",
|
||||
"sha256:89b3f2da57a27913d15d2a07d58482f33d0a5b28abd20b8e643ab4d625e36257",
|
||||
"sha256:8e559116e4128630ad3b7e788e2e5da81cbc2344dee246af44471fa650486a70",
|
||||
"sha256:9aff6279e405dc953eeb540ab061e72c03cf38119613fce183a8e94f31be608f",
|
||||
"sha256:9c04b9560872fa9a91251030c488e0a73bce9321a70f991f830c72b3f8115d0d",
|
||||
"sha256:9d2346b00af524757576cc2406414562cced1d4349c92166a0ee377a2a483a80",
|
||||
"sha256:a253b765b7cbc4209f1d8ee16c7287c4268d3243070bf72d7eec5aa9dfe2a2c2",
|
||||
"sha256:a8127437ebc196a6f5e8fddd746bd0903a400dc6b5ae35df672dd1ccc7170a2a",
|
||||
"sha256:b25f7bea162b3dbec6d33c522097ef81df7c19a9300722fa6853f5b495aecb77",
|
||||
"sha256:b33c17d9e694b66d7e96977e9e56df19d662031483efe121a24772a44ccbbc7e",
|
||||
"sha256:b4fe46b58010115514b842c669a0ed9b6a342017b15905653a5b1724ab80917f",
|
||||
"sha256:b520aafc6cb148bac09ccf532f52cbd31d83acf4d3e5070d84efe3c019a1adbf",
|
||||
"sha256:b5bbb87fa0511bd313d9a2c90294c88db837667c2bda2ea3fa7a35b59fd93b1f",
|
||||
"sha256:b6d2a56fc2354bb6378f3634402eec788a8f3facf0b3e7d468db5f2b5a78d763",
|
||||
"sha256:bbd705d5f3c3d1ff2d169e418bb789ff07ab3c70d567cc6ba6b72b04b9143481",
|
||||
"sha256:bc5d8c3647b8ae28e4312f1492b8f29deebd31479cd3abaa989090fb1d66db83",
|
||||
"sha256:c3c3525ea8658ee1192cdddf5faf99b07ebe1eeaa61bf32821126df6d1b8072b",
|
||||
"sha256:c9a9a39b7cac81dca79fca8c2a6479ef4c7b1aab95fad7544cc0e8fd943595a2",
|
||||
"sha256:cd4c8d6aa91d3e35016847cbe8d73106e3d1c9a4e6578d38e2c346bfe8edb3ca",
|
||||
"sha256:cf62da7a4cdec9a4b2981fcbd5e08053edffccf20e845c0b6ec1e77eb7fab61d",
|
||||
"sha256:d67225f05f6ea27c8dc57f3fa6397c96d09c42af69d46629f71e82e66d33fa4f",
|
||||
"sha256:dfcd2b9f510411de615ccedd47462dae80e82fdc09fe9ab0f0f32f11cf57eeb5",
|
||||
"sha256:e1f61355c821e870fb4c17cdb318669cfbcf245a291ce5053b41140870c3e5cc",
|
||||
"sha256:e249190b018d63c901678053b4a43e797ca78b93fb6d17633e3567d4b3ec6107",
|
||||
"sha256:e2654d1278384cff75952682d17c718ecc1ad1d6227bb0068fd826ba47d426a5",
|
||||
"sha256:e57d859b972c75ee44ea2ef4758f12821243e99de814030f69a3decb2aa86807",
|
||||
"sha256:e5a27f348909235a106a3903fc8e70f573d89b41d723a500869c6569a391cff7",
|
||||
"sha256:ead4f19d0257a756b21ac2e0e85a37a7245ddec36d3b6008d5bfe416525967dc",
|
||||
"sha256:f076b779aa3dc179aa3ed861be063a313ed4e48ae9f6a8370a9b1295d4502111",
|
||||
"sha256:f1bb3a62395ffe835dbef3a1cbff48fbcce709c78bd1f52e896aee990928432b",
|
||||
"sha256:f2227a08b091bd41df5aadee0a5037673f691e2aa000e1968b1ea2342afc6880",
|
||||
"sha256:f3754acbd7efc7f1b529039fcffc092a15e1cf045e31f22f6c9c5950c613ec4d",
|
||||
"sha256:fe48f50fb6348511a3268a893bfd4ab5f263f5ac220782449d03cd05964d1ae7",
|
||||
"sha256:fff7d17d30b2cd45afd654b3fc117755c5d84506ed25fda386494e4e0a3416e1"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==4.1.1"
|
||||
"version": "==4.5.0"
|
||||
},
|
||||
"requests": {
|
||||
"hashes": [
|
||||
"sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983",
|
||||
"sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349"
|
||||
"sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f",
|
||||
"sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.28.1"
|
||||
"version": "==2.31.0"
|
||||
},
|
||||
"scipy": {
|
||||
"hashes": [
|
||||
"sha256:02b567e722d62bddd4ac253dafb01ce7ed8742cf8031aea030a41414b86c1125",
|
||||
"sha256:1166514aa3bbf04cb5941027c6e294a000bba0cf00f5cdac6c77f2dad479b434",
|
||||
"sha256:1da52b45ce1a24a4a22db6c157c38b39885a990a566748fc904ec9f03ed8c6ba",
|
||||
"sha256:23b22fbeef3807966ea42d8163322366dd89da9bebdc075da7034cee3a1441ca",
|
||||
"sha256:28d2cab0c6ac5aa131cc5071a3a1d8e1366dad82288d9ec2ca44df78fb50e649",
|
||||
"sha256:2ef0fbc8bcf102c1998c1f16f15befe7cffba90895d6e84861cd6c6a33fb54f6",
|
||||
"sha256:3b69b90c9419884efeffaac2c38376d6ef566e6e730a231e15722b0ab58f0328",
|
||||
"sha256:4b93ec6f4c3c4d041b26b5f179a6aab8f5045423117ae7a45ba9710301d7e462",
|
||||
"sha256:4e53a55f6a4f22de01ffe1d2f016e30adedb67a699a310cdcac312806807ca81",
|
||||
"sha256:6311e3ae9cc75f77c33076cb2794fb0606f14c8f1b1c9ff8ce6005ba2c283621",
|
||||
"sha256:65b77f20202599c51eb2771d11a6b899b97989159b7975e9b5259594f1d35ef4",
|
||||
"sha256:6cc6b33139eb63f30725d5f7fa175763dc2df6a8f38ddf8df971f7c345b652dc",
|
||||
"sha256:70de2f11bf64ca9921fda018864c78af7147025e467ce9f4a11bc877266900a6",
|
||||
"sha256:70ebc84134cf0c504ce6a5f12d6db92cb2a8a53a49437a6bb4edca0bc101f11c",
|
||||
"sha256:83606129247e7610b58d0e1e93d2c5133959e9cf93555d3c27e536892f1ba1f2",
|
||||
"sha256:93d07494a8900d55492401917a119948ed330b8c3f1d700e0b904a578f10ead4",
|
||||
"sha256:9c4e3ae8a716c8b3151e16c05edb1daf4cb4d866caa385e861556aff41300c14",
|
||||
"sha256:9dd4012ac599a1e7eb63c114d1eee1bcfc6dc75a29b589ff0ad0bb3d9412034f",
|
||||
"sha256:9e3fb1b0e896f14a85aa9a28d5f755daaeeb54c897b746df7a55ccb02b340f33",
|
||||
"sha256:a0aa8220b89b2e3748a2836fbfa116194378910f1a6e78e4675a095bcd2c762d",
|
||||
"sha256:d3b3c8924252caaffc54d4a99f1360aeec001e61267595561089f8b5900821bb",
|
||||
"sha256:e013aed00ed776d790be4cb32826adb72799c61e318676172495383ba4570aa4",
|
||||
"sha256:f3e7a8867f307e3359cc0ed2c63b61a1e33a19080f92fe377bc7d49f646f2ec1"
|
||||
"sha256:0f3261f14b767b316d7137c66cc4f33a80ea05841b9c87ad83a726205b901423",
|
||||
"sha256:10eb6af2f751aa3424762948e5352f707b0dece77288206f227864ddf675aca0",
|
||||
"sha256:1342ca385c673208f32472830c10110a9dcd053cf0c4b7d4cd7026d0335a6c1d",
|
||||
"sha256:214cdf04bbae7a54784f8431f976704ed607c4bc69ba0d5d5d6a9df84374df76",
|
||||
"sha256:2b997a5369e2d30c97995dcb29d638701f8000d04df01b8e947f206e5d0ac788",
|
||||
"sha256:2c91cf049ffb5575917f2a01da1da082fd24ed48120d08a6e7297dfcac771dcd",
|
||||
"sha256:3aeb87661de987f8ec56fa6950863994cd427209158255a389fc5aea51fa7055",
|
||||
"sha256:4447ad057d7597476f9862ecbd9285bbf13ba9d73ce25acfa4e4b11c6801b4c9",
|
||||
"sha256:542a757e2a6ec409e71df3d8fd20127afbbacb1c07990cb23c5870c13953d899",
|
||||
"sha256:8d9886f44ef8c9e776cb7527fb01455bf4f4a46c455c4682edc2c2cc8cd78562",
|
||||
"sha256:90d3b1364e751d8214e325c371f0ee0dd38419268bf4888b2ae1040a6b266b2a",
|
||||
"sha256:95763fbda1206bec41157582bea482f50eb3702c85fffcf6d24394b071c0e87a",
|
||||
"sha256:ac74b1512d38718fb6a491c439aa7b3605b96b1ed3be6599c17d49d6c60fca18",
|
||||
"sha256:afdb0d983f6135d50770dd979df50bf1c7f58b5b33e0eb8cf5c73c70600eae1d",
|
||||
"sha256:b0620240ef445b5ddde52460e6bc3483b7c9c750275369379e5f609a1050911c",
|
||||
"sha256:b133f237bd8ba73bad51bc12eb4f2d84cbec999753bf25ba58235e9fc2096d80",
|
||||
"sha256:b29318a5e39bd200ca4381d80b065cdf3076c7d7281c5e36569e99273867f61d",
|
||||
"sha256:b8425fa963a32936c9773ee3ce44a765d8ff67eed5f4ac81dc1e4a819a238ee9",
|
||||
"sha256:d2b813bfbe8dec6a75164523de650bad41f4405d35b0fa24c2c28ae07fcefb20",
|
||||
"sha256:d690e1ca993c8f7ede6d22e5637541217fc6a4d3f78b3672a6fe454dbb7eb9a7",
|
||||
"sha256:e367904a0fec76433bf3fbf3e85bf60dae8e9e585ffd21898ab1085a29a04d16",
|
||||
"sha256:ea932570b1c2a30edafca922345854ff2cd20d43cd9123b6dacfdecebfc1a80b",
|
||||
"sha256:f28f1f6cfeb48339c192efc6275749b2a25a7e49c4d8369a28b6591da02fbc9a",
|
||||
"sha256:f73102f769ee06041a3aa26b5841359b1a93cc364ce45609657751795e8f4a4a",
|
||||
"sha256:fa4909c6c20c3d91480533cddbc0e7c6d849e7d9ded692918c76ce5964997898"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==1.8.1"
|
||||
"version": "==1.11.2"
|
||||
},
|
||||
"sortedcontainers": {
|
||||
"hashes": [
|
||||
"sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88",
|
||||
"sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==2.4.0"
|
||||
},
|
||||
"update-checker": {
|
||||
"hashes": [
|
||||
"sha256:6a2d45bb4ac585884a6b03f9eade9161cedd9e8111545141e9aa9058932acb13",
|
||||
"sha256:cbba64760a36fe2640d80d85306e8fe82b6816659190993b7bdabadee4d4bbfd"
|
||||
],
|
||||
"version": "==0.18.0"
|
||||
},
|
||||
"urllib3": {
|
||||
"hashes": [
|
||||
"sha256:8298d6d56d39be0e3bc13c1c97d133f9b45d797169a0e11cdd0e0489d786f7ec",
|
||||
"sha256:879ba4d1e89654d9769ce13121e0f94310ea32e8d2f8cf587b77c08bbcdb30d6"
|
||||
"sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11",
|
||||
"sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4"
|
||||
],
|
||||
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5' and python_version < '4'",
|
||||
"version": "==1.26.10"
|
||||
"markers": "python_version >= '3.7'",
|
||||
"version": "==2.0.4"
|
||||
},
|
||||
"websocket-client": {
|
||||
"hashes": [
|
||||
"sha256:53e95c826bf800c4c465f50093a8c4ff091c7327023b10bfaff40cf1ef170eaa",
|
||||
"sha256:ce54f419dfae71f4bdba69ebe65bf7f0a93fe71bc009ad3a010aacc3eebad537"
|
||||
],
|
||||
"markers": "python_version >= '3.8'",
|
||||
"version": "==1.6.2"
|
||||
},
|
||||
"zstandard": {
|
||||
"hashes": [
|
||||
"sha256:083dc08abf03807af9beeb2b6a91c23ad78add2499f828176a3c7b742c44df02",
|
||||
"sha256:0ac0357a0d985b4ff31a854744040d7b5754385d1f98f7145c30e02c6865cb6f",
|
||||
"sha256:19cac7108ff2c342317fad6dc97604b47a41f403c8f19d0bfc396dfadc3638b8",
|
||||
"sha256:1af1268a7dc870eb27515fb8db1f3e6c5a555d2b7bcc476fc3bab8886c7265ab",
|
||||
"sha256:1be31e9e3f7607ee0cdd60915410a5968b205d3e7aa83b7fcf3dd76dbbdb39e0",
|
||||
"sha256:1dc2d3809e763055a1a6c1a73f2b677320cc9a5aa1a7c6cfb35aee59bddc42d9",
|
||||
"sha256:266aba27fa9cc5e9091d3d325ebab1fa260f64e83e42516d5e73947c70216a5b",
|
||||
"sha256:28723a1d2e4df778573b76b321ebe9f3469ac98988104c2af116dd344802c3f8",
|
||||
"sha256:2dc466207016564805e56d28375f4f533b525ff50d6776946980dff5465566ac",
|
||||
"sha256:39e98cf4773234bd9cebf9f9db730e451dfcfe435e220f8921242afda8321887",
|
||||
"sha256:3af8c2383d02feb6650e9255491ec7d0824f6e6dd2bbe3e521c469c985f31fb1",
|
||||
"sha256:46f679bc5dfd938db4fb058218d9dc4db1336ffaf1ea774ff152ecadabd40805",
|
||||
"sha256:490d11b705b8ae9dc845431bacc8dd1cef2408aede176620a5cd0cd411027936",
|
||||
"sha256:49685bf9a55d1ab34bd8423ea22db836ba43a181ac6b045ac4272093d5cb874e",
|
||||
"sha256:4a2ee1d4f98447f3e5183ecfce5626f983504a4a0c005fbe92e60fa8e5d547ec",
|
||||
"sha256:4cbb85f29a990c2fdbf7bc63246567061a362ddca886d7fae6f780267c0a9e67",
|
||||
"sha256:5228e596eb1554598c872a337bbe4e5afe41cd1f8b1b15f2e35b50d061e35244",
|
||||
"sha256:533db8a6fac6248b2cb2c935e7b92f994efbdeb72e1ffa0b354432e087bb5a3e",
|
||||
"sha256:63694a376cde0aa8b1971d06ca28e8f8b5f492779cb6ee1cc46bbc3f019a42a5",
|
||||
"sha256:702a8324cd90c74d9c8780d02bf55e79da3193c870c9665ad3a11647e3ad1435",
|
||||
"sha256:7231543d38d2b7e02ef7cc78ef7ffd86419437e1114ff08709fe25a160e24bd6",
|
||||
"sha256:75479e7c2b3eebf402c59fbe57d21bc400cefa145ca356ee053b0a08908c5784",
|
||||
"sha256:76725d1ee83a8915100a310bbad5d9c1fc6397410259c94033b8318d548d9990",
|
||||
"sha256:8677ffc6a6096cccbd892e558471c901fd821aba12b7fbc63833c7346f549224",
|
||||
"sha256:8b2260c4e07dd0723eadb586de7718b61acca4083a490dda69c5719d79bc715c",
|
||||
"sha256:999a4e1768f219826ba3fa2064fab1c86dd72fdd47a42536235478c3bb3ca3e2",
|
||||
"sha256:9df59cd1cf3c62075ee2a4da767089d19d874ac3ad42b04a71a167e91b384722",
|
||||
"sha256:a7fa67cba473623848b6e88acf8d799b1906178fd883fb3a1da24561c779593b",
|
||||
"sha256:bd3220d7627fd4d26397211cb3b560ec7cc4a94b75cfce89e847e8ce7fabe32d",
|
||||
"sha256:bfa6c8549fa18e6497a738b7033c49f94a8e2e30c5fbe2d14d0b5aa8bbc1695d",
|
||||
"sha256:c86befac87445927488f5c8f205d11566f64c11519db223e9d282b945fa60dab",
|
||||
"sha256:c990063664c08169c84474acecc9251ee035871589025cac47c060ff4ec4bc1a",
|
||||
"sha256:cdb44d7284c8c5dd1b66dfb86dda7f4560fa94bfbbc1d2da749ba44831335e32",
|
||||
"sha256:ce6f59cba9854fd14da5bfe34217a1501143057313966637b7291d1b0267bd1e",
|
||||
"sha256:d4a8fd45746a6c31e729f35196e80b8f1e9987c59f5ccb8859d7c6a6fbeb9c63",
|
||||
"sha256:d6c85ca5162049ede475b7ec98e87f9390501d44a3d6776ddd504e872464ec25",
|
||||
"sha256:d716a7694ce1fa60b20bc10f35c4a22be446ef7f514c8dbc8f858b61976de2fb",
|
||||
"sha256:d85bfabad444812133a92fc6fbe463e1d07581dba72f041f07a360e63808b23c",
|
||||
"sha256:d956e2f03c7200d7e61345e0880c292783ec26618d0d921dcad470cb195bbce2",
|
||||
"sha256:dbb3cb8a082d62b8a73af42291569d266b05605e017a3d8a06a0e5c30b5f10f0",
|
||||
"sha256:dc2a4de9f363b3247d472362a65041fe4c0f59e01a2846b15d13046be866a885",
|
||||
"sha256:e02043297c1832f2666cd2204f381bef43b10d56929e13c42c10c732c6e3b4ed",
|
||||
"sha256:eea18c1e7442f2aa9aff1bb84550dbb6a1f711faf6e48e7319de8f2b2e923c2a",
|
||||
"sha256:ef7e8a200e4c8ac9102ed3c90ed2aa379f6b880f63032200909c1be21951f556"
|
||||
"sha256:0aad6090ac164a9d237d096c8af241b8dcd015524ac6dbec1330092dba151657",
|
||||
"sha256:0bdbe350691dec3078b187b8304e6a9c4d9db3eb2d50ab5b1d748533e746d099",
|
||||
"sha256:0e1e94a9d9e35dc04bf90055e914077c80b1e0c15454cc5419e82529d3e70728",
|
||||
"sha256:1243b01fb7926a5a0417120c57d4c28b25a0200284af0525fddba812d575f605",
|
||||
"sha256:144a4fe4be2e747bf9c646deab212666e39048faa4372abb6a250dab0f347a29",
|
||||
"sha256:14e10ed461e4807471075d4b7a2af51f5234c8f1e2a0c1d37d5ca49aaaad49e8",
|
||||
"sha256:1545fb9cb93e043351d0cb2ee73fa0ab32e61298968667bb924aac166278c3fc",
|
||||
"sha256:1e6e131a4df2eb6f64961cea6f979cdff22d6e0d5516feb0d09492c8fd36f3bc",
|
||||
"sha256:25fbfef672ad798afab12e8fd204d122fca3bc8e2dcb0a2ba73bf0a0ac0f5f07",
|
||||
"sha256:2769730c13638e08b7a983b32cb67775650024632cd0476bf1ba0e6360f5ac7d",
|
||||
"sha256:48b6233b5c4cacb7afb0ee6b4f91820afbb6c0e3ae0fa10abbc20000acdf4f11",
|
||||
"sha256:4af612c96599b17e4930fe58bffd6514e6c25509d120f4eae6031b7595912f85",
|
||||
"sha256:52b2b5e3e7670bd25835e0e0730a236f2b0df87672d99d3bf4bf87248aa659fb",
|
||||
"sha256:57ac078ad7333c9db7a74804684099c4c77f98971c151cee18d17a12649bc25c",
|
||||
"sha256:62957069a7c2626ae80023998757e27bd28d933b165c487ab6f83ad3337f773d",
|
||||
"sha256:649a67643257e3b2cff1c0a73130609679a5673bf389564bc6d4b164d822a7ce",
|
||||
"sha256:67829fdb82e7393ca68e543894cd0581a79243cc4ec74a836c305c70a5943f07",
|
||||
"sha256:7d3bc4de588b987f3934ca79140e226785d7b5e47e31756761e48644a45a6766",
|
||||
"sha256:7f2afab2c727b6a3d466faee6974a7dad0d9991241c498e7317e5ccf53dbc766",
|
||||
"sha256:8070c1cdb4587a8aa038638acda3bd97c43c59e1e31705f2766d5576b329e97c",
|
||||
"sha256:8257752b97134477fb4e413529edaa04fc0457361d304c1319573de00ba796b1",
|
||||
"sha256:9980489f066a391c5572bc7dc471e903fb134e0b0001ea9b1d3eff85af0a6f1b",
|
||||
"sha256:9cff89a036c639a6a9299bf19e16bfb9ac7def9a7634c52c257166db09d950e7",
|
||||
"sha256:a8d200617d5c876221304b0e3fe43307adde291b4a897e7b0617a61611dfff6a",
|
||||
"sha256:a9fec02ce2b38e8b2e86079ff0b912445495e8ab0b137f9c0505f88ad0d61296",
|
||||
"sha256:b1367da0dde8ae5040ef0413fb57b5baeac39d8931c70536d5f013b11d3fc3a5",
|
||||
"sha256:b69cccd06a4a0a1d9fb3ec9a97600055cf03030ed7048d4bcb88c574f7895773",
|
||||
"sha256:b72060402524ab91e075881f6b6b3f37ab715663313030d0ce983da44960a86f",
|
||||
"sha256:c053b7c4cbf71cc26808ed67ae955836232f7638444d709bfc302d3e499364fa",
|
||||
"sha256:cff891e37b167bc477f35562cda1248acc115dbafbea4f3af54ec70821090965",
|
||||
"sha256:d12fa383e315b62630bd407477d750ec96a0f438447d0e6e496ab67b8b451d39",
|
||||
"sha256:d2d61675b2a73edcef5e327e38eb62bdfc89009960f0e3991eae5cc3d54718de",
|
||||
"sha256:db62cbe7a965e68ad2217a056107cc43d41764c66c895be05cf9c8b19578ce9c",
|
||||
"sha256:ddb086ea3b915e50f6604be93f4f64f168d3fc3cef3585bb9a375d5834392d4f",
|
||||
"sha256:df28aa5c241f59a7ab524f8ad8bb75d9a23f7ed9d501b0fed6d40ec3064784e8",
|
||||
"sha256:e1e0c62a67ff425927898cf43da2cf6b852289ebcc2054514ea9bf121bec10a5",
|
||||
"sha256:e6048a287f8d2d6e8bc67f6b42a766c61923641dd4022b7fd3f7439e17ba5a4d",
|
||||
"sha256:e7d560ce14fd209db6adacce8908244503a009c6c39eee0c10f138996cd66d3e",
|
||||
"sha256:ea68b1ba4f9678ac3d3e370d96442a6332d431e5050223626bdce748692226ea",
|
||||
"sha256:f08e3a10d01a247877e4cb61a82a319ea746c356a3786558bed2481e6c405546",
|
||||
"sha256:f1b9703fe2e6b6811886c44052647df7c37478af1b4a1a9078585806f42e5b15",
|
||||
"sha256:fe6c821eb6870f81d73bf10e5deed80edcac1e63fbc40610e61f340723fd5f7c",
|
||||
"sha256:ff0852da2abe86326b20abae912d0367878dd0854b8931897d44cfeb18985472"
|
||||
],
|
||||
"index": "pypi",
|
||||
"version": "==0.18.0"
|
||||
"version": "==0.21.0"
|
||||
}
|
||||
},
|
||||
"develop": {}
|
||||
|
|
|
|||
|
|
@ -1,12 +1,12 @@
|
|||
import sys
|
||||
|
||||
import requests
|
||||
import time
|
||||
import discord_logging
|
||||
import argparse
|
||||
import os
|
||||
import re
|
||||
import zstandard
|
||||
from datetime import datetime, timedelta
|
||||
from collections import defaultdict
|
||||
import json
|
||||
import praw
|
||||
from praw import endpoints
|
||||
|
|
@ -16,13 +16,35 @@ log = discord_logging.init_logging(debug=False)
|
|||
import utils
|
||||
import classes
|
||||
from classes import IngestType
|
||||
from merge import ObjectType
|
||||
|
||||
|
||||
NEWLINE_ENCODED = "\n".encode('utf-8')
|
||||
reg = re.compile(r"\d\d-\d\d-\d\d_\d\d-\d\d")
|
||||
|
||||
|
||||
def query_pushshift(ids, bearer, object_type):
|
||||
object_name = "comment" if object_type == ObjectType.COMMENT else "submission"
|
||||
url = f"https://api.pushshift.io/reddit/{object_name}/search?limit=1000&ids={','.join(ids)}"
|
||||
log.debug(f"pushshift query: {url}")
|
||||
response = None
|
||||
for i in range(4):
|
||||
response = requests.get(url, headers={
|
||||
'User-Agent': "In script by /u/Watchful1",
|
||||
'Authorization': f"Bearer {bearer}"})
|
||||
if response.status_code == 200:
|
||||
break
|
||||
if response.status_code == 403:
|
||||
log.warning(f"Pushshift unauthorized, aborting")
|
||||
sys.exit(2)
|
||||
time.sleep(2)
|
||||
if response.status_code != 200:
|
||||
log.warning(f"4 requests failed with status code {response.status_code}")
|
||||
return response.json()['data']
|
||||
|
||||
|
||||
def build_day(day_to_process, input_folders, output_folder, object_type, reddit, pushshift_token):
|
||||
file_type = "comments" if object_type == utils.ObjectType.COMMENT else "submissions"
|
||||
file_type = "comments" if object_type == ObjectType.COMMENT else "submissions"
|
||||
|
||||
file_minutes = {}
|
||||
minute_iterator = day_to_process - timedelta(minutes=2)
|
||||
|
|
@ -72,7 +94,7 @@ def build_day(day_to_process, input_folders, output_folder, object_type, reddit,
|
|||
f"{working_highest_minute.strftime('%y-%m-%d_%H-%M')} with {len(missing_ids)} ids")
|
||||
|
||||
for chunk in utils.chunk_list(missing_ids, 50):
|
||||
pushshift_objects = utils.query_pushshift(chunk, pushshift_token, object_type)
|
||||
pushshift_objects = query_pushshift(chunk, pushshift_token, object_type)
|
||||
for pushshift_object in pushshift_objects:
|
||||
if objects.add_object(pushshift_object, IngestType.PUSHSHIFT):
|
||||
unmatched_field = True
|
||||
|
|
@ -137,9 +159,9 @@ if __name__ == "__main__":
|
|||
|
||||
object_type = None
|
||||
if args.type == "comments":
|
||||
object_type = utils.ObjectType.COMMENT
|
||||
object_type = ObjectType.COMMENT
|
||||
elif args.type == "submissions":
|
||||
object_type = utils.ObjectType.SUBMISSION
|
||||
object_type = ObjectType.SUBMISSION
|
||||
else:
|
||||
log.error(f"Invalid type: {args.type}")
|
||||
sys.exit(2)
|
||||
|
|
@ -160,6 +182,3 @@ if __name__ == "__main__":
|
|||
start_date = start_date + timedelta(days=1)
|
||||
|
||||
#log.info(f"{len(file_minutes)} : {count_ingest_minutes} : {count_rescan_minutes} : {day_highest_id - day_lowest_id:,} - {count_objects:,} = {(day_highest_id - day_lowest_id) - count_objects:,}: {utils.base36encode(day_lowest_id)}-{utils.base36encode(day_highest_id)}")
|
||||
|
||||
# minute_iterator = datetime.strptime("23-07-10 23:30", '%y-%m-%d %H:%M')
|
||||
# working_lowest_minute = datetime.strptime("23-07-10 23:30", '%y-%m-%d %H:%M')
|
||||
|
|
@ -1,85 +1,10 @@
|
|||
import re
|
||||
import sys
|
||||
from enum import Enum
|
||||
import discord_logging
|
||||
import zstandard
|
||||
import json
|
||||
from datetime import datetime
|
||||
import requests
|
||||
import time
|
||||
import discord_logging
|
||||
|
||||
import counters
|
||||
|
||||
log = discord_logging.get_logger(init=True)
|
||||
|
||||
|
||||
def parse_ingest_string(ingest_string):
|
||||
ingest_ids = []
|
||||
for char in ingest_string:
|
||||
ingest_ids.append(char)
|
||||
return ingest_ids
|
||||
|
||||
|
||||
def read_obj_zst(file_name):
|
||||
with open(file_name, 'rb') as file_handle:
|
||||
buffer = ''
|
||||
reader = zstandard.ZstdDecompressor(max_window_size=2**31).stream_reader(file_handle)
|
||||
while True:
|
||||
chunk = read_and_decode(reader, 2**27, (2**29) * 2)
|
||||
|
||||
if not chunk:
|
||||
break
|
||||
lines = (buffer + chunk).split("\n")
|
||||
|
||||
for line in lines[:-1]:
|
||||
if line == "":
|
||||
continue
|
||||
yield json.loads(line.strip())
|
||||
|
||||
buffer = lines[-1]
|
||||
|
||||
reader.close()
|
||||
|
||||
|
||||
def read_and_decode(reader, chunk_size, max_window_size, previous_chunk=None, bytes_read=0):
|
||||
chunk = reader.read(chunk_size)
|
||||
bytes_read += chunk_size
|
||||
if previous_chunk is not None:
|
||||
chunk = previous_chunk + chunk
|
||||
try:
|
||||
return chunk.decode()
|
||||
except UnicodeDecodeError:
|
||||
if bytes_read > max_window_size:
|
||||
raise UnicodeError(f"Unable to decode frame after reading {bytes_read:,} bytes")
|
||||
return read_and_decode(reader, chunk_size, max_window_size, chunk, bytes_read)
|
||||
|
||||
|
||||
def base36encode(integer: int) -> str:
|
||||
chars = '0123456789abcdefghijklmnopqrstuvwxyz'
|
||||
sign = '-' if integer < 0 else ''
|
||||
integer = abs(integer)
|
||||
result = ''
|
||||
while integer > 0:
|
||||
integer, remainder = divmod(integer, 36)
|
||||
result = chars[remainder] + result
|
||||
return sign + result
|
||||
|
||||
|
||||
def base36decode(base36: str) -> int:
|
||||
return int(base36, 36)
|
||||
|
||||
|
||||
def next_string_id(string_id):
|
||||
return base36encode(base36decode(string_id) + 1)
|
||||
|
||||
|
||||
def get_next_hundred_ids(start_id):
|
||||
start_num = base36decode(start_id)
|
||||
ids = []
|
||||
id_num = -1
|
||||
for id_num in range(start_num, start_num + 100):
|
||||
ids.append(base36encode(id_num))
|
||||
return ids, base36encode(id_num)
|
||||
log = discord_logging.get_logger()
|
||||
|
||||
|
||||
class FieldAction(Enum):
|
||||
|
|
@ -496,54 +421,3 @@ def parse_fields(new_obj, obj_type):
|
|||
new_obj['retrieved_on'] = int(datetime.utcnow().timestamp())
|
||||
|
||||
return unmatched_field
|
||||
|
||||
|
||||
def merge_lowest_highest_id(str_id, lowest_id, highest_id):
|
||||
int_id = base36decode(str_id)
|
||||
if lowest_id is None or int_id < lowest_id:
|
||||
lowest_id = int_id
|
||||
if highest_id is None or int_id > highest_id:
|
||||
highest_id = int_id
|
||||
return lowest_id, highest_id
|
||||
|
||||
|
||||
async def record_rate_limits(reddit, client):
|
||||
reddit_user = await reddit.user.me()
|
||||
remaining = int(reddit._core._rate_limiter.remaining)
|
||||
used = int(reddit._core._rate_limiter.used)
|
||||
counters.rate_requests_remaining.labels(username=reddit_user.name, client=client).set(remaining)
|
||||
counters.rate_requests_used.labels(username=reddit_user.name, client=client).set(used)
|
||||
|
||||
reset_timestamp = reddit._core._rate_limiter.reset_timestamp
|
||||
seconds_to_reset = (datetime.utcfromtimestamp(reset_timestamp) - datetime.utcnow()).total_seconds()
|
||||
counters.rate_seconds_remaining.labels(username=reddit_user.name, client=client).set(int(seconds_to_reset))
|
||||
window_size = int(reddit._core._rate_limiter.window_size) if reddit._core._rate_limiter.window_size is not None else reddit._core._rate_limiter.window_size
|
||||
time_to_next_request = max((datetime.utcnow() - datetime.utcfromtimestamp(reddit._core._rate_limiter.next_request_timestamp)).total_seconds(), 0)
|
||||
#log.info(f"Rate: u/{reddit_user.name}: {window_size} : {remaining} : {used} : {seconds_to_reset:.2f} : {time_to_next_request:.3f} ")
|
||||
|
||||
return
|
||||
|
||||
|
||||
def chunk_list(items, chunk_size):
|
||||
for i in range(0, len(items), chunk_size):
|
||||
yield items[i:i + chunk_size]
|
||||
|
||||
|
||||
def query_pushshift(ids, bearer, object_type):
|
||||
object_name = "comment" if object_type == ObjectType.COMMENT else "submission"
|
||||
url = f"https://api.pushshift.io/reddit/{object_name}/search?limit=1000&ids={','.join(ids)}"
|
||||
log.debug(f"pushshift query: {url}")
|
||||
response = None
|
||||
for i in range(4):
|
||||
response = requests.get(url, headers={
|
||||
'User-Agent': "In script by /u/Watchful1",
|
||||
'Authorization': f"Bearer {bearer}"})
|
||||
if response.status_code == 200:
|
||||
break
|
||||
if response.status_code == 403:
|
||||
log.warning(f"Pushshift unauthorized, aborting")
|
||||
sys.exit(2)
|
||||
time.sleep(2)
|
||||
if response.status_code != 200:
|
||||
log.warning(f"4 requests failed with status code {response.status_code}")
|
||||
return response.json()['data']
|
||||
|
|
@ -7,13 +7,14 @@ def read_obj_zst(file_name):
|
|||
buffer = ''
|
||||
reader = zstandard.ZstdDecompressor(max_window_size=2**31).stream_reader(file_handle)
|
||||
while True:
|
||||
chunk = reader.read(2**27).decode()
|
||||
chunk = read_and_decode(reader, 2**27, (2**29) * 2)
|
||||
if not chunk:
|
||||
break
|
||||
lines = (buffer + chunk).split("\n")
|
||||
|
||||
for line in lines[:-1]:
|
||||
yield json.loads(line)
|
||||
if line == "":
|
||||
continue
|
||||
yield json.loads(line.strip())
|
||||
|
||||
buffer = lines[-1]
|
||||
reader.close()
|
||||
|
|
@ -71,3 +72,32 @@ class OutputZst:
|
|||
def __exit__(self, exc_type, exc_value, exc_traceback):
|
||||
self.close()
|
||||
return True
|
||||
|
||||
|
||||
def base36encode(integer: int) -> str:
|
||||
chars = '0123456789abcdefghijklmnopqrstuvwxyz'
|
||||
sign = '-' if integer < 0 else ''
|
||||
integer = abs(integer)
|
||||
result = ''
|
||||
while integer > 0:
|
||||
integer, remainder = divmod(integer, 36)
|
||||
result = chars[remainder] + result
|
||||
return sign + result
|
||||
|
||||
|
||||
def base36decode(base36: str) -> int:
|
||||
return int(base36, 36)
|
||||
|
||||
|
||||
def merge_lowest_highest_id(str_id, lowest_id, highest_id):
|
||||
int_id = base36decode(str_id)
|
||||
if lowest_id is None or int_id < lowest_id:
|
||||
lowest_id = int_id
|
||||
if highest_id is None or int_id > highest_id:
|
||||
highest_id = int_id
|
||||
return lowest_id, highest_id
|
||||
|
||||
|
||||
def chunk_list(items, chunk_size):
|
||||
for i in range(0, len(items), chunk_size):
|
||||
yield items[i:i + chunk_size]
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue