diff --git a/.gitignore b/.gitignore index 3b42084..4a71093 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,6 @@ .idea/* logs/* __pycache__/* -personal/* *.db *.ini *.txt \ No newline at end of file diff --git a/Pipfile b/Pipfile index 59b1b98..939c202 100644 --- a/Pipfile +++ b/Pipfile @@ -8,6 +8,7 @@ zstandard = "*" discord-logging = {editable = true, git = "https://github.com/Watchful1/DiscordLogging.git"} requests = "*" pymongo = {extras = ["srv"], version = "*"} +scipy = "*" [dev-packages] diff --git a/Pipfile.lock b/Pipfile.lock index 34528d2..0a55dbb 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "e4f28d9f24e5ee1a4cbc98c2b1f4c21ee3f77e61e24cbbbba195dbd705369d73" + "sha256": "7b82a2e74212a0cbf95cc089a5f0752f547e89f374d95e3d84ccf22b1ecbd6bd" }, "pipfile-spec": 6, "requires": { @@ -25,11 +25,11 @@ }, "charset-normalizer": { "hashes": [ - "sha256:e019de665e2bcf9c2b64e2e5aa025fa991da8720daa3c1138cadd2fd1856aed0", - "sha256:f7af805c321bfa1ce6714c51f254e0d5bb5e5834039bc17db7ebe3a4cec9492b" + "sha256:1eecaa09422db5be9e29d7fc65664e6c33bd06f9ced7838578ba40d58bdf3721", + "sha256:b0b883e8e874edfdece9c28f314e3dd5badf067342e42fb162203335ae61aa2c" ], "markers": "python_version >= '3'", - "version": "==2.0.7" + "version": "==2.0.9" }, "discord-logging": { "editable": true, @@ -51,121 +51,133 @@ "markers": "python_version >= '3'", "version": "==3.3" }, + "numpy": { + "hashes": [ + "sha256:0b78ecfa070460104934e2caf51694ccd00f37d5e5dbe76f021b1b0b0d221823", + "sha256:1247ef28387b7bb7f21caf2dbe4767f4f4175df44d30604d42ad9bd701ebb31f", + "sha256:1403b4e2181fc72664737d848b60e65150f272fe5a1c1cbc16145ed43884065a", + "sha256:170b2a0805c6891ca78c1d96ee72e4c3ed1ae0a992c75444b6ab20ff038ba2cd", + "sha256:2e4ed57f45f0aa38beca2a03b6532e70e548faf2debbeb3291cfc9b315d9be8f", + "sha256:32fe5b12061f6446adcbb32cf4060a14741f9c21e15aaee59a207b6ce6423469", + "sha256:34f3456f530ae8b44231c63082c8899fe9c983fd9b108c997c4b1c8c2d435333", + "sha256:4c9c23158b87ed0e70d9a50c67e5c0b3f75bcf2581a8e34668d4e9d7474d76c6", + "sha256:5d95668e727c75b3f5088ec7700e260f90ec83f488e4c0aaccb941148b2cd377", + "sha256:615d4e328af7204c13ae3d4df7615a13ff60a49cb0d9106fde07f541207883ca", + "sha256:69077388c5a4b997442b843dbdc3a85b420fb693ec8e33020bb24d647c164fa5", + "sha256:74b85a17528ca60cf98381a5e779fc0264b4a88b46025e6bcbe9621f46bb3e63", + "sha256:81225e58ef5fce7f1d80399575576fc5febec79a8a2742e8ef86d7b03beef49f", + "sha256:8890b3360f345e8360133bc078d2dacc2843b6ee6059b568781b15b97acbe39f", + "sha256:92aafa03da8658609f59f18722b88f0a73a249101169e28415b4fa148caf7e41", + "sha256:9864424631775b0c052f3bd98bc2712d131b3e2cd95d1c0c68b91709170890b0", + "sha256:9e6f5f50d1eff2f2f752b3089a118aee1ea0da63d56c44f3865681009b0af162", + "sha256:a3deb31bc84f2b42584b8c4001c85d1934dbfb4030827110bc36bfd11509b7bf", + "sha256:ad010846cdffe7ec27e3f933397f8a8d6c801a48634f419e3d075db27acf5880", + "sha256:b1e2312f5b8843a3e4e8224b2b48fe16119617b8fc0a54df8f50098721b5bed2", + "sha256:bc988afcea53e6156546e5b2885b7efab089570783d9d82caf1cfd323b0bb3dd", + "sha256:c449eb870616a7b62e097982c622d2577b3dbc800aaf8689254ec6e0197cbf1e", + "sha256:c74c699b122918a6c4611285cc2cad4a3aafdb135c22a16ec483340ef97d573c", + "sha256:c885bfc07f77e8fee3dc879152ba993732601f1f11de248d4f357f0ffea6a6d4", + "sha256:e3c3e990274444031482a31280bf48674441e0a5b55ddb168f3a6db3e0c38ec8", + "sha256:e4799be6a2d7d3c33699a6f77201836ac975b2e1b98c2a07f66a38f499cb50ce", + "sha256:e6c76a87633aa3fa16614b61ccedfae45b91df2767cf097aa9c933932a7ed1e0", + "sha256:e89717274b41ebd568cd7943fc9418eeb49b1785b66031bc8a7f6300463c5898", + "sha256:f5162ec777ba7138906c9c274353ece5603646c6965570d82905546579573f73", + "sha256:fde96af889262e85aa033f8ee1d3241e32bf36228318a61f1ace579df4e8170d" + ], + "markers": "python_version < '3.11' and python_version >= '3.7'", + "version": "==1.21.4" + }, "pymongo": { "extras": [ "srv" ], "hashes": [ - "sha256:02e0c088f189ca69fac094cb5f851b43bbbd7cec42114495777d4d8f297f7f8a", - "sha256:138248c542051eb462f88b50b0267bd5286d6661064bab06faa0ef6ac30cdb4b", - "sha256:13a7c6d055af58a1e9c505e736da8b6a2e95ccc8cec10b008143f7a536e5de8a", - "sha256:13d74bf3435c1e58d8fafccc0d5e87f246ae2c6e9cbef4b35e32a1c3759e354f", - "sha256:15dae01341571d0af51526b7a21648ca575e9375e16ba045c9860848dfa8952f", - "sha256:17238115e6d37f5423b046cb829f1ca02c4ea7edb163f5b8b88e0c975dc3fec9", - "sha256:180b405e17b90a877ea5dbc5efe7f4c171af4c89323148e100c0f12cedb86f12", - "sha256:1821ce4e5a293313947fd017bbd2d2535aa6309680fa29b33d0442d15da296ec", - "sha256:1a7b138a04fdd17849930dc8bf664002e17db38448850bfb96d200c9c5a8b3a1", - "sha256:1c4e51a3b69789b6f468a8e881a13f2d1e8f5e99e41f80fd44845e6ec0f701e1", - "sha256:1d55982e5335925c55e2b87467043866ce72bd30ea7e7e3eeed6ec3d95a806d4", - "sha256:1fa6f08ddb6975371777f97592d35c771e713ee2250e55618148a5e57e260aff", - "sha256:2174d3279b8e2b6d7613b338f684cd78ff7adf1e7ec5b7b7bde5609a129c9898", - "sha256:2462a68f6675da548e333fa299d8e9807e00f95a4d198cfe9194d7be69f40c9b", - "sha256:25fd76deabe9ea37c8360c362b32f702cc095a208dd1c5328189938ca7685847", - "sha256:287c2a0063267c1458c4ddf528b44063ce7f376a6436eea5bccd7f625bbc3b5e", - "sha256:2d3abe548a280b49269c7907d5b71199882510c484d680a5ea7860f30c4a695f", - "sha256:2fa101bb23619120673899694a65b094364269e597f551a87c4bdae3a474d726", - "sha256:2fda3b3fb5c0d159195ab834b322a23808f1b059bcc7e475765abeddee6a2529", - "sha256:303531649fa45f96b694054c1aa02f79bda32ef57affe42c5c339336717eed74", - "sha256:36806ee53a85c3ba73939652f2ced2961e6a77cfbae385cd83f2e24cd97964b7", - "sha256:37a63da5ee623acdf98e6d511171c8a5827a6106b0712c18af4441ef4f11e6be", - "sha256:3a2fcbd04273a509fa85285d9eccf17ab65ce440bd4f5e5a58c978e563cd9e9a", - "sha256:3b40e36d3036bfe69ba63ec8e746a390721f75467085a0384b528e1dda532c69", - "sha256:4168b6c425d783e81723fc3dc382d374a228ff29530436a472a36d9f27593e73", - "sha256:444c00ebc20f2f9dc62e34f7dc9453dc2f5f5a72419c8dccad6e26d546c35712", - "sha256:45d6b47d70ed44e3c40bef618ed61866c48176e7e5dff80d06d8b1a6192e8584", - "sha256:460bdaa3f65ddb5b7474ae08589a1763b5da1a78b8348351b9ba1c63b459d67d", - "sha256:47ed77f62c8417a86f9ad158b803f3459a636386cb9d3d4e9e7d6a82d051f907", - "sha256:48722e91981bb22a16b0431ea01da3e1cc5b96805634d3b8d3c2a5315c1ce7f1", - "sha256:49b0d92724d3fce1174fd30b0b428595072d5c6b14d6203e46a9ea347ae7b439", - "sha256:4a2d73a9281faefb273a5448f6d25f44ebd311ada9eb79b6801ae890508fe231", - "sha256:4f4bc64fe9cbd70d46f519f1e88c9e4677f7af18ab9cd4942abce2bcfa7549c3", - "sha256:5067c04d3b19c820faac6342854d887ade58e8d38c3db79b68c2a102bbb100e7", - "sha256:51437c77030bed72d57d8a61e22758e3c389b13fea7787c808030002bb05ca39", - "sha256:515e4708d6567901ffc06476a38abe2c9093733f52638235d9f149579c1d3de0", - "sha256:5183b698d6542219e4135de583b57bc6286bd37df7f645b688278eb919bfa785", - "sha256:56feb80ea1f5334ccab9bd16a5161571ab70392e51fcc752fb8a1dc67125f663", - "sha256:573e2387d0686976642142c50740dfc4d3494cc627e2a7d22782b99f70879055", - "sha256:58a67b3800476232f9989e533d0244060309451b436d46670a53e6d189f1a7e7", - "sha256:5e3833c001a04aa06a28c6fd9628256862a654c09b0f81c07734b5629bc014ab", - "sha256:5f5fe59328838fa28958cc06ecf94be585726b97d637012f168bc3c7abe4fd81", - "sha256:6235bf2157aa46e53568ed79b70603aa8874baa202d5d1de82fa0eb917696e73", - "sha256:63be03f7ae1e15e72a234637ec7941ef229c7ab252c9ff6af48bba1e5418961c", - "sha256:65f159c445761cab04b665fc448b3fc008aebc98e54fdcbfd1aff195ef1b1408", - "sha256:67e0b2ad3692f6d0335ae231a40de55ec395b6c2e971ad6f55b162244d1ec542", - "sha256:68409171ab2aa7ccd6e8e839233e4b8ddeec246383c9a3698614e814739356f9", - "sha256:6a96c04ce39d66df60d9ce89f4c254c4967bc7d9e2e2c52adc58f47be826ee96", - "sha256:6ead0126fb4424c6c6a4fdc603d699a9db7c03cdb8eac374c352a75fec8a820a", - "sha256:6eb6789f26c398c383225e1313c8e75a7d290d323b8eaf65f3f3ddd0eb8a5a3c", - "sha256:6f07888e3b73c0dfa46f12d098760494f5f23fd66923a6615edfe486e6a7649c", - "sha256:6f0f0a10f128ea0898e607d351ebfabf70941494fc94e87f12c76e2894d8e6c4", - "sha256:704879b6a54c45ad76cea7c6789c1ae7185050acea7afd15b58318fa1932ed45", - "sha256:7117bfd8827cfe550f65a3c399dcd6e02226197a91c6d11a3540c3e8efc686d6", - "sha256:712de1876608fd5d76abc3fc8ec55077278dd5044073fbe9492631c9a2c58351", - "sha256:75c7ef67b4b8ec070e7a4740764f6c03ec9246b59d95e2ae45c029d41cb9efa1", - "sha256:77dddf596fb065de29fb39992fbc81301f7fd0003be649b7fa7448c77ca53bed", - "sha256:7abc87e45b572eb6d17a50422e69a9e5d6f13e691e821fe2312df512500faa50", - "sha256:7d8cdd2f070c71366e64990653522cce84b08dc26ab0d1fa19aa8d14ee0cf9ba", - "sha256:81ce5f871f5d8e82615c8bd0b34b68a9650204c8b1a04ce7890d58c98eb66e39", - "sha256:837cdef094f39c6f4a2967abc646a412999c2540fbf5d3cce1dd3b671f4b876c", - "sha256:849e641cfed05c75d772f9e9018f42c5fbd00655d43d52da1b9c56346fd3e4cc", - "sha256:87114b995506e7584cf3daf891e419b5f6e7e383e7df6267494da3a76312aa22", - "sha256:87db421c9eb915b8d9a9a13c5b2ee338350e36ee83e26ff0adfc48abc5db3ac3", - "sha256:8851544168703fb519e95556e3b463fca4beeef7ed3f731d81a68c8268515d9d", - "sha256:891f541c7ed29b95799da0cd249ae1db1842777b564e8205a197b038c5df6135", - "sha256:8f87f53c9cd89010ae45490ec2c963ff18b31f5f290dc08b04151709589fe8d9", - "sha256:9641be893ccce7d192a0094efd0a0d9f1783a1ebf314b4128f8a27bfadb8a77c", - "sha256:979e34db4f3dc5710c18db437aaf282f691092b352e708cb2afd4df287698c76", - "sha256:9b62d84478f471fdb0dcea3876acff38f146bd23cbdbed15074fb4622064ec2e", - "sha256:a472ca3d43d33e596ff5836c6cc71c3e61be33f44fe1cfdab4a1100f4af60333", - "sha256:a5dbeeea6a375fbd79448b48a54c46fc9351611a03ef8398d2a40b684ce46194", - "sha256:a7430f3987d232e782304c109be1d0e6fff46ca6405cb2479e4d8d08cd29541e", - "sha256:a81e52dbf95f236a0c89a5abcd2b6e1331da0c0312f471c73fae76c79d2acf6b", - "sha256:aa434534cc91f51a85e3099dc257ee8034b3d2be77f2ca58fb335a686e3a681f", - "sha256:ab27d6d7d41a66d9e54269a290d27cd5c74f08e9add0054a754b4821026c4f42", - "sha256:adb37bf22d25a51b84d989a2a5c770d4514ac590201eea1cb50ce8c9c5257f1d", - "sha256:afb16330ab6efbbf995375ad94e970fa2f89bb46bd10d854b7047620fdb0d67d", - "sha256:b1b06038c9940a49c73db0aeb0f6809b308e198da1326171768cf68d843af521", - "sha256:b1e6d1cf4bd6552b5f519432cce1530c09e6b0aab98d44803b991f7e880bd332", - "sha256:bf2d9d62178bb5c05e77d40becf89c309b1966fbcfb5c306238f81bf1ec2d6a2", - "sha256:bfd073fea04061019a103a288847846b5ef40dfa2f73b940ed61e399ca95314f", - "sha256:c04e84ccf590933a266180286d8b6a5fc844078a5d934432628301bd8b5f9ca7", - "sha256:c0947d7be30335cb4c3d5d0983d8ebc8294ae52503cf1d596c926f7e7183900b", - "sha256:c2a17752f97a942bdb4ff4a0516a67c5ade1658ebe1ab2edacdec0b42e39fa75", - "sha256:c4653830375ab019b86d218c749ad38908b74182b2863d09936aa8d7f990d30e", - "sha256:c660fd1e4a4b52f79f7d134a3d31d452948477b7f46ff5061074a534c5805ba6", - "sha256:cb48ff6cc6109190e1ccf8ea1fc71cc244c9185813ce7d1c415dce991cfb8709", - "sha256:cef2675004d85d85a4ccc24730b73a99931547368d18ceeed1259a2d9fcddbc1", - "sha256:d1b98539b0de822b6f717498e59ae3e5ae2e7f564370ab513e6d0c060753e447", - "sha256:d6c6989c10008ac70c2bb2ad2b940fcfe883712746c89f7e3308c14c213a70d7", - "sha256:db3efec9dcecd96555d752215797816da40315d61878f90ca39c8e269791bf17", - "sha256:dc4749c230a71b34db50ac2481d9008bb17b67c92671c443c3b40e192fbea78e", - "sha256:dcf906c1f7a33e4222e4bff18da1554d69323bc4dd95fe867a6fa80709ee5f93", - "sha256:e2bccadbe313b11704160aaba5eec95d2da1aa663f02f41d2d1520d02bbbdcd5", - "sha256:e30cce3cc86d6082c8596b3fbee0d4f54bc4d337a4fa1bf536920e2e319e24f0", - "sha256:e5d6428b8b422ba5205140e8be11722fa7292a0bedaa8bc80fb34c92eb19ba45", - "sha256:e841695b5dbea38909ab2dbf17e91e9a823412d8d88d1ef77f1b94a7bc551c0f", - "sha256:eb65ec0255a0fccc47c87d44e505ef5180bfd71690bd5f84161b1f23949fb209", - "sha256:ed20ec5a01c43254f6047c5d8124b70d28e39f128c8ad960b437644fe94e1827", - "sha256:ed751a20840a31242e7bea566fcf93ba75bc11b33afe2777bbf46069c1af5094", - "sha256:ef8b927813c27c3bdfc82c55682d7767403bcdadfd9f9c0fc49f4be4553a877b", - "sha256:f43cacda46fc188f998e6d308afe1c61ff41dcb300949f4cbf731e9a0a5eb2d3", - "sha256:f44bea60fd2178d7153deef9621c4b526a93939da30010bba24d3408a98b0f79", - "sha256:fcc021530b7c71069132fe4846d95a3cdd74d143adc2f7e398d5fabf610f111c", - "sha256:fe16517b275031d61261a4e3941c411fb7c46a9cd012f02381b56e7907cc9e06", - "sha256:fe3ae4294d593da54862f0140fdcc89d1aeeb94258ca97f094119ed7f0e5882d" + "sha256:0238e53b452ab699b5e2e3f8af2557844c80ab0d0c7a0e066226882838e72756", + "sha256:0271bbba36bb130202e011171c1883c4c193036ad0b1e02ecfbea6837790b7de", + "sha256:069d49b193f94bb1d748cfd8faf697060a2299f40d86bf5b6d41dd3cedf0fd48", + "sha256:06af6e6374ee2bb70f724e09ddf9402907a3d6714828b908737948cd83e5685c", + "sha256:0c77cd3dbe0dd9e7cdf8c93dc24e5a4fcb56e115ffb259d4f399e4aaf3f3c62d", + "sha256:0e9a2628bcd896368ede456bcfe189d9ca65b18fb0dd91974cb734baf2e24af9", + "sha256:12d336bdbe60982de55651be397b5e49d7eadd2aa144f11da353002cd52502ed", + "sha256:132cc67b909835d7c230888387b4cc9596d4559a3ce90d947e03bc0b0ffe420b", + "sha256:13d0624c13a91da71fa0d960205d93b3d98344481be865ee7cc238c972d41d73", + "sha256:1498f388181ae5592c7b60549faaefaffc62d6e3754097576611cb642d21d37b", + "sha256:1617fd52da7b208fe5ea176d251dd7cf1b5309e5a4272754b9599edfdf7e64e5", + "sha256:177ed1b14aa4f84f00ebef1b0f785680fbaa610361942b23eb54f562fe4c6b34", + "sha256:186b2ff4518c1c169fcef5047deb0e6c13a2354d143859587e745fd9f2cf68e9", + "sha256:1ba8eb426d56556fffec53d600a8f2572589c19d50b30f61daa8f4d72ab92fbe", + "sha256:1c153274699424e8f89f2097d5113f8cbe7898a8d62afaad0270a0f0bd0af53b", + "sha256:1fd71b4d7070b01c7f66edc44c1ec2f8bcace2761c3a6ecc10449a40e474d2fa", + "sha256:28afb00423e521f4b04fb8f75da7c0215e46631e821e27abf5a7176f9b671f47", + "sha256:349c8e522e0b785f442fc9d7fc01c59f7f13f1abe9395310d0d817cff03ec034", + "sha256:35a5843546bcbe0422f30b4b2bd5e0b630b04cc4006492c70e8168a921d94b9e", + "sha256:38b21eddd021a943b1978b0a3d42e974956a338e3dbb88d56aeb8b8799abd6e8", + "sha256:3a4eb0a4db8a2d960bdd5354f05e2e57530e83d333cb644fb2b7120a7a954a69", + "sha256:40269fe6bb79fe00c8ba7c2f2d542a82711eb234c3dedb90b7e489386120e9d1", + "sha256:426584e99af31ad2398e617c3eb0f1ebcda37f0ffb2d3e56087cdaf23a2f1689", + "sha256:47a58f15fc70198cf95982f9699e17fec12287b90f30e90c5e2b7c1c1bc07914", + "sha256:512059a902ea2cbcd0afac370af580e67ccd4c7e41ecaff0f0fbd03653b25ca2", + "sha256:51664dac8d9b138259876f324adca5ab31d991acf88d1d0ffcc94f423ff2e31b", + "sha256:59a4a5fe5379e4fa93380fd0b55bccbdbeb8d04fcfbbad8b42bd31610d5ed3ad", + "sha256:5cbfa85a12cfe3dca21951cd432051c505ac461bd9f4a635207d982dd9df2373", + "sha256:5fea4207fec8909e155a7948c987eac61949dbbe97fd0c388e587d06ba9bc78d", + "sha256:6183476860511cb553a7e4c40936221b6985af7852029c84df898370ec8a028c", + "sha256:62459b91a513a7b441cfd70ea7fd15c50b858877ca823915d32bab08fe173edb", + "sha256:633ca2001f80900142068bab907feca99554b557ac105c74a9ed157ed38ca5d6", + "sha256:65f8a93816dcb2202710839907759aca9eece94d9f13215686f224fcc8966f9e", + "sha256:686c40344f7f82c4deaa4e17aa46ad97df51263be1434aeedd2d6b6f38c7f44a", + "sha256:6cd7a4321e718cb98a7c7c475b0757e77fdaf1cdb013d7d2e781ba45219e1144", + "sha256:6f0605b1146bc24c720aac0e806492144aea9d5a4dc956589e0544301862756a", + "sha256:716499113650aacfe1b94d37e0a863f1e84b8d47737c74a2f44f8dfccad46952", + "sha256:71810eade75ae1c466adc158d1fa8141040f75427b76240316d97f3c89edd72f", + "sha256:72a0c06b76b254bdec18af9add3b8d35796dda51e64a5e0e48d40bff7b41ab13", + "sha256:7450b25a803b0f57dae4c3fbd0df742f7f3344c3c9cabb86e4180083c3ebd893", + "sha256:75e449ab068af63b7729195343315bc63d242166d88467314be182cc54ce235d", + "sha256:7629abba158610cb5db6c22041b287f9398555d72bf9468d44d2efc03d837b81", + "sha256:774b9f48bdc385af6654def31e7a7617e01b99cc8aaca1ab3ef6ea0492205e57", + "sha256:7a091050bb8d54a5200193b4998e0cf763d083f93d97c7780963c09996f85a38", + "sha256:7bdb66340e246b5dcddfcfe79a63ac2ec3808dc394853476f49fc785425040f4", + "sha256:812650a2e8a08b812d6a3c937f482bd2c9355e90574964fa283b4d8ef4ae665e", + "sha256:84eec41ed982f21ceb58689e16a630a70301eb14499c929388a5bf6464518d9d", + "sha256:86d0e28dd5867153d9d9963a4eb17764854a925758fc2db0a814260f82fd4319", + "sha256:87dce7c85387ca033cf76cce773ace7675550dcffc456db32a34403439e53e45", + "sha256:8869feff59f08cd63979df26aa12343a85bdc7fbd1b79fda8ae39f31a310fa62", + "sha256:8baf23d6a0a08b697854e5bcdf82afb91da732cf575fd47ee93945c3654132d8", + "sha256:8da525765dbcc1b7abf1bba623f9f701d8759a8fb19594cd71a13b7b0c2c56bd", + "sha256:9043bfb816ed50d831acc8d06469dcc41597b4f50c30e62227a93f9f9e37d6c7", + "sha256:91c049104b51321e4e18d41edc6850d9f0890ac609b3cb3b8db86dc51666de17", + "sha256:93c25fbb5dbc436edbb74101f4da49a42bd3af534513fdf8e75fc72ef035d5e0", + "sha256:953129b6b952a9d22042ac23050053444624f630e1928f5f590788660905fa9c", + "sha256:9ff0dbec451a2c6226bbd6f2bbbde438bc263e002f3f97d151c8708732ba5197", + "sha256:a47f4b24b1360da172cae07ce90e9bd425b6db0052d76142c7fef47173a27283", + "sha256:a57e271a0647002b5683dd0c7c2fd7f5fb939357c44396d85298e51a3561b9e3", + "sha256:b0606d14892ae2a2b1450e37c8924381e9b64683386a9853e4467f02fd5b44b6", + "sha256:b73ff8582964f52ab1bf1a9fdddc1454143172a0b8a9d7d6e3972dd1134f7982", + "sha256:bf6047dea1bc8ae19fc14e01b5cb70b3810f91b100d9a535751dd3eadcd3016c", + "sha256:c0efc5ab7d9b9e64726496bf650dbc7f1754124a48d076e5292cc5306e61a530", + "sha256:c86a0614eda95db036fae01a89f3917d7abdc657c806bac2a32eec74724d9330", + "sha256:c878286b1464f462616a47f315d14f02f03512c6b81cb568e996c3f1f79bff8a", + "sha256:cd4cde3dfdd347d638171eca53ee6e787d4b1247c6e182f8616039b1df6278d5", + "sha256:ceb9a4986f56595e73fffeef3ec037280eda938ed5fe6e4e0961656669d89b32", + "sha256:d419e2dbc4943ad6df7ee05e707d7b2c2b512b92407bb6ff643bccbdea399c3a", + "sha256:d66462f740dcea496bd779775688a0f805860f0b01998bb59ca22566b098ee26", + "sha256:d7514231a03e95072b32d9b335b96253799802ab94647ce83585d5010749380a", + "sha256:d9f61b08b60909d936c1f3a4e12c163ca71fd1a4665fc6e078afc6f54f886977", + "sha256:da576e59f5f8a642ee26d027479325a45be45defe075b6fa7c84506dabc76883", + "sha256:ddaf391ba74eef47eb5afbc40d0b6ddcdbdb417ec8edc8ae95352d25485076db", + "sha256:e2b6a323ca545bcb4286d14c0bd75d9a1f5bce2fa1d7fa3621e5f71fd9b8d196", + "sha256:e3f6faea65a73ed54111f209b4a411fe012c68f04e8bde96dd7af89b13cac92b", + "sha256:e4e36810c541bd1976cd05452e797860b775886cf32c3e8136b9fe48c2c8ba95", + "sha256:e5441f4c8142a250695e249e432637c14f79d856a2b60e0974da082e006c53e2", + "sha256:e7aedefc87cb46544a3865a19c1d5ca7ddf5ec5ed7dfe162d9538d7543aef499", + "sha256:ee2c1fd5bd57fd0092dfa31c1f9f166cf2850f191311603ce343cadcc8608d60", + "sha256:f2b6e12f98cce588525f3db802c88f9795d294549ebfe7c2c9bb81333f533ecd", + "sha256:f333c0d71dd892683e608f8d1731785a0aa67b1ec012b0d9fc863e8d7224f64e", + "sha256:f3e20eb096deea92350f7198a4287d45883a62fe4459d027ce789e72ceba12ee", + "sha256:f785375ca2b4e2192786f1e0d2a94c66900d12e780ebae1eccbbab85eb9a7054" ], "index": "pypi", - "version": "==3.12.1" + "version": "==4.0.1" }, "requests": { "hashes": [ @@ -175,6 +187,41 @@ "index": "pypi", "version": "==2.26.0" }, + "scipy": { + "hashes": [ + "sha256:033ce76ed4e9f62923e1f8124f7e2b0800db533828c853b402c7eec6e9465d80", + "sha256:173308efba2270dcd61cd45a30dfded6ec0085b4b6eb33b5eb11ab443005e088", + "sha256:21b66200cf44b1c3e86495e3a436fc7a26608f92b8d43d344457c54f1c024cbc", + "sha256:2c56b820d304dffcadbbb6cbfbc2e2c79ee46ea291db17e288e73cd3c64fefa9", + "sha256:304dfaa7146cffdb75fbf6bb7c190fd7688795389ad060b970269c8576d038e9", + "sha256:3f78181a153fa21c018d346f595edd648344751d7f03ab94b398be2ad083ed3e", + "sha256:4d242d13206ca4302d83d8a6388c9dfce49fc48fdd3c20efad89ba12f785bf9e", + "sha256:5d1cc2c19afe3b5a546ede7e6a44ce1ff52e443d12b231823268019f608b9b12", + "sha256:5f2cfc359379c56b3a41b17ebd024109b2049f878badc1e454f31418c3a18436", + "sha256:65bd52bf55f9a1071398557394203d881384d27b9c2cad7df9a027170aeaef93", + "sha256:7edd9a311299a61e9919ea4192dd477395b50c014cdc1a1ac572d7c27e2207fa", + "sha256:8499d9dd1459dc0d0fe68db0832c3d5fc1361ae8e13d05e6849b358dc3f2c279", + "sha256:866ada14a95b083dd727a845a764cf95dd13ba3dc69a16b99038001b05439709", + "sha256:87069cf875f0262a6e3187ab0f419f5b4280d3dcf4811ef9613c605f6e4dca95", + "sha256:93378f3d14fff07572392ce6a6a2ceb3a1f237733bd6dcb9eb6a2b29b0d19085", + "sha256:95c2d250074cfa76715d58830579c64dff7354484b284c2b8b87e5a38321672c", + "sha256:ab5875facfdef77e0a47d5fd39ea178b58e60e454a4c85aa1e52fcb80db7babf", + "sha256:b0e0aeb061a1d7dcd2ed59ea57ee56c9b23dd60100825f98238c06ee5cc4467e", + "sha256:b78a35c5c74d336f42f44106174b9851c783184a85a3fe3e68857259b37b9ffb", + "sha256:c9e04d7e9b03a8a6ac2045f7c5ef741be86727d8f49c45db45f244bdd2bcff17", + "sha256:ca36e7d9430f7481fc7d11e015ae16fbd5575615a8e9060538104778be84addf", + "sha256:ceebc3c4f6a109777c0053dfa0282fddb8893eddfb0d598574acfb734a926168", + "sha256:e2c036492e673aad1b7b0d0ccdc0cb30a968353d2c4bf92ac8e73509e1bf212c", + "sha256:eb326658f9b73c07081300daba90a8746543b5ea177184daed26528273157294", + "sha256:eb7ae2c4dbdb3c9247e07acc532f91077ae6dbc40ad5bd5dca0bb5a176ee9bda", + "sha256:edad1cf5b2ce1912c4d8ddad20e11d333165552aba262c882e28c78bbc09dbf6", + "sha256:eef93a446114ac0193a7b714ce67659db80caf940f3232bad63f4c7a81bc18df", + "sha256:f7eaea089345a35130bc9a39b89ec1ff69c208efa97b3f8b25ea5d4c41d88094", + "sha256:f99d206db1f1ae735a8192ab93bd6028f3a42f6fa08467d37a14eb96c9dd34a3" + ], + "index": "pypi", + "version": "==1.7.3" + }, "urllib3": { "hashes": [ "sha256:4987c65554f7a2dbf30c18fd48778ef124af6fab771a377103da0585e2336ece", diff --git a/personal/comments_per_day.py b/personal/comments_per_day.py new file mode 100644 index 0000000..797ba07 --- /dev/null +++ b/personal/comments_per_day.py @@ -0,0 +1,21 @@ +import utils +import discord_logging +from datetime import datetime + +log = discord_logging.init_logging() + + +if __name__ == "__main__": + day = None + day_comments = 0 + for comment in utils.read_obj_zst(r"\\MYCLOUDPR4100\Public\reddit_final\wallstreetbets_comments.zst"): + created_day = datetime.utcfromtimestamp(int(comment['created_utc'])).strftime("%m/%d/%y") + if day is None: + day = created_day + if day != created_day: + log.info(f"{day} {day_comments}") + day_comments = 0 + day = created_day + day_comments += 1 + + log.info(f"{day} {day_comments}") diff --git a/personal/compare_lines.py b/personal/compare_lines.py new file mode 100644 index 0000000..98ac8ea --- /dev/null +++ b/personal/compare_lines.py @@ -0,0 +1,33 @@ +import utils +import discord_logging +import os +import sys +from datetime import datetime + +log = discord_logging.init_logging() + + +if __name__ == "__main__": + file_one = open(r"\\MYCLOUDPR4100\Public\reddit_final\RelationshipsOver35_comments_dump.txt", 'r') + file_two = open(r"\\MYCLOUDPR4100\Public\reddit_final\RelationshipsOver35_comments_mongo.txt", 'r') + + file_lines = 0 + while True: + file_lines += 1 + line_one = file_one.readline().rstrip() + line_two = file_two.readline().rstrip() + if line_one != line_two: + log.info(f"lines not matching: {file_lines}") + log.info(line_one) + log.info(line_two) + #break + + if file_lines % 100000 == 0: + log.info(f"{file_lines:,}") + + if not line_one: + break + + log.info(f"{file_lines:,}") + file_one.close() + file_two.close() diff --git a/personal/count_by_subreddit.py b/personal/count_by_subreddit.py new file mode 100644 index 0000000..b5d4884 --- /dev/null +++ b/personal/count_by_subreddit.py @@ -0,0 +1,33 @@ +import utils +import discord_logging +import os +from datetime import datetime + +log = discord_logging.init_logging() + + +if __name__ == "__main__": + subreddits = {} + object_type = "submissions" + folder = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\{object_type}" + if not os.path.exists(folder): + os.makedirs(folder) + input_file = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\relationships_{object_type}.zst" + input_file_size = os.stat(input_file).st_size + total_lines = 0 + for comment, line, file_bytes_processed in utils.read_obj_zst_meta(input_file): + if comment['subreddit'] not in subreddits: + subreddits[comment['subreddit']] = {'writer': utils.OutputZst(os.path.join(folder, comment['subreddit'] + f"_{object_type}.zst")), 'lines': 0} + subreddit = subreddits[comment['subreddit']] + subreddit['writer'].write(line) + subreddit['writer'].write("\n") + subreddit['lines'] += 1 + total_lines += 1 + if total_lines % 100000 == 0: + log.info(f"{total_lines:,} lines, {(file_bytes_processed / input_file_size) * 100:.0f}%") + + log.info(f"{total_lines:,} lines, 100%") + + for name, subreddit in subreddits.items(): + log.info(f"r/{name}: {subreddit['lines']:,} lines") + subreddit['writer'].close() diff --git a/personal/export_mongo.py b/personal/export_mongo.py new file mode 100644 index 0000000..fc45054 --- /dev/null +++ b/personal/export_mongo.py @@ -0,0 +1,47 @@ +import json + +import utils +import discord_logging +import pymongo +import time +import sys + +log = discord_logging.init_logging() + + +if __name__ == "__main__": + mongo_address = sys.argv[1] # 192.168.1.131 + client = pymongo.MongoClient(f"mongodb://{mongo_address}:27017", serverSelectionTimeoutMS=5000) + log.info(f"Database connected at {mongo_address} on {client.admin.command('serverStatus')['host']}") + + count = 0 + start_time = time.time() + cursor = client.reddit_database.comments.find( + filter={"subreddit": "RelationshipsOver35"}, + projection={'_id': False}, + sort=[('created_utc', pymongo.ASCENDING)] + ) + log.info(f"Got cursor in {int(time.time() - start_time)} seconds") + + output_writer = utils.OutputZst(r"\\MYCLOUDPR4100\Public\reddit_final\RelationshipsOver35_comments.zst") + start_time = time.time() + for comment in cursor: + count += 1 + output_writer.write(json.dumps(comment, separators=(',', ':'))) + output_writer.write("\n") + if count % 100000 == 0: + log.info(f"{count,} in {int(time.time() - start_time)} seconds") + + output_writer.close() + log.info(f"{count,} in {int(time.time() - start_time)} seconds") + + +# db.comments.createIndex({subreddit:1}) // remove +# db.comments.createIndex({subreddit:1, created_utc:1}) +# db.comments.createIndex({author:1, created_utc:1}) +# db.comments.createIndex({id:1}) +# db.submissions.createIndex({subreddit:1, created_utc:1}) +# db.submissions.createIndex({author:1, created_utc:1}) +# db.submissions.createIndex({id:1}) +# db.submissions.createIndex({created_utc:1}) +# db.comments.createIndex({created_utc:1}) diff --git a/personal/group_subs.py b/personal/group_subs.py new file mode 100644 index 0000000..eaa3bc7 --- /dev/null +++ b/personal/group_subs.py @@ -0,0 +1,57 @@ +import json +from datetime import datetime +import utils +import discord_logging +import pymongo +import time +import sys + +log = discord_logging.init_logging() + + +if __name__ == "__main__": + mongo_address = sys.argv[1] # 192.168.1.131 + client = pymongo.MongoClient(f"mongodb://{mongo_address}:27017", serverSelectionTimeoutMS=5000) + log.info(f"Database connected at {mongo_address} on {client.admin.command('serverStatus')['host']}") + + count = 0 + start_time = time.time() + start_date = int(datetime(2021, 6, 1).timestamp()) + cursor = client.reddit_database.submissions.aggregate( + [ + {"$match": {"created_utc": {"$gt": start_date}}}, + {"$project": {"subreddit": 1, "over_18": {"$cond": ["$over_18", 1, 0]}}}, + {"$group": {"_id": "$subreddit", "countTotal": {"$count": {}}, "countNsfw": {"$sum": "$over_18"}}}, + {"$match": {"countTotal": {"$gt": 100}}}, + ], + allowDiskUse=True + ) + log.info(f"Got cursor in {int(time.time() - start_time)} seconds") + + start_time = time.time() + subreddits = [] + for subreddit in cursor: + subreddit['percent'] = int((subreddit['countNsfw']/subreddit['countTotal'])*100) + if subreddit['percent'] >= 10: + subreddits.append(subreddit) + count += 1 + if count % 100000 == 0: + log.info(f"{count:,} in {int(time.time() - start_time)} seconds") + + log.info(f"{count:,} in {int(time.time() - start_time)} seconds") + + file_out = open(r"\\MYCLOUDPR4100\Public\reddit_final\subreddits.txt", 'w') + for subreddit in sorted(subreddits, key=lambda item: (item['percent'], item['countTotal']), reverse=True): + file_out.write(f"{subreddit['_id']: <22}{subreddit['countTotal']: <8}{subreddit['countNsfw']: <8}{subreddit['percent']}%\n") + file_out.close() + + +# db.comments.createIndex({subreddit:1}) // remove +# db.comments.createIndex({subreddit:1, created_utc:1}) +# db.comments.createIndex({author:1, created_utc:1}) +# db.comments.createIndex({id:1}) +# db.submissions.createIndex({subreddit:1, created_utc:1}) +# db.submissions.createIndex({author:1, created_utc:1}) +# db.submissions.createIndex({id:1}) +# db.submissions.createIndex({created_utc:1}) +# db.comments.createIndex({created_utc:1}) diff --git a/personal/insert_mongo.py b/personal/insert_mongo.py new file mode 100644 index 0000000..e3c6dd3 --- /dev/null +++ b/personal/insert_mongo.py @@ -0,0 +1,62 @@ +import utils +import discord_logging +import os +import pymongo +import sys +from datetime import datetime + +log = discord_logging.init_logging() + + +if __name__ == "__main__": + mongo_address = sys.argv[1] # 192.168.1.131 + client = pymongo.MongoClient(f"mongodb://{mongo_address}:27017", serverSelectionTimeoutMS=5000) + + log.info(f"Database connected at {mongo_address} on {client.admin.command('serverStatus')['host']}") + + object_type = sys.argv[2] + input_folder = sys.argv[3] + input_files = [] + total_size = 0 + for subdir, dirs, files in os.walk(input_folder + os.sep + object_type): + files.sort() + for filename in files: + input_path = os.path.join(subdir, filename) + if input_path.endswith(".zst"): + file_size = os.stat(input_path).st_size + total_size += file_size + input_files.append([input_path, file_size]) + + log.info(f"Processing {len(input_files)} files of {(total_size / (2 ** 30)):.2f} gigabytes") + + collection = client.reddit_database[object_type] + + log.info(f"Using collection {object_type} which has {collection.estimated_document_count()} objects already") + + total_lines = 0 + total_bytes_processed = 0 + for input_file in input_files: + file_lines = 0 + file_bytes_processed = 0 + created = None + inserts = [] + for obj, line, file_bytes_processed in utils.read_obj_zst_meta(input_file[0]): + inserts.append(obj) + if len(inserts) >= 10000: + collection.insert_many(inserts) + inserts = [] + + created = datetime.utcfromtimestamp(int(obj['created_utc'])) + file_lines += 1 + if file_lines == 1: + log.info(f"{created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines + total_lines:,} : 0% : {(total_bytes_processed / total_size) * 100:.0f}%") + if file_lines % 100000 == 0: + log.info(f"{created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines + total_lines:,} : {(file_bytes_processed / input_file[1]) * 100:.0f}% : {(total_bytes_processed / total_size) * 100:.0f}%") + + if len(inserts) >= 0: + collection.insert_many(inserts) + total_lines += file_lines + total_bytes_processed += input_file[1] + log.info(f"{created.strftime('%Y-%m-%d %H:%M:%S')} : {total_lines:,} : 100% : {(total_bytes_processed / total_size) * 100:.0f}%") + + log.info(f"Total: {total_lines}") diff --git a/personal/split_by_subreddit.py b/personal/split_by_subreddit.py new file mode 100644 index 0000000..b5d4884 --- /dev/null +++ b/personal/split_by_subreddit.py @@ -0,0 +1,33 @@ +import utils +import discord_logging +import os +from datetime import datetime + +log = discord_logging.init_logging() + + +if __name__ == "__main__": + subreddits = {} + object_type = "submissions" + folder = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\{object_type}" + if not os.path.exists(folder): + os.makedirs(folder) + input_file = f"\\\\MYCLOUDPR4100\\Public\\reddit_final\\relationships_{object_type}.zst" + input_file_size = os.stat(input_file).st_size + total_lines = 0 + for comment, line, file_bytes_processed in utils.read_obj_zst_meta(input_file): + if comment['subreddit'] not in subreddits: + subreddits[comment['subreddit']] = {'writer': utils.OutputZst(os.path.join(folder, comment['subreddit'] + f"_{object_type}.zst")), 'lines': 0} + subreddit = subreddits[comment['subreddit']] + subreddit['writer'].write(line) + subreddit['writer'].write("\n") + subreddit['lines'] += 1 + total_lines += 1 + if total_lines % 100000 == 0: + log.info(f"{total_lines:,} lines, {(file_bytes_processed / input_file_size) * 100:.0f}%") + + log.info(f"{total_lines:,} lines, 100%") + + for name, subreddit in subreddits.items(): + log.info(f"r/{name}: {subreddit['lines']:,} lines") + subreddit['writer'].close() diff --git a/personal/test_file.py b/personal/test_file.py new file mode 100644 index 0000000..50222ec --- /dev/null +++ b/personal/test_file.py @@ -0,0 +1,25 @@ +import utils +import discord_logging +import os +import sys +from datetime import datetime + +log = discord_logging.init_logging() + + +if __name__ == "__main__": + file_path = r"\\MYCLOUDPR4100\Public\reddit\submissions\RS_2011-01.zst" + file_size = os.stat(file_path).st_size + + file_lines = 0 + file_bytes_processed = 0 + created = None + inserts = [] + for obj, line, file_bytes_processed in utils.read_obj_zst_meta(file_path): + created = datetime.utcfromtimestamp(int(obj['created_utc'])) + file_lines += 1 + if file_lines % 100000 == 0: + log.info(f"{created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines:,} : {(file_bytes_processed / file_size) * 100:.0f}%") + + log.info(f"{created.strftime('%Y-%m-%d %H:%M:%S')} : {file_lines:,} : 100%") + diff --git a/personal/utils.py b/personal/utils.py new file mode 100644 index 0000000..746669f --- /dev/null +++ b/personal/utils.py @@ -0,0 +1,60 @@ +import zstandard +import json + + +def read_obj_zst(file_name): + with open(file_name, 'rb') as file_handle: + buffer = '' + reader = zstandard.ZstdDecompressor(max_window_size=2**31).stream_reader(file_handle) + while True: + chunk = reader.read(2**27).decode() + if not chunk: + break + lines = (buffer + chunk).split("\n") + + for line in lines[:-1]: + yield json.loads(line) + + buffer = lines[-1] + reader.close() + + +def read_obj_zst_meta(file_name): + with open(file_name, 'rb') as file_handle: + buffer = '' + reader = zstandard.ZstdDecompressor(max_window_size=2**31).stream_reader(file_handle) + while True: + chunk = reader.read(2**27).decode() + if not chunk: + break + lines = (buffer + chunk).split("\n") + + for line in lines[:-1]: + try: + json_object = json.loads(line) + except (KeyError, json.JSONDecodeError) as err: + continue + yield json_object, line, file_handle.tell() + + buffer = lines[-1] + reader.close() + + +class OutputZst: + def __init__(self, file_name): + output_file = open(file_name, 'wb') + self.writer = zstandard.ZstdCompressor().stream_writer(output_file) + + def write(self, line): + encoded_line = line.encode('utf-8') + self.writer.write(encoded_line) + + def close(self): + self.writer.close() + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc_value, exc_traceback): + self.close() + return True