From 629f263cf501b6bd8f01e0285c4429f3c3079f67 Mon Sep 17 00:00:00 2001 From: El RIDO Date: Tue, 11 Mar 2025 07:59:10 +0100 Subject: [PATCH 1/5] pass by reference, closes #858 --- CHANGELOG.md | 1 + lib/Data/Filesystem.php | 9 ++++----- lib/Data/GoogleCloudStorage.php | 3 ++- lib/FormatV2.php | 2 +- lib/I18n.php | 12 ++++++++---- lib/Json.php | 4 ++-- lib/Model/AbstractModel.php | 9 ++++----- lib/Model/Comment.php | 6 ++---- lib/Model/Paste.php | 8 +++----- lib/Request.php | 5 ++--- tst/FormatV2Test.php | 10 ++++++---- tst/ModelTest.php | 3 ++- 12 files changed, 37 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fbe5818b..ce4408ee 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## 1.7.7 (not yet released) * ADDED: Switching templates using the web ui (#1501) +* CHANGED: Passing large data structures by reference to reduce memory consumption (#858) * CHANGED: Upgrading libraries to: ip-lib 1.20.0 ## 1.7.6 (2025-02-01) diff --git a/lib/Data/Filesystem.php b/lib/Data/Filesystem.php index 506ab4ee..be32c6ec 100644 --- a/lib/Data/Filesystem.php +++ b/lib/Data/Filesystem.php @@ -343,12 +343,11 @@ class Filesystem extends AbstractData */ private function _get($filename) { - return Json::decode( - substr( - file_get_contents($filename), - strlen(self::PROTECTION_LINE . PHP_EOL) - ) + $data = substr( + file_get_contents($filename), + strlen(self::PROTECTION_LINE . PHP_EOL) ); + return Json::decode($data); } /** diff --git a/lib/Data/GoogleCloudStorage.php b/lib/Data/GoogleCloudStorage.php index 45af05ee..c4354e83 100644 --- a/lib/Data/GoogleCloudStorage.php +++ b/lib/Data/GoogleCloudStorage.php @@ -219,7 +219,8 @@ class GoogleCloudStorage extends AbstractData $prefix = $this->_getKey($pasteid) . '/discussion/'; try { foreach ($this->_bucket->objects(array('prefix' => $prefix)) as $key) { - $comment = JSON::decode($this->_bucket->object($key->name())->downloadAsString()); + $data = $this->_bucket->object($key->name())->downloadAsString(); + $comment = Json::decode($data); $comment['id'] = basename($key->name()); $slot = $this->getOpenSlot($comments, (int) $comment['meta']['created']); $comments[$slot] = $comment; diff --git a/lib/FormatV2.php b/lib/FormatV2.php index 070c820a..6660c4f3 100644 --- a/lib/FormatV2.php +++ b/lib/FormatV2.php @@ -29,7 +29,7 @@ class FormatV2 * @param bool $isComment * @return bool */ - public static function isValid($message, $isComment = false) + public static function isValid(&$message, $isComment = false) { $required_keys = array('adata', 'v', 'ct'); if ($isComment) { diff --git a/lib/I18n.php b/lib/I18n.php index 2ef9effd..e99946f6 100644 --- a/lib/I18n.php +++ b/lib/I18n.php @@ -183,9 +183,12 @@ class I18n // load translations self::$_language = $match; - self::$_translations = ($match == 'en') ? array() : Json::decode( - file_get_contents(self::_getPath($match . '.json')) - ); + if ($match == 'en') { + self::$_translations = array(); + } else { + $data = file_get_contents(self::_getPath($match . '.json')); + self::$_translations = Json::decode($data); + } } /** @@ -273,7 +276,8 @@ class I18n { $file = self::_getPath('languages.json'); if (count(self::$_languageLabels) == 0 && is_readable($file)) { - self::$_languageLabels = Json::decode(file_get_contents($file)); + $data = file_get_contents($file); + self::$_languageLabels = Json::decode($data); } if (count($languages) == 0) { return self::$_languageLabels; diff --git a/lib/Json.php b/lib/Json.php index 9985f16e..cc405575 100644 --- a/lib/Json.php +++ b/lib/Json.php @@ -29,7 +29,7 @@ class Json * @throws Exception * @return string */ - public static function encode($input) + public static function encode(&$input) { $jsonString = json_encode($input); self::_detectError(); @@ -45,7 +45,7 @@ class Json * @throws Exception * @return mixed */ - public static function decode($input) + public static function decode(&$input) { $output = json_decode($input, true); self::_detectError(); diff --git a/lib/Model/AbstractModel.php b/lib/Model/AbstractModel.php index fb43400e..71f7c14a 100644 --- a/lib/Model/AbstractModel.php +++ b/lib/Model/AbstractModel.php @@ -100,9 +100,9 @@ abstract class AbstractModel * @param array $data * @throws Exception */ - public function setData(array $data) + public function setData(array &$data) { - $data = $this->_sanitize($data); + $this->_sanitize($data); $this->_validate($data); $this->_data = $data; @@ -163,9 +163,8 @@ abstract class AbstractModel * * @access protected * @param array $data - * @return array */ - abstract protected function _sanitize(array $data); + abstract protected function _sanitize(array &$data); /** * Validate data. @@ -174,7 +173,7 @@ abstract class AbstractModel * @param array $data * @throws Exception */ - protected function _validate(array $data) + protected function _validate(array &$data) { } } diff --git a/lib/Model/Comment.php b/lib/Model/Comment.php index 5250eb83..eb4d371d 100644 --- a/lib/Model/Comment.php +++ b/lib/Model/Comment.php @@ -104,7 +104,7 @@ class Comment extends AbstractModel * @param Paste $paste * @throws Exception */ - public function setPaste(Paste $paste) + public function setPaste(Paste &$paste) { $this->_paste = $paste; $this->_data['pasteid'] = $paste->getId(); @@ -155,9 +155,8 @@ class Comment extends AbstractModel * * @access protected * @param array $data - * @return array */ - protected function _sanitize(array $data) + protected function _sanitize(array &$data) { // we generate an icon based on a SHA512 HMAC of the users IP, if configured $icon = $this->_conf->getKey('icon'); @@ -190,6 +189,5 @@ class Comment extends AbstractModel $data['meta']['icon'] = $pngdata; } } - return $data; } } diff --git a/lib/Model/Paste.php b/lib/Model/Paste.php index 76dc4b4c..d95dc7c0 100644 --- a/lib/Model/Paste.php +++ b/lib/Model/Paste.php @@ -219,11 +219,10 @@ class Paste extends AbstractModel * * @access protected * @param array $data - * @return array */ - protected function _sanitize(array $data) + protected function _sanitize(array &$data) { - $expiration = $data['meta']['expire']; + $expiration = $data['meta']['expire'] ?? 0; unset($data['meta']['expire']); $expire_options = $this->_conf->getSection('expire_options'); if (array_key_exists($expiration, $expire_options)) { @@ -235,7 +234,6 @@ class Paste extends AbstractModel if ($expire > 0) { $data['meta']['expire_date'] = time() + $expire; } - return $data; } /** @@ -245,7 +243,7 @@ class Paste extends AbstractModel * @param array $data * @throws Exception */ - protected function _validate(array $data) + protected function _validate(array &$data) { // reject invalid or disabled formatters if (!array_key_exists($data['adata'][1], $this->_conf->getSection('formatter_options'))) { diff --git a/lib/Request.php b/lib/Request.php index 53a8adb0..0a2c11d5 100644 --- a/lib/Request.php +++ b/lib/Request.php @@ -110,9 +110,8 @@ class Request // it might be a creation or a deletion, the latter is detected below $this->_operation = 'create'; try { - $this->_params = Json::decode( - file_get_contents(self::$_inputStream) - ); + $data = file_get_contents(self::$_inputStream); + $this->_params = Json::decode($data); } catch (Exception $e) { // ignore error, $this->_params will remain empty } diff --git a/tst/FormatV2Test.php b/tst/FormatV2Test.php index cf6cf565..2e13d8f5 100644 --- a/tst/FormatV2Test.php +++ b/tst/FormatV2Test.php @@ -7,10 +7,11 @@ class FormatV2Test extends TestCase { public function testFormatV2ValidatorValidatesCorrectly() { - $this->assertTrue(FormatV2::isValid(Helper::getPastePost()), 'valid format'); - $this->assertTrue(FormatV2::isValid(Helper::getCommentPost(), true), 'valid format'); + $paste = Helper::getPastePost(); + $comment = Helper::getCommentPost(); + $this->assertTrue(FormatV2::isValid($paste), 'valid format'); + $this->assertTrue(FormatV2::isValid($comment, true), 'valid format'); - $paste = Helper::getPastePost(); $paste['adata'][0][0] = '$'; $this->assertFalse(FormatV2::isValid($paste), 'invalid base64 encoding of iv'); @@ -68,6 +69,7 @@ class FormatV2Test extends TestCase $paste['adata'][0][7] = '!#@'; $this->assertFalse(FormatV2::isValid($paste), 'invalid compression'); - $this->assertFalse(FormatV2::isValid(Helper::getPaste()), 'invalid meta key'); + $paste = Helper::getPaste(); + $this->assertFalse(FormatV2::isValid($paste), 'invalid meta key'); } } diff --git a/tst/ModelTest.php b/tst/ModelTest.php index 04609e22..99acba1f 100644 --- a/tst/ModelTest.php +++ b/tst/ModelTest.php @@ -163,7 +163,8 @@ class ModelTest extends TestCase $this->_conf->getSection('model_options') ) ); - $comment->setPaste($this->_model->getPaste(Helper::getPasteId())); + $paste = $this->_model->getPaste(Helper::getPasteId()); + $comment->setPaste($paste); $this->assertEquals(Helper::getPasteId(), $comment->getParentId(), 'comment parent ID gets initialized to paste ID'); } From 7825471d70c39baf6042c52a453c8fe705d9ed75 Mon Sep 17 00:00:00 2001 From: El RIDO Date: Thu, 13 Mar 2025 08:14:01 +0100 Subject: [PATCH 2/5] avoid duplication of ID check --- lib/Model/AbstractModel.php | 2 +- lib/Request.php | 3 ++- tst/ModelTest.php | 6 +++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/Model/AbstractModel.php b/lib/Model/AbstractModel.php index 71f7c14a..59edb10d 100644 --- a/lib/Model/AbstractModel.php +++ b/lib/Model/AbstractModel.php @@ -155,7 +155,7 @@ abstract class AbstractModel */ public static function isValidId($id) { - return (bool) preg_match('#\A[a-f\d]{16}\z#', (string) $id); + return (bool) preg_match('#\A[a-f0-9]{16}\z#', (string) $id); } /** diff --git a/lib/Request.php b/lib/Request.php index 0a2c11d5..c72a59fe 100644 --- a/lib/Request.php +++ b/lib/Request.php @@ -12,6 +12,7 @@ namespace PrivateBin; use Exception; +use PrivateBin\Model\Paste; /** * Request @@ -84,7 +85,7 @@ class Request foreach ($_GET as $key => $value) { // only return if value is empty and key is 16 hex chars $key = (string) $key; - if (($value === '') && strlen($key) === 16 && ctype_xdigit($key)) { + if (empty($value) && Paste::isValidId($key)) { return $key; } } diff --git a/tst/ModelTest.php b/tst/ModelTest.php index 99acba1f..301856bc 100644 --- a/tst/ModelTest.php +++ b/tst/ModelTest.php @@ -317,7 +317,11 @@ class ModelTest extends TestCase public function testPasteIdValidation() { $this->assertTrue(Paste::isValidId('a242ab7bdfb2581a'), 'valid paste id'); - $this->assertFalse(Paste::isValidId('foo'), 'invalid hex values'); + $this->assertFalse(Paste::isValidId('foo'), 'invalid hex values & length'); + $this->assertFalse(Paste::isValidId('f00'), 'invalid length'); + $this->assertFalse(Paste::isValidId('foo bar baz quux'), 'invalid hex values'); + $this->assertFalse(Paste::isValidId("\n01234567feedcafe"), 'invalid line breaks'); + $this->assertFalse(Paste::isValidId("deadbeef01234567\n"), 'invalid line breaks'); $this->assertFalse(Paste::isValidId('../bar/baz'), 'path attack'); } From 5d4561bd0a5f333bc943dded9b4899b28de1b632 Mon Sep 17 00:00:00 2001 From: El RIDO Date: Thu, 13 Mar 2025 08:29:53 +0100 Subject: [PATCH 3/5] drop ctype requirement (only one use left) --- CHANGELOG.md | 1 + composer.json | 3 +- composer.lock | 93 +------ doc/Installation.md | 1 - lib/View.php | 2 +- vendor/composer/autoload_classmap.php | 2 +- vendor/composer/autoload_files.php | 1 - vendor/composer/autoload_psr4.php | 1 - vendor/composer/autoload_static.php | 8 +- vendor/composer/installed.php | 19 +- .../ip-lib/src/Address/AddressInterface.php | 26 ++ vendor/mlocati/ip-lib/src/Address/IPv4.php | 58 +++++ vendor/mlocati/ip-lib/src/Address/IPv6.php | 58 +++++ .../ip-lib/src/Range/AbstractRange.php | 45 ++++ vendor/mlocati/ip-lib/src/Range/Pattern.php | 6 +- .../ip-lib/src/Range/RangeInterface.php | 26 ++ vendor/mlocati/ip-lib/src/Range/Single.php | 22 +- vendor/mlocati/ip-lib/src/Range/Subnet.php | 5 +- vendor/symfony/polyfill-ctype/Ctype.php | 232 ------------------ vendor/symfony/polyfill-ctype/bootstrap.php | 50 ---- vendor/symfony/polyfill-ctype/bootstrap80.php | 46 ---- 21 files changed, 252 insertions(+), 453 deletions(-) delete mode 100644 vendor/symfony/polyfill-ctype/Ctype.php delete mode 100644 vendor/symfony/polyfill-ctype/bootstrap.php delete mode 100644 vendor/symfony/polyfill-ctype/bootstrap80.php diff --git a/CHANGELOG.md b/CHANGELOG.md index ce4408ee..8c987850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## 1.7.7 (not yet released) * ADDED: Switching templates using the web ui (#1501) * CHANGED: Passing large data structures by reference to reduce memory consumption (#858) +* CHANGED: Removed use of ctype functions and polyfill library for ctype * CHANGED: Upgrading libraries to: ip-lib 1.20.0 ## 1.7.6 (2025-02-01) diff --git a/composer.json b/composer.json index 5188b14a..47aa2ef8 100644 --- a/composer.json +++ b/composer.json @@ -27,8 +27,7 @@ "php": "^7.3 || ^8.0", "jdenticon/jdenticon": "1.0.2", "mlocati/ip-lib": "1.20.0", - "symfony/polyfill-ctype": "^1.31", - "symfony/polyfill-php80": "^1.31", + "symfony/polyfill-php80": "1.31.0", "yzalis/identicon": "2.0.0" }, "suggest" : { diff --git a/composer.lock b/composer.lock index f1cedb0f..dfb58c46 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "b6e6a0433b36e6c81fcb3cb58b22a269", + "content-hash": "6c7e6dea19e8bfd5641b220cb68c4b65", "packages": [ { "name": "jdenticon/jdenticon", @@ -126,85 +126,6 @@ ], "time": "2025-02-04T17:30:58+00:00" }, - { - "name": "symfony/polyfill-ctype", - "version": "v1.31.0", - "source": { - "type": "git", - "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "a3cc8b044a6ea513310cbd48ef7333b384945638" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/a3cc8b044a6ea513310cbd48ef7333b384945638", - "reference": "a3cc8b044a6ea513310cbd48ef7333b384945638", - "shasum": "" - }, - "require": { - "php": ">=7.2" - }, - "provide": { - "ext-ctype": "*" - }, - "suggest": { - "ext-ctype": "For best performance" - }, - "type": "library", - "extra": { - "thanks": { - "url": "https://github.com/symfony/polyfill", - "name": "symfony/polyfill" - } - }, - "autoload": { - "files": [ - "bootstrap.php" - ], - "psr-4": { - "Symfony\\Polyfill\\Ctype\\": "" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Gert de Pagter", - "email": "BackEndTea@gmail.com" - }, - { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" - } - ], - "description": "Symfony polyfill for ctype functions", - "homepage": "https://symfony.com", - "keywords": [ - "compatibility", - "ctype", - "polyfill", - "portable" - ], - "support": { - "source": "https://github.com/symfony/polyfill-ctype/tree/v1.31.0" - }, - "funding": [ - { - "url": "https://symfony.com/sponsor", - "type": "custom" - }, - { - "url": "https://github.com/fabpot", - "type": "github" - }, - { - "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", - "type": "tidelift" - } - ], - "time": "2024-09-09T11:45:10+00:00" - }, { "name": "symfony/polyfill-php80", "version": "v1.31.0", @@ -416,16 +337,16 @@ }, { "name": "myclabs/deep-copy", - "version": "1.12.1", + "version": "1.13.0", "source": { "type": "git", "url": "https://github.com/myclabs/DeepCopy.git", - "reference": "123267b2c49fbf30d78a7b2d333f6be754b94845" + "reference": "024473a478be9df5fdaca2c793f2232fe788e414" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/123267b2c49fbf30d78a7b2d333f6be754b94845", - "reference": "123267b2c49fbf30d78a7b2d333f6be754b94845", + "url": "https://api.github.com/repos/myclabs/DeepCopy/zipball/024473a478be9df5fdaca2c793f2232fe788e414", + "reference": "024473a478be9df5fdaca2c793f2232fe788e414", "shasum": "" }, "require": { @@ -464,7 +385,7 @@ ], "support": { "issues": "https://github.com/myclabs/DeepCopy/issues", - "source": "https://github.com/myclabs/DeepCopy/tree/1.12.1" + "source": "https://github.com/myclabs/DeepCopy/tree/1.13.0" }, "funding": [ { @@ -472,7 +393,7 @@ "type": "tidelift" } ], - "time": "2024-11-08T17:47:46+00:00" + "time": "2025-02-12T12:17:51+00:00" }, { "name": "nikic/php-parser", diff --git a/doc/Installation.md b/doc/Installation.md index f9f4fac5..57cf7622 100644 --- a/doc/Installation.md +++ b/doc/Installation.md @@ -22,7 +22,6 @@ for more information. ### Minimal Requirements - PHP version 7.3 or above -- ctype extension - GD extension (when using identicon or vizhash icons, jdenticon works without it) - zlib extension - some disk space or a database supported by [PDO](https://php.net/manual/book.pdo.php) diff --git a/lib/View.php b/lib/View.php index 958be833..666a03f7 100644 --- a/lib/View.php +++ b/lib/View.php @@ -70,7 +70,7 @@ class View $sri = array_key_exists($file, $this->_variables['SRI']) ? ' integrity="' . $this->_variables['SRI'][$file] . '"' : ''; // if the file isn't versioned (ends in a digit), add our own version - $cacheBuster = ctype_digit(substr($file, -4, 1)) ? + $cacheBuster = (bool) preg_match('#[0-9]\.js$#', (string) $file) ? '' : '?' . rawurlencode($this->_variables['VERSION']); echo '