From e1ea14627f8a4dafba4e6db5023b991d0ef6c89c Mon Sep 17 00:00:00 2001 From: El RIDO Date: Sun, 26 Mar 2017 06:47:57 +0200 Subject: [PATCH] handling JSVerify RNG state 88caf85079d32e416b --- js/test.js | 2 +- tst/README.md | 69 ++++++++++++++++++++++++++++++++++++++++++++++++--- 2 files changed, 66 insertions(+), 5 deletions(-) diff --git a/js/test.js b/js/test.js index f328b51d..6f193f1d 100644 --- a/js/test.js +++ b/js/test.js @@ -277,7 +277,7 @@ describe('Helper', function () { var key = item.replace(/[\s;,=]/g, Array(i+2).join('£')), value = (values[i] || values[0]).replace(/[\s;,=]/g, ''); cookieArray.push(key + '=' + value); - if (Math.random() < 1 / i) + if (Math.random() < 1 / i || selectedKey === key) { selectedKey = key; selectedValue = value; diff --git a/tst/README.md b/tst/README.md index 76e69ee1..e11bc495 100644 --- a/tst/README.md +++ b/tst/README.md @@ -2,7 +2,7 @@ Running PHP unit tests ====================== In order to run these tests, you will need to install the following packages -and its dependencies: +and their dependencies: * phpunit * php-gd * php-sqlite3 @@ -13,12 +13,30 @@ Example for Debian and Ubuntu: $ sudo apt install phpunit php-gd php-sqlite php-xdebug ``` -To run the tests, just change into this directory and run phpunit: +To run the tests, change into the `tst` directory and run phpunit: ```console $ cd PrivateBin/tst $ phpunit ``` +Additionally there is the `ConfigurationTestGenerator`. Based on the +configurations defined in its constructor, it generates the unit test file +`tst/ConfigurationCombinationsTest.php`, containing all possible combinations +of these configurations and tests for (most of the) valid combinations. Some of +combinations can't be tested with this method, i.e. a valid option combined with +an invalid one. Other very specific test cases (i.e. to trigger multiple errors) +are covered in `tst/PrivateBinTest.php`. Here is how to generate the +configuration test and run it: + +```console +$ cd PrivateBin/tst +$ php ConfigurationTestGenerator.php +$ phpunit ConfigurationCombinationsTest.php +``` + +Note that it can take an hour or longer to run the several thousand tests. + + Running JavaScript unit tests ============================= @@ -36,8 +54,8 @@ $ cd PrivateBin/js $ npm install jsverify jsdom jsdom-global ``` -Example for Debian and Ubuntu, including steps to allow current user to install -node modules globally: +Example for Debian and Ubuntu, including steps to allow the current user to +install node modules globally: ```console $ sudo apt install npm $ sudo mkdir /usr/local/lib/node_modules @@ -54,3 +72,46 @@ $ cd PrivateBin/js $ istanbul cover _mocha ``` +Property based unit testing +--------------------------- + +In the JavaScript unit tests we use the JSVerify library to leverage property +based unit testing. Instead of artificially creating specific test cases to +cover all relevant paths of the tested code (with the generated coverage reports +providing means to check the tested paths), property based testing allows us to +describe the patterns of data that are valid input. + +With each run of the tests, for each `jsc.property` 100 random inputs are +generated and tested. For example we tell the test to generate random strings, +which will include empty strings, numeric strings, long strings, unicode +sequences, etc. This is great for finding corner cases that one might not think +of when explicitly writing one test case at a time. + +There is another benefit, too: When an error is found, JSVerify will try to find +the smallest, still failing test case for you and print this out including the +associated random number generator (RNG) state, so you can reproduce it easily: + +```console +[...] + + 30 passing (3s) + 1 failing + + 1) Helper getCookie returns the requested cookie: + Error: Failed after 30 tests and 11 shrinks. rngState: 88caf85079d32e416b; Counterexample: ["{", "9", "9", "YD8%fT"]; [" ", "_|K:"]; + +[...] +``` + +Of course it may just be that you need to adjust a test case if the random +pattern generated is ambiguous. In the above example the cookie string would +contain two identical keys "9", something that may not be valid, but that our +code could encounter and needs to be able to handle. + +After you adjusted the code of the library or the test you can rerun the test +with the same RNG state as follows: + +```console +$ istanbul cover _mocha -- test.js --jsverifyRngState 88caf85079d32e416b +``` +