From fdc0eafc679a21c54e0ef0666109e697fd142da7 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Fri, 22 Apr 2016 09:29:12 +0200
Subject: [PATCH 001/101] Add skeleton for network stress unit test

---
 .../p2p/network/NetworkStressTest.java        | 55 +++++++++++++++++++
 1 file changed, 55 insertions(+)
 create mode 100644 network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
new file mode 100644
index 0000000000..6902cfc459
--- /dev/null
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -0,0 +1,55 @@
+package io.bitsquare.p2p.network;
+
+import org.jetbrains.annotations.NotNull;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.nio.file.FileVisitResult;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.SimpleFileVisitor;
+import java.nio.file.attribute.BasicFileAttributes;
+
+public class NetworkStressTest {
+    private Path tempDir;
+
+    @Before
+    public void setup() throws IOException {
+        tempDir = createTempDirectory();
+    }
+
+    @After
+    public void tearDown() throws IOException {
+        if (tempDir != null) {
+            deleteRecursively(tempDir);
+        }
+    }
+
+    @Test
+    public void test() {
+        // do nothing
+    }
+
+    private Path createTempDirectory() throws IOException {
+        return Files.createTempDirectory("bsq" + this.getClass().getSimpleName());
+    }
+
+    private static void deleteRecursively(@NotNull final Path path) throws IOException {
+        // Based on <https://stackoverflow.com/a/27917071/6239236> by Tomasz Dzięcielewski.
+        Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                Files.delete(path);
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+                Files.delete(dir);
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+}

From a9d0272e886a18991fa1f9b1698b11faa6434958 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Fri, 22 Apr 2016 11:28:49 +0200
Subject: [PATCH 002/101] Start seed node in stress test, not working yet
 (locking)

---
 .../p2p/network/NetworkStressTest.java        | 83 ++++++++++++++++++-
 1 file changed, 81 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 6902cfc459..c9ad657b05 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -1,5 +1,10 @@
 package io.bitsquare.p2p.network;
 
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import io.bitsquare.common.UserThread;
+import io.bitsquare.p2p.NodeAddress;
+import io.bitsquare.p2p.P2PServiceListener;
+import io.bitsquare.p2p.seed.SeedNode;
 import org.jetbrains.annotations.NotNull;
 import org.junit.After;
 import org.junit.Before;
@@ -11,17 +16,91 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
 
 public class NetworkStressTest {
     private Path tempDir;
+    private SeedNode seedNode;
 
     @Before
-    public void setup() throws IOException {
+    public void setup() throws IOException, InterruptedException {
+        // Use an executor that uses a single daemon thread.
+        final ThreadFactory threadFactory = new ThreadFactoryBuilder()
+                .setNameFormat(this.getClass().getSimpleName())
+                .setDaemon(true)
+                .build();
+        UserThread.setExecutor(Executors.newSingleThreadExecutor(threadFactory));
+
         tempDir = createTempDirectory();
+        seedNode = new SeedNode(tempDir.toString());
+
+        // Use as a barrier to wait for concurrent tasks.
+        final CountDownLatch latch = new CountDownLatch(1 /*seed node*/);
+        // Start the seed node.
+        final NodeAddress seedNodeAddress = new NodeAddress("localhost:8002");
+        UserThread.execute(() -> {
+            try {
+                seedNode.createAndStartP2PService(seedNodeAddress, true /*localhost*/,
+                        2 /*regtest*/, false /*detailed logging*/, null /*seed nodes*/,
+                        new P2PServiceListener() {
+                            @Override
+                            public void onRequestingDataCompleted() {
+                                // do nothing
+                            }
+
+                            @Override
+                            public void onNoSeedNodeAvailable() {
+                                // expected, do nothing
+                            }
+
+                            @Override
+                            public void onNoPeersAvailable() {
+                                // expected, do nothing
+                            }
+
+                            @Override
+                            public void onBootstrapComplete() {
+                                latch.countDown();  // one less task to wait on
+                            }
+
+                            @Override
+                            public void onTorNodeReady() {
+                                // do nothing
+                            }
+
+                            @Override
+                            public void onHiddenServicePublished() {
+                                // do nothing
+                            }
+
+                            @Override
+                            public void onSetupFailed(Throwable throwable) {
+                                //XXXX
+                            }
+                        });
+            } catch (Throwable t) {
+                //log.error("Executing task failed. " + t.getMessage());
+                t.printStackTrace();
+            }
+        });
+
+        // Wait for concurrent tasks to finish.
+        latch.await();
     }
 
     @After
-    public void tearDown() throws IOException {
+    public void tearDown() throws InterruptedException, IOException {
+        // Use as a barrier to wait for concurrent tasks.
+        final CountDownLatch latch = new CountDownLatch(1 /*seed node*/);
+        // Stop the seed node.
+        if (seedNode != null) {
+            seedNode.shutDown(() -> {latch.countDown();});
+        }
+        // Wait for concurrent tasks to finish.
+        latch.await();
+
         if (tempDir != null) {
             deleteRecursively(tempDir);
         }

From 0977ef398bacd6620b00bdb45fa7e746c896c516 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Fri, 22 Apr 2016 13:08:00 +0200
Subject: [PATCH 003/101] Move seed note running out of user thread, still
 locking

---
 .../p2p/network/NetworkStressTest.java        | 89 +++++++++----------
 1 file changed, 41 insertions(+), 48 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index c9ad657b05..1a20476bb8 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -16,7 +16,10 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.HashSet;
+import java.util.Set;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.ThreadFactory;
 
@@ -26,65 +29,55 @@ public class NetworkStressTest {
 
     @Before
     public void setup() throws IOException, InterruptedException {
-        // Use an executor that uses a single daemon thread.
-        final ThreadFactory threadFactory = new ThreadFactoryBuilder()
-                .setNameFormat(this.getClass().getSimpleName())
-                .setDaemon(true)
-                .build();
-        UserThread.setExecutor(Executors.newSingleThreadExecutor(threadFactory));
-
         tempDir = createTempDirectory();
         seedNode = new SeedNode(tempDir.toString());
+        final NodeAddress seedNodeAddress = new NodeAddress("localhost:8002");
+        final Set<NodeAddress> seedNodes = new HashSet<>(1);
+        seedNodes.add(seedNodeAddress);  // the only seed node in tests
 
         // Use as a barrier to wait for concurrent tasks.
         final CountDownLatch latch = new CountDownLatch(1 /*seed node*/);
         // Start the seed node.
-        final NodeAddress seedNodeAddress = new NodeAddress("localhost:8002");
-        UserThread.execute(() -> {
-            try {
-                seedNode.createAndStartP2PService(seedNodeAddress, true /*localhost*/,
-                        2 /*regtest*/, false /*detailed logging*/, null /*seed nodes*/,
-                        new P2PServiceListener() {
-                            @Override
-                            public void onRequestingDataCompleted() {
-                                // do nothing
-                            }
+        seedNode.createAndStartP2PService(seedNodeAddress, true /*localhost*/,
+                2 /*regtest*/, true /*detailed logging*/, seedNodes,
+                new P2PServiceListener() {
+                    @Override
+                    public void onRequestingDataCompleted() {
+                        // do nothing
+                    }
 
-                            @Override
-                            public void onNoSeedNodeAvailable() {
-                                // expected, do nothing
-                            }
+                    @Override
+                    public void onNoSeedNodeAvailable() {
+                        // expected, do nothing
+                    }
 
-                            @Override
-                            public void onNoPeersAvailable() {
-                                // expected, do nothing
-                            }
+                    @Override
+                    public void onNoPeersAvailable() {
+                        // expected, do nothing
+                    }
 
-                            @Override
-                            public void onBootstrapComplete() {
-                                latch.countDown();  // one less task to wait on
-                            }
+                    @Override
+                    public void onBootstrapComplete() {
+                        // do nothing
+                    }
 
-                            @Override
-                            public void onTorNodeReady() {
-                                // do nothing
-                            }
+                    @Override
+                    public void onTorNodeReady() {
+                        // do nothing
+                        System.out.println("TOR NODE READY");
+                    }
 
-                            @Override
-                            public void onHiddenServicePublished() {
-                                // do nothing
-                            }
+                    @Override
+                    public void onHiddenServicePublished() {
+                        latch.countDown();  // one less task to wait on
+                    }
 
-                            @Override
-                            public void onSetupFailed(Throwable throwable) {
-                                //XXXX
-                            }
-                        });
-            } catch (Throwable t) {
-                //log.error("Executing task failed. " + t.getMessage());
-                t.printStackTrace();
-            }
-        });
+                    @Override
+                    public void onSetupFailed(Throwable throwable) {
+                        //XXXX
+                    }
+                }
+        );
 
         // Wait for concurrent tasks to finish.
         latch.await();
@@ -96,7 +89,7 @@ public class NetworkStressTest {
         final CountDownLatch latch = new CountDownLatch(1 /*seed node*/);
         // Stop the seed node.
         if (seedNode != null) {
-            seedNode.shutDown(() -> {latch.countDown();});
+            seedNode.shutDown(latch::countDown);
         }
         // Wait for concurrent tasks to finish.
         latch.await();

From c9f0155473f6e7fe71922b9b9a37514e0bf8d9b4 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Fri, 22 Apr 2016 13:13:56 +0200
Subject: [PATCH 004/101] Remove unnecessary imports from network stress test

---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java     | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 1a20476bb8..1df7fc3882 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -1,7 +1,5 @@
 package io.bitsquare.p2p.network;
 
-import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import io.bitsquare.common.UserThread;
 import io.bitsquare.p2p.NodeAddress;
 import io.bitsquare.p2p.P2PServiceListener;
 import io.bitsquare.p2p.seed.SeedNode;
@@ -19,9 +17,6 @@ import java.nio.file.attribute.BasicFileAttributes;
 import java.util.HashSet;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.ThreadFactory;
 
 public class NetworkStressTest {
     private Path tempDir;

From 9f0dfb5512060aadb9016511bf2a7b6100aa0e14 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Fri, 22 Apr 2016 13:18:34 +0200
Subject: [PATCH 005/101] Use variable to indicate whether a localhost address
 is being used

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 1df7fc3882..f66e190875 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -27,13 +27,14 @@ public class NetworkStressTest {
         tempDir = createTempDirectory();
         seedNode = new SeedNode(tempDir.toString());
         final NodeAddress seedNodeAddress = new NodeAddress("localhost:8002");
+        final boolean useLocalhost = true;
         final Set<NodeAddress> seedNodes = new HashSet<>(1);
         seedNodes.add(seedNodeAddress);  // the only seed node in tests
 
         // Use as a barrier to wait for concurrent tasks.
         final CountDownLatch latch = new CountDownLatch(1 /*seed node*/);
         // Start the seed node.
-        seedNode.createAndStartP2PService(seedNodeAddress, true /*localhost*/,
+        seedNode.createAndStartP2PService(seedNodeAddress, useLocalhost,
                 2 /*regtest*/, true /*detailed logging*/, seedNodes,
                 new P2PServiceListener() {
                     @Override

From dc149f30e3fe288d94edcb02adbbf133c9659c74 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Fri, 22 Apr 2016 13:23:01 +0200
Subject: [PATCH 006/101] Wait for 30 seconds in test

To allow test connections.  Also remove stale print.
---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index f66e190875..e5f0e6edbd 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -60,7 +60,6 @@ public class NetworkStressTest {
                     @Override
                     public void onTorNodeReady() {
                         // do nothing
-                        System.out.println("TOR NODE READY");
                     }
 
                     @Override
@@ -96,8 +95,9 @@ public class NetworkStressTest {
     }
 
     @Test
-    public void test() {
+    public void test() throws InterruptedException {
         // do nothing
+        Thread.sleep(30_000);
     }
 
     private Path createTempDirectory() throws IOException {

From 718c935fef01dff33098df66f7a673c64fd51e64 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Mon, 25 Apr 2016 10:15:28 +0200
Subject: [PATCH 007/101] Some comments and better names in network stress test

---
 .../p2p/network/NetworkStressTest.java        | 25 +++++++++++--------
 1 file changed, 15 insertions(+), 10 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index e5f0e6edbd..d9fc6bd7ac 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -19,21 +19,26 @@ import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 
 public class NetworkStressTest {
+    /** A directory to temporarily hold seed and normal nodes' configuration and state files. */
     private Path tempDir;
+    /** A single seed node that other nodes will contact to request initial data. */
     private SeedNode seedNode;
 
     @Before
     public void setup() throws IOException, InterruptedException {
+        /** A barrier to wait for concurrent tasks. */
+        final CountDownLatch pendingTasks = new CountDownLatch(1 /*seed node*/);
+
+        // Create the temporary directory.
         tempDir = createTempDirectory();
+
+        // Create and start the seed node.
         seedNode = new SeedNode(tempDir.toString());
         final NodeAddress seedNodeAddress = new NodeAddress("localhost:8002");
         final boolean useLocalhost = true;
         final Set<NodeAddress> seedNodes = new HashSet<>(1);
         seedNodes.add(seedNodeAddress);  // the only seed node in tests
-
-        // Use as a barrier to wait for concurrent tasks.
-        final CountDownLatch latch = new CountDownLatch(1 /*seed node*/);
-        // Start the seed node.
+        boolean seedNodeSetupFailed = false;
         seedNode.createAndStartP2PService(seedNodeAddress, useLocalhost,
                 2 /*regtest*/, true /*detailed logging*/, seedNodes,
                 new P2PServiceListener() {
@@ -64,7 +69,7 @@ public class NetworkStressTest {
 
                     @Override
                     public void onHiddenServicePublished() {
-                        latch.countDown();  // one less task to wait on
+                        pendingTasks.countDown();  // one less task to wait on
                     }
 
                     @Override
@@ -75,19 +80,19 @@ public class NetworkStressTest {
         );
 
         // Wait for concurrent tasks to finish.
-        latch.await();
+        pendingTasks.await();
     }
 
     @After
     public void tearDown() throws InterruptedException, IOException {
-        // Use as a barrier to wait for concurrent tasks.
-        final CountDownLatch latch = new CountDownLatch(1 /*seed node*/);
+        /** A barrier to wait for concurrent tasks. */
+        final CountDownLatch pendingTasks = new CountDownLatch(1 /*seed node*/);
         // Stop the seed node.
         if (seedNode != null) {
-            seedNode.shutDown(latch::countDown);
+            seedNode.shutDown(pendingTasks::countDown);
         }
         // Wait for concurrent tasks to finish.
-        latch.await();
+        pendingTasks.await();
 
         if (tempDir != null) {
             deleteRecursively(tempDir);

From 260840a9b2ba3d078e57fbedffac756fa88917bf Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Mon, 25 Apr 2016 10:33:33 +0200
Subject: [PATCH 008/101] Do not wait on seed node if not created in network
 stress test

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index d9fc6bd7ac..c3280a01c6 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -86,7 +86,7 @@ public class NetworkStressTest {
     @After
     public void tearDown() throws InterruptedException, IOException {
         /** A barrier to wait for concurrent tasks. */
-        final CountDownLatch pendingTasks = new CountDownLatch(1 /*seed node*/);
+        final CountDownLatch pendingTasks = new CountDownLatch(seedNode != null? 1 : 0);
         // Stop the seed node.
         if (seedNode != null) {
             seedNode.shutDown(pendingTasks::countDown);

From 9c58d9f74087d5fbc28f5263c5773073b4108ab2 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Mon, 25 Apr 2016 10:53:00 +0200
Subject: [PATCH 009/101] Minor changes to network stress test

I was going to override the ``runBare()`` method to cleanup on ``setUp()`` failures, but ``tearDown()`` is called all the same.
---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index c3280a01c6..3d8c39ca76 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -25,7 +25,7 @@ public class NetworkStressTest {
     private SeedNode seedNode;
 
     @Before
-    public void setup() throws IOException, InterruptedException {
+    public void setUp() throws IOException, InterruptedException {
         /** A barrier to wait for concurrent tasks. */
         final CountDownLatch pendingTasks = new CountDownLatch(1 /*seed node*/);
 
@@ -87,6 +87,7 @@ public class NetworkStressTest {
     public void tearDown() throws InterruptedException, IOException {
         /** A barrier to wait for concurrent tasks. */
         final CountDownLatch pendingTasks = new CountDownLatch(seedNode != null? 1 : 0);
+
         // Stop the seed node.
         if (seedNode != null) {
             seedNode.shutDown(pendingTasks::countDown);

From 2c61596871f17cbbd507f8fe2ef5bbbc81744d07 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Mon, 25 Apr 2016 11:15:05 +0200
Subject: [PATCH 010/101] Add property to check if network stress test thread
 report setup failure

---
 .../p2p/network/NetworkStressTest.java         | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 3d8c39ca76..6a0a457353 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -3,6 +3,8 @@ package io.bitsquare.p2p.network;
 import io.bitsquare.p2p.NodeAddress;
 import io.bitsquare.p2p.P2PServiceListener;
 import io.bitsquare.p2p.seed.SeedNode;
+import javafx.beans.property.BooleanProperty;
+import javafx.beans.property.SimpleBooleanProperty;
 import org.jetbrains.annotations.NotNull;
 import org.junit.After;
 import org.junit.Before;
@@ -25,7 +27,9 @@ public class NetworkStressTest {
     private SeedNode seedNode;
 
     @Before
-    public void setUp() throws IOException, InterruptedException {
+    public void setUp() throws Exception {
+        /** A property where threads can indicate setup failure. */
+        BooleanProperty setupFailed = new SimpleBooleanProperty(false);
         /** A barrier to wait for concurrent tasks. */
         final CountDownLatch pendingTasks = new CountDownLatch(1 /*seed node*/);
 
@@ -38,7 +42,6 @@ public class NetworkStressTest {
         final boolean useLocalhost = true;
         final Set<NodeAddress> seedNodes = new HashSet<>(1);
         seedNodes.add(seedNodeAddress);  // the only seed node in tests
-        boolean seedNodeSetupFailed = false;
         seedNode.createAndStartP2PService(seedNodeAddress, useLocalhost,
                 2 /*regtest*/, true /*detailed logging*/, seedNodes,
                 new P2PServiceListener() {
@@ -69,18 +72,25 @@ public class NetworkStressTest {
 
                     @Override
                     public void onHiddenServicePublished() {
-                        pendingTasks.countDown();  // one less task to wait on
+                        // successful result
+                        pendingTasks.countDown();
                     }
 
                     @Override
                     public void onSetupFailed(Throwable throwable) {
-                        //XXXX
+                        // failed result
+                        setupFailed.set(true);
+                        pendingTasks.countDown();
                     }
                 }
         );
 
         // Wait for concurrent tasks to finish.
         pendingTasks.await();
+
+        // Check if nodes started correctly.
+        if (setupFailed.get())
+            throw new Exception("nodes failed to start");
     }
 
     @After

From 30e980748d0fc521b8875999907f2307f75f9b32 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Mon, 25 Apr 2016 11:35:57 +0200
Subject: [PATCH 011/101] Make methods for seed node address and setup listener
 creation

This will ease redefining node type (Tor or localhost) and creating more nodes that can report setup result.
---
 .../p2p/network/NetworkStressTest.java        | 96 +++++++++++--------
 1 file changed, 54 insertions(+), 42 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 6a0a457353..64850170f4 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -38,61 +38,73 @@ public class NetworkStressTest {
 
         // Create and start the seed node.
         seedNode = new SeedNode(tempDir.toString());
-        final NodeAddress seedNodeAddress = new NodeAddress("localhost:8002");
-        final boolean useLocalhost = true;
+        final NodeAddress seedNodeAddress = getSeedNodeAddress();
+        final boolean useLocalhost = seedNodeAddress.getFullAddress().startsWith("localhost:");
         final Set<NodeAddress> seedNodes = new HashSet<>(1);
         seedNodes.add(seedNodeAddress);  // the only seed node in tests
         seedNode.createAndStartP2PService(seedNodeAddress, useLocalhost,
                 2 /*regtest*/, true /*detailed logging*/, seedNodes,
-                new P2PServiceListener() {
-                    @Override
-                    public void onRequestingDataCompleted() {
-                        // do nothing
-                    }
-
-                    @Override
-                    public void onNoSeedNodeAvailable() {
-                        // expected, do nothing
-                    }
-
-                    @Override
-                    public void onNoPeersAvailable() {
-                        // expected, do nothing
-                    }
-
-                    @Override
-                    public void onBootstrapComplete() {
-                        // do nothing
-                    }
-
-                    @Override
-                    public void onTorNodeReady() {
-                        // do nothing
-                    }
-
-                    @Override
-                    public void onHiddenServicePublished() {
-                        // successful result
-                        pendingTasks.countDown();
-                    }
-
-                    @Override
-                    public void onSetupFailed(Throwable throwable) {
-                        // failed result
-                        setupFailed.set(true);
-                        pendingTasks.countDown();
-                    }
-                }
+                getSetupListener(setupFailed, pendingTasks)
         );
 
         // Wait for concurrent tasks to finish.
         pendingTasks.await();
 
-        // Check if nodes started correctly.
+        // Check if any node reported setup failure on start.
         if (setupFailed.get())
             throw new Exception("nodes failed to start");
     }
 
+    @NotNull
+    private static NodeAddress getSeedNodeAddress() {
+        return new NodeAddress("localhost:8002");
+    }
+
+    @NotNull
+    private static P2PServiceListener getSetupListener(
+            final BooleanProperty setupFailed,
+            final CountDownLatch pendingTasks) {
+        return new P2PServiceListener() {
+            @Override
+            public void onRequestingDataCompleted() {
+                // do nothing
+            }
+
+            @Override
+            public void onNoSeedNodeAvailable() {
+                // expected, do nothing
+            }
+
+            @Override
+            public void onNoPeersAvailable() {
+                // expected, do nothing
+            }
+
+            @Override
+            public void onBootstrapComplete() {
+                // do nothing
+            }
+
+            @Override
+            public void onTorNodeReady() {
+                // do nothing
+            }
+
+            @Override
+            public void onHiddenServicePublished() {
+                // successful result
+                pendingTasks.countDown();
+            }
+
+            @Override
+            public void onSetupFailed(Throwable throwable) {
+                // failed result
+                setupFailed.set(true);
+                pendingTasks.countDown();
+            }
+        };
+    }
+
     @After
     public void tearDown() throws InterruptedException, IOException {
         /** A barrier to wait for concurrent tasks. */

From f238870bce237c3d96efbd31142ab6091a6c56c5 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Mon, 25 Apr 2016 12:38:28 +0200
Subject: [PATCH 012/101] Extract constant with regtest network id

---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java   | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 64850170f4..749cea1f55 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -21,6 +21,8 @@ import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 
 public class NetworkStressTest {
+    private static final int REGTEST_NETWORK_ID = 2;
+
     /** A directory to temporarily hold seed and normal nodes' configuration and state files. */
     private Path tempDir;
     /** A single seed node that other nodes will contact to request initial data. */
@@ -43,9 +45,8 @@ public class NetworkStressTest {
         final Set<NodeAddress> seedNodes = new HashSet<>(1);
         seedNodes.add(seedNodeAddress);  // the only seed node in tests
         seedNode.createAndStartP2PService(seedNodeAddress, useLocalhost,
-                2 /*regtest*/, true /*detailed logging*/, seedNodes,
-                getSetupListener(setupFailed, pendingTasks)
-        );
+                REGTEST_NETWORK_ID, true /*detailed logging*/, seedNodes,
+                getSetupListener(setupFailed, pendingTasks));
 
         // Wait for concurrent tasks to finish.
         pendingTasks.await();

From f7625796c8d8ad8be0152d2575a111129567c9b2 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Mon, 25 Apr 2016 12:48:04 +0200
Subject: [PATCH 013/101] Tune creation of seed node address in network stress
 test

---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java     | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 749cea1f55..e8524c470e 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -2,6 +2,7 @@ package io.bitsquare.p2p.network;
 
 import io.bitsquare.p2p.NodeAddress;
 import io.bitsquare.p2p.P2PServiceListener;
+import io.bitsquare.p2p.Utils;
 import io.bitsquare.p2p.seed.SeedNode;
 import javafx.beans.property.BooleanProperty;
 import javafx.beans.property.SimpleBooleanProperty;
@@ -41,7 +42,7 @@ public class NetworkStressTest {
         // Create and start the seed node.
         seedNode = new SeedNode(tempDir.toString());
         final NodeAddress seedNodeAddress = getSeedNodeAddress();
-        final boolean useLocalhost = seedNodeAddress.getFullAddress().startsWith("localhost:");
+        final boolean useLocalhost = seedNodeAddress.hostName.equals("localhost");
         final Set<NodeAddress> seedNodes = new HashSet<>(1);
         seedNodes.add(seedNodeAddress);  // the only seed node in tests
         seedNode.createAndStartP2PService(seedNodeAddress, useLocalhost,
@@ -58,7 +59,7 @@ public class NetworkStressTest {
 
     @NotNull
     private static NodeAddress getSeedNodeAddress() {
-        return new NodeAddress("localhost:8002");
+        return new NodeAddress("localhost", Utils.findFreeSystemPort());
     }
 
     @NotNull

From 67e68cca85179f22c8eb205b0640c63c5b66f24d Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Mon, 25 Apr 2016 14:00:44 +0200
Subject: [PATCH 014/101] Add creation of peer nodes to network stress test

Only one peer for the moment, still not working since no encryption service and key ring are created.
---
 .../p2p/network/NetworkStressTest.java        | 40 +++++++++++++++++--
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index e8524c470e..702247fcef 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -1,9 +1,12 @@
 package io.bitsquare.p2p.network;
 
+import io.bitsquare.common.Clock;
 import io.bitsquare.p2p.NodeAddress;
+import io.bitsquare.p2p.P2PService;
 import io.bitsquare.p2p.P2PServiceListener;
 import io.bitsquare.p2p.Utils;
 import io.bitsquare.p2p.seed.SeedNode;
+import io.bitsquare.p2p.seed.SeedNodesRepository;
 import javafx.beans.property.BooleanProperty;
 import javafx.beans.property.SimpleBooleanProperty;
 import org.jetbrains.annotations.NotNull;
@@ -11,30 +14,38 @@ import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
 
+import java.io.File;
 import java.io.IOException;
 import java.nio.file.FileVisitResult;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.util.ArrayList;
 import java.util.HashSet;
+import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
 
 public class NetworkStressTest {
+    /** Numeric identifier of the regtest Bitcoin network. */
     private static final int REGTEST_NETWORK_ID = 2;
+    /** Number of peer nodes to create. */
+    private static final int NPEERS = 1;
 
     /** A directory to temporarily hold seed and normal nodes' configuration and state files. */
     private Path tempDir;
     /** A single seed node that other nodes will contact to request initial data. */
     private SeedNode seedNode;
+    /** A list of peer nodes represented as P2P services. */
+    private List<P2PService> peerNodes = new ArrayList<>();
 
     @Before
     public void setUp() throws Exception {
         /** A property where threads can indicate setup failure. */
         BooleanProperty setupFailed = new SimpleBooleanProperty(false);
         /** A barrier to wait for concurrent tasks. */
-        final CountDownLatch pendingTasks = new CountDownLatch(1 /*seed node*/);
+        final CountDownLatch pendingTasks = new CountDownLatch(1 /*seed node*/ + NPEERS);
 
         // Create the temporary directory.
         tempDir = createTempDirectory();
@@ -49,12 +60,31 @@ public class NetworkStressTest {
                 REGTEST_NETWORK_ID, true /*detailed logging*/, seedNodes,
                 getSetupListener(setupFailed, pendingTasks));
 
+        // Create and start peer nodes.
+        SeedNodesRepository seedNodesRepository = new SeedNodesRepository();
+        if (useLocalhost) {
+            seedNodesRepository.setLocalhostSeedNodeAddresses(seedNodes);
+        } else {
+            seedNodesRepository.setTorSeedNodeAddresses(seedNodes);
+        }
+        for (int p = 0; p < NPEERS; p++) {
+            final int peerPort = Utils.findFreeSystemPort();
+            final File peerDir = new File(tempDir.toFile(), "Bitsquare_peer_" + peerPort);
+            final File peerTorDir = new File(peerDir, "tor");
+            final File peerStorageDir = new File(peerDir, "db");
+            final P2PService peer = new P2PService(seedNodesRepository, peerPort, peerTorDir, useLocalhost,
+                    REGTEST_NETWORK_ID, peerStorageDir, new Clock(), null /*TODO:enc svc*/, null /*TODO:key ring*/);
+            peerNodes.add(peer);
+            peer.start(getSetupListener(setupFailed, pendingTasks));
+        }
+
         // Wait for concurrent tasks to finish.
         pendingTasks.await();
 
         // Check if any node reported setup failure on start.
-        if (setupFailed.get())
+        if (setupFailed.get()) {
             throw new Exception("nodes failed to start");
+        }
     }
 
     @NotNull
@@ -110,12 +140,16 @@ public class NetworkStressTest {
     @After
     public void tearDown() throws InterruptedException, IOException {
         /** A barrier to wait for concurrent tasks. */
-        final CountDownLatch pendingTasks = new CountDownLatch(seedNode != null? 1 : 0);
+        final CountDownLatch pendingTasks = new CountDownLatch((seedNode != null? 1 : 0) + peerNodes.size());
 
         // Stop the seed node.
         if (seedNode != null) {
             seedNode.shutDown(pendingTasks::countDown);
         }
+        // Stop peer nodes.
+        for (P2PService peer : peerNodes) {
+            peer.shutDown(pendingTasks::countDown);
+        }
         // Wait for concurrent tasks to finish.
         pendingTasks.await();
 

From 29c72cdc49a16438f5f7e65ede1b7294db41d0fb Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Mon, 25 Apr 2016 14:43:38 +0200
Subject: [PATCH 015/101] Create peer keys in network stress test

We should keep an eye on the setup phase getting too slow (and eating the computer's randomness) because of the generation of peer keys.
---
 .../bitsquare/p2p/network/NetworkStressTest.java | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 702247fcef..bf10ded459 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -1,6 +1,9 @@
 package io.bitsquare.p2p.network;
 
 import io.bitsquare.common.Clock;
+import io.bitsquare.common.crypto.KeyRing;
+import io.bitsquare.common.crypto.KeyStorage;
+import io.bitsquare.crypto.EncryptionService;
 import io.bitsquare.p2p.NodeAddress;
 import io.bitsquare.p2p.P2PService;
 import io.bitsquare.p2p.P2PServiceListener;
@@ -9,6 +12,7 @@ import io.bitsquare.p2p.seed.SeedNode;
 import io.bitsquare.p2p.seed.SeedNodesRepository;
 import javafx.beans.property.BooleanProperty;
 import javafx.beans.property.SimpleBooleanProperty;
+import org.bouncycastle.jce.provider.BouncyCastleProvider;
 import org.jetbrains.annotations.NotNull;
 import org.junit.After;
 import org.junit.Before;
@@ -21,6 +25,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.SimpleFileVisitor;
 import java.nio.file.attribute.BasicFileAttributes;
+import java.security.Security;
 import java.util.ArrayList;
 import java.util.HashSet;
 import java.util.List;
@@ -47,6 +52,9 @@ public class NetworkStressTest {
         /** A barrier to wait for concurrent tasks. */
         final CountDownLatch pendingTasks = new CountDownLatch(1 /*seed node*/ + NPEERS);
 
+        // Set a security provider to allow key generation.
+        Security.addProvider(new BouncyCastleProvider());
+
         // Create the temporary directory.
         tempDir = createTempDirectory();
 
@@ -72,8 +80,14 @@ public class NetworkStressTest {
             final File peerDir = new File(tempDir.toFile(), "Bitsquare_peer_" + peerPort);
             final File peerTorDir = new File(peerDir, "tor");
             final File peerStorageDir = new File(peerDir, "db");
+            final File peerKeysDir = new File(peerDir, "keys");
+            //noinspection ResultOfMethodCallIgnored
+            peerKeysDir.mkdirs();  // needed for creating the key ring
+            final KeyStorage peerKeyStorage = new KeyStorage(peerKeysDir);
+            final KeyRing peerKeyRing = new KeyRing(peerKeyStorage);
+            final EncryptionService peerEncryptionService = new EncryptionService(peerKeyRing);
             final P2PService peer = new P2PService(seedNodesRepository, peerPort, peerTorDir, useLocalhost,
-                    REGTEST_NETWORK_ID, peerStorageDir, new Clock(), null /*TODO:enc svc*/, null /*TODO:key ring*/);
+                    REGTEST_NETWORK_ID, peerStorageDir, new Clock(), peerEncryptionService, peerKeyRing);
             peerNodes.add(peer);
             peer.start(getSetupListener(setupFailed, pendingTasks));
         }

From f4aa8480f7e71044e8b51ac4512e849d81ee6a84 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 26 Apr 2016 10:10:43 +0200
Subject: [PATCH 016/101] Fix removal of temporary directory in network stress
 test

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index bf10ded459..c40e106386 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -187,7 +187,7 @@ public class NetworkStressTest {
         Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                Files.delete(path);
+                Files.delete(file);
                 return FileVisitResult.CONTINUE;
             }
 

From a2d130219c214d3b1f2cae1fd9502e8e8b0bf450 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 26 Apr 2016 11:33:03 +0200
Subject: [PATCH 017/101] Rename latch and failure variables in network stress
 test

---
 .../p2p/network/NetworkStressTest.java        | 38 +++++++++----------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index c40e106386..52d62ad9bc 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -47,10 +47,10 @@ public class NetworkStressTest {
 
     @Before
     public void setUp() throws Exception {
-        /** A property where threads can indicate setup failure. */
-        BooleanProperty setupFailed = new SimpleBooleanProperty(false);
-        /** A barrier to wait for concurrent tasks. */
-        final CountDownLatch pendingTasks = new CountDownLatch(1 /*seed node*/ + NPEERS);
+        /** A property where threads can indicate setup failure of local services (Tor node, hidden service). */
+        BooleanProperty localServicesFailed = new SimpleBooleanProperty(false);
+        /** A barrier to wait for concurrent setup of local services (Tor node, hidden service). */
+        final CountDownLatch localServicesLatch = new CountDownLatch(1 /*seed node*/ + NPEERS);
 
         // Set a security provider to allow key generation.
         Security.addProvider(new BouncyCastleProvider());
@@ -66,7 +66,7 @@ public class NetworkStressTest {
         seedNodes.add(seedNodeAddress);  // the only seed node in tests
         seedNode.createAndStartP2PService(seedNodeAddress, useLocalhost,
                 REGTEST_NETWORK_ID, true /*detailed logging*/, seedNodes,
-                getSetupListener(setupFailed, pendingTasks));
+                getSetupListener(localServicesFailed, localServicesLatch));
 
         // Create and start peer nodes.
         SeedNodesRepository seedNodesRepository = new SeedNodesRepository();
@@ -89,14 +89,14 @@ public class NetworkStressTest {
             final P2PService peer = new P2PService(seedNodesRepository, peerPort, peerTorDir, useLocalhost,
                     REGTEST_NETWORK_ID, peerStorageDir, new Clock(), peerEncryptionService, peerKeyRing);
             peerNodes.add(peer);
-            peer.start(getSetupListener(setupFailed, pendingTasks));
+            peer.start(getSetupListener(localServicesFailed, localServicesLatch));
         }
 
         // Wait for concurrent tasks to finish.
-        pendingTasks.await();
+        localServicesLatch.await();
 
         // Check if any node reported setup failure on start.
-        if (setupFailed.get()) {
+        if (localServicesFailed.get()) {
             throw new Exception("nodes failed to start");
         }
     }
@@ -107,9 +107,9 @@ public class NetworkStressTest {
     }
 
     @NotNull
-    private static P2PServiceListener getSetupListener(
-            final BooleanProperty setupFailed,
-            final CountDownLatch pendingTasks) {
+    private P2PServiceListener getSetupListener(
+            final BooleanProperty localServicesFailed,
+            final CountDownLatch localServicesLatch) {
         return new P2PServiceListener() {
             @Override
             public void onRequestingDataCompleted() {
@@ -139,33 +139,33 @@ public class NetworkStressTest {
             @Override
             public void onHiddenServicePublished() {
                 // successful result
-                pendingTasks.countDown();
+                localServicesLatch.countDown();
             }
 
             @Override
             public void onSetupFailed(Throwable throwable) {
                 // failed result
-                setupFailed.set(true);
-                pendingTasks.countDown();
+                localServicesFailed.set(true);
+                localServicesLatch.countDown();
             }
         };
     }
 
     @After
     public void tearDown() throws InterruptedException, IOException {
-        /** A barrier to wait for concurrent tasks. */
-        final CountDownLatch pendingTasks = new CountDownLatch((seedNode != null? 1 : 0) + peerNodes.size());
+        /** A barrier to wait for concurrent shutdown of services. */
+        final CountDownLatch shutdownLatch = new CountDownLatch((seedNode != null? 1 : 0) + peerNodes.size());
 
         // Stop the seed node.
         if (seedNode != null) {
-            seedNode.shutDown(pendingTasks::countDown);
+            seedNode.shutDown(shutdownLatch::countDown);
         }
         // Stop peer nodes.
         for (P2PService peer : peerNodes) {
-            peer.shutDown(pendingTasks::countDown);
+            peer.shutDown(shutdownLatch::countDown);
         }
         // Wait for concurrent tasks to finish.
-        pendingTasks.await();
+        shutdownLatch.await();
 
         if (tempDir != null) {
             deleteRecursively(tempDir);

From 1f2d755c643543153faa0b0498ac65c742c7df85 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 26 Apr 2016 11:37:51 +0200
Subject: [PATCH 018/101] Check reception of preliminary data in peers in
 network stress test

Need to check that the seed node boots first to avoid spurious failures.
---
 .../io/bitsquare/p2p/network/NetworkStressTest.java  | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 52d62ad9bc..630abe6f86 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -31,6 +31,7 @@ import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
 
 public class NetworkStressTest {
     /** Numeric identifier of the regtest Bitcoin network. */
@@ -45,6 +46,9 @@ public class NetworkStressTest {
     /** A list of peer nodes represented as P2P services. */
     private List<P2PService> peerNodes = new ArrayList<>();
 
+    /** A barrier to wait for concurrent reception of preliminary data in peers. */
+    private CountDownLatch prelimDataLatch = new CountDownLatch(NPEERS);
+
     @Before
     public void setUp() throws Exception {
         /** A property where threads can indicate setup failure of local services (Tor node, hidden service). */
@@ -113,7 +117,8 @@ public class NetworkStressTest {
         return new P2PServiceListener() {
             @Override
             public void onRequestingDataCompleted() {
-                // do nothing
+                // successful result
+                NetworkStressTest.this.prelimDataLatch.countDown();
             }
 
             @Override
@@ -174,8 +179,9 @@ public class NetworkStressTest {
 
     @Test
     public void test() throws InterruptedException {
-        // do nothing
-        Thread.sleep(30_000);
+        // Wait for peers to get their preliminary data.
+        org.junit.Assert.assertTrue("timed out while waiting for preliminary data",
+                prelimDataLatch.await(30, TimeUnit.SECONDS));
     }
 
     private Path createTempDirectory() throws IOException {

From 59f66f8ef962f9fee10a6cfa41992ccc4d78e5b5 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 26 Apr 2016 11:41:37 +0200
Subject: [PATCH 019/101] Some comments to separate declarations

---
 .../io/bitsquare/p2p/network/NetworkStressTest.java   | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 630abe6f86..60651c6da7 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -34,11 +34,18 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
 public class NetworkStressTest {
-    /** Numeric identifier of the regtest Bitcoin network. */
-    private static final int REGTEST_NETWORK_ID = 2;
+    // Test parameters
+
     /** Number of peer nodes to create. */
     private static final int NPEERS = 1;
 
+    // Constants
+
+    /** Numeric identifier of the regtest Bitcoin network. */
+    private static final int REGTEST_NETWORK_ID = 2;
+
+    // Instance fields
+
     /** A directory to temporarily hold seed and normal nodes' configuration and state files. */
     private Path tempDir;
     /** A single seed node that other nodes will contact to request initial data. */

From fe9b3ec5f6643452f0a71837222f1a1884b6a051 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 26 Apr 2016 13:44:47 +0200
Subject: [PATCH 020/101] Use localhost port ending in net id digit

Otherwise ``SeedNodesRepository`` ignores the address and peers fail to connect to the seed node.
---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java  | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 60651c6da7..603dfb0380 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -114,7 +114,13 @@ public class NetworkStressTest {
 
     @NotNull
     private static NodeAddress getSeedNodeAddress() {
-        return new NodeAddress("localhost", Utils.findFreeSystemPort());
+        // The address is only considered by ``SeedNodesRepository`` if
+        // it ends in the digit matching the network identifier.
+        int port;
+        do {
+            port = Utils.findFreeSystemPort();
+        } while (port % 10 != REGTEST_NETWORK_ID);
+        return new NodeAddress("localhost", port);
     }
 
     @NotNull

From 1910b66aef09c2ce6104eecb2baf05043f1b64a2 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 26 Apr 2016 14:04:41 +0200
Subject: [PATCH 021/101] Wait for peer nodes bootstrap in network stress test

Once it threw a ``ConcurrentModificationException`` while shutting down but I've not been able to reproduce it, so keep an eye on it happening again.
---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java  | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 603dfb0380..4dc0e00822 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -55,6 +55,8 @@ public class NetworkStressTest {
 
     /** A barrier to wait for concurrent reception of preliminary data in peers. */
     private CountDownLatch prelimDataLatch = new CountDownLatch(NPEERS);
+    /** A barrier to wait for concurrent bootstrap of peers. */
+    private CountDownLatch bootstrapLatch = new CountDownLatch(NPEERS);
 
     @Before
     public void setUp() throws Exception {
@@ -146,7 +148,8 @@ public class NetworkStressTest {
 
             @Override
             public void onBootstrapComplete() {
-                // do nothing
+                // successful result
+                NetworkStressTest.this.bootstrapLatch.countDown();
             }
 
             @Override
@@ -195,6 +198,9 @@ public class NetworkStressTest {
         // Wait for peers to get their preliminary data.
         org.junit.Assert.assertTrue("timed out while waiting for preliminary data",
                 prelimDataLatch.await(30, TimeUnit.SECONDS));
+        // Wait for peers to complete their bootstrapping.
+        org.junit.Assert.assertTrue("timed out while waiting for bootstrap",
+                bootstrapLatch.await(30, TimeUnit.SECONDS));
     }
 
     private Path createTempDirectory() throws IOException {

From b3932d0bf47a7dc9067dee94c22bbbabecf16921 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 26 Apr 2016 14:45:02 +0200
Subject: [PATCH 022/101] Separate seed and peer listeners in network stress
 test

In this way we ensure that latches are not accidentally modified if the semantics on callbacks change, so we may be more sure that latch initial values are ok.
---
 .../p2p/network/NetworkStressTest.java        | 135 +++++++++++-------
 1 file changed, 86 insertions(+), 49 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 4dc0e00822..269ce756da 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -79,7 +79,7 @@ public class NetworkStressTest {
         seedNodes.add(seedNodeAddress);  // the only seed node in tests
         seedNode.createAndStartP2PService(seedNodeAddress, useLocalhost,
                 REGTEST_NETWORK_ID, true /*detailed logging*/, seedNodes,
-                getSetupListener(localServicesFailed, localServicesLatch));
+                new SeedServiceListener(localServicesLatch, localServicesFailed));
 
         // Create and start peer nodes.
         SeedNodesRepository seedNodesRepository = new SeedNodesRepository();
@@ -102,7 +102,7 @@ public class NetworkStressTest {
             final P2PService peer = new P2PService(seedNodesRepository, peerPort, peerTorDir, useLocalhost,
                     REGTEST_NETWORK_ID, peerStorageDir, new Clock(), peerEncryptionService, peerKeyRing);
             peerNodes.add(peer);
-            peer.start(getSetupListener(localServicesFailed, localServicesLatch));
+            peer.start(new PeerServiceListener(localServicesLatch, localServicesFailed));
         }
 
         // Wait for concurrent tasks to finish.
@@ -125,53 +125,6 @@ public class NetworkStressTest {
         return new NodeAddress("localhost", port);
     }
 
-    @NotNull
-    private P2PServiceListener getSetupListener(
-            final BooleanProperty localServicesFailed,
-            final CountDownLatch localServicesLatch) {
-        return new P2PServiceListener() {
-            @Override
-            public void onRequestingDataCompleted() {
-                // successful result
-                NetworkStressTest.this.prelimDataLatch.countDown();
-            }
-
-            @Override
-            public void onNoSeedNodeAvailable() {
-                // expected, do nothing
-            }
-
-            @Override
-            public void onNoPeersAvailable() {
-                // expected, do nothing
-            }
-
-            @Override
-            public void onBootstrapComplete() {
-                // successful result
-                NetworkStressTest.this.bootstrapLatch.countDown();
-            }
-
-            @Override
-            public void onTorNodeReady() {
-                // do nothing
-            }
-
-            @Override
-            public void onHiddenServicePublished() {
-                // successful result
-                localServicesLatch.countDown();
-            }
-
-            @Override
-            public void onSetupFailed(Throwable throwable) {
-                // failed result
-                localServicesFailed.set(true);
-                localServicesLatch.countDown();
-            }
-        };
-    }
-
     @After
     public void tearDown() throws InterruptedException, IOException {
         /** A barrier to wait for concurrent shutdown of services. */
@@ -223,4 +176,88 @@ public class NetworkStressTest {
             }
         });
     }
+
+    // P2P service listener classes
+
+    private class TestSetupListener implements SetupListener {
+        private final CountDownLatch localServicesLatch;
+        private final BooleanProperty localServicesFailed;
+
+        TestSetupListener(CountDownLatch localServicesLatch, BooleanProperty localServicesFailed) {
+            this.localServicesLatch = localServicesLatch;
+            this.localServicesFailed = localServicesFailed;
+        }
+
+        @Override
+        public void onTorNodeReady() {
+            // do nothing
+        }
+
+        @Override
+        public void onHiddenServicePublished() {
+            // successful result
+            localServicesLatch.countDown();
+        }
+
+        @Override
+        public void onSetupFailed(Throwable throwable) {
+            // failed result
+            localServicesFailed.set(true);
+            localServicesLatch.countDown();
+        }
+    }
+
+    private class SeedServiceListener extends TestSetupListener implements P2PServiceListener {
+        SeedServiceListener(CountDownLatch localServicesLatch, BooleanProperty localServicesFailed) {
+            super(localServicesLatch, localServicesFailed);
+        }
+
+        @Override
+        public void onRequestingDataCompleted() {
+            // preliminary data not used in single seed node
+        }
+
+        @Override
+        public void onNoSeedNodeAvailable() {
+            // expected in single seed node
+        }
+
+        @Override
+        public void onNoPeersAvailable() {
+            // expected in single seed node
+        }
+
+        @Override
+        public void onBootstrapComplete() {
+            // not used in single seed node
+        }
+    }
+
+    private class PeerServiceListener extends TestSetupListener implements P2PServiceListener {
+        PeerServiceListener(CountDownLatch localServicesLatch, BooleanProperty localServicesFailed) {
+            super(localServicesLatch, localServicesFailed);
+        }
+
+        @Override
+        public void onRequestingDataCompleted() {
+            // preliminary data received
+            NetworkStressTest.this.prelimDataLatch.countDown();
+        }
+
+        @Override
+        public void onNoSeedNodeAvailable() {
+            // do nothing
+        }
+
+        @Override
+        public void onNoPeersAvailable() {
+            // do nothing
+        }
+
+        @Override
+        public void onBootstrapComplete() {
+            // peer bootstrapped
+            NetworkStressTest.this.bootstrapLatch.countDown();
+        }
+    }
 }

From 3a7359247bb7858c535c382be19ef084e66f0e72 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 27 Apr 2016 12:50:14 +0200
Subject: [PATCH 023/101] Use sequentially named directories for peers in
 network stress test

This may allow to reuse the directory (to avoid regenerating keys, basically).
---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 269ce756da..c568c83246 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -90,7 +90,7 @@ public class NetworkStressTest {
         }
         for (int p = 0; p < NPEERS; p++) {
             final int peerPort = Utils.findFreeSystemPort();
-            final File peerDir = new File(tempDir.toFile(), "Bitsquare_peer_" + peerPort);
+            final File peerDir = new File(tempDir.toFile(), String.format("peer-%06d", p));
             final File peerTorDir = new File(peerDir, "tor");
             final File peerStorageDir = new File(peerDir, "db");
             final File peerKeysDir = new File(peerDir, "keys");

From e12d61d026fdab7973a35b1a3be96f67e5d9ae04 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 27 Apr 2016 13:24:20 +0200
Subject: [PATCH 024/101] Keep network stress test data if environment variable
 ``STRESS_TEST_DIR`` is specified

---
 .../p2p/network/NetworkStressTest.java        | 27 ++++++++++++++-----
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index c568c83246..f787fdacb3 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -20,10 +20,7 @@ import org.junit.Test;
 
 import java.io.File;
 import java.io.IOException;
-import java.nio.file.FileVisitResult;
-import java.nio.file.Files;
-import java.nio.file.Path;
-import java.nio.file.SimpleFileVisitor;
+import java.nio.file.*;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.security.Security;
 import java.util.ArrayList;
@@ -37,12 +34,14 @@ public class NetworkStressTest {
     // Test parameters
 
     /** Number of peer nodes to create. */
-    private static final int NPEERS = 1;
+    private static final int NPEERS = 4;
 
     // Constants
 
     /** Numeric identifier of the regtest Bitcoin network. */
     private static final int REGTEST_NETWORK_ID = 2;
+    /** Environment variable to specify a persistent test data directory. */
+    private static final String TEST_DIR_ENVVAR = "STRESS_TEST_DIR";
 
     // Instance fields
 
@@ -141,7 +140,8 @@ public class NetworkStressTest {
         // Wait for concurrent tasks to finish.
         shutdownLatch.await();
 
-        if (tempDir != null) {
+        // Cleanup directory if not given explicitly.
+        if (tempDir != null && System.getenv(TEST_DIR_ENVVAR) == null) {
             deleteRecursively(tempDir);
         }
     }
@@ -157,7 +157,20 @@ public class NetworkStressTest {
     }
 
     private Path createTempDirectory() throws IOException {
-        return Files.createTempDirectory("bsq" + this.getClass().getSimpleName());
+        Path stressTestDirPath;
+
+        String stressTestDir = System.getenv(TEST_DIR_ENVVAR);
+        if (stressTestDir != null) {
+            // Test directory specified, use and create if missing.
+            stressTestDirPath = Paths.get(stressTestDir);
+            if (!Files.isDirectory(stressTestDirPath)) {
+                //noinspection ResultOfMethodCallIgnored
+                stressTestDirPath.toFile().mkdirs();
+            }
+        } else {
+            stressTestDirPath = Files.createTempDirectory("bsq" + this.getClass().getSimpleName());
+        }
+        return stressTestDirPath;
     }
 
     private static void deleteRecursively(@NotNull final Path path) throws IOException {

From b02c5bf2be6a0db33244bedf443404aeb1d88401 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 27 Apr 2016 13:32:45 +0200
Subject: [PATCH 025/101] Move test data removal logic to instance method

---
 .../io/bitsquare/p2p/network/NetworkStressTest.java  | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index f787fdacb3..1e4d729982 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -140,9 +140,9 @@ public class NetworkStressTest {
         // Wait for concurrent tasks to finish.
         shutdownLatch.await();
 
-        // Cleanup directory if not given explicitly.
-        if (tempDir != null && System.getenv(TEST_DIR_ENVVAR) == null) {
-            deleteRecursively(tempDir);
+        // Cleanup temporary directory.
+        if (tempDir != null) {
+            deleteTempDirectory();
         }
     }
 
@@ -173,9 +173,11 @@ public class NetworkStressTest {
         return stressTestDirPath;
     }
 
-    private static void deleteRecursively(@NotNull final Path path) throws IOException {
+    private void deleteTempDirectory() throws IOException {
         // Based on <https://stackoverflow.com/a/27917071/6239236> by Tomasz Dzięcielewski.
-        Files.walkFileTree(path, new SimpleFileVisitor<Path>() {
+        if (System.getenv(TEST_DIR_ENVVAR) != null)
+            return;  // do not remove if given explicitly
+        Files.walkFileTree(tempDir, new SimpleFileVisitor<Path>() {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                 Files.delete(file);

From 6c5c3371e6fc942ca61b5603e0a5dc911542ddad Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 27 Apr 2016 13:38:09 +0200
Subject: [PATCH 026/101] Rename "temporary" to "test data" directory in
 network stress test

---
 .../p2p/network/NetworkStressTest.java        | 24 +++++++++----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 1e4d729982..12402802ed 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -45,8 +45,8 @@ public class NetworkStressTest {
 
     // Instance fields
 
-    /** A directory to temporarily hold seed and normal nodes' configuration and state files. */
-    private Path tempDir;
+    /** A directory to (temporarily) hold seed and normal nodes' configuration and state files. */
+    private Path testDataDir;
     /** A single seed node that other nodes will contact to request initial data. */
     private SeedNode seedNode;
     /** A list of peer nodes represented as P2P services. */
@@ -67,11 +67,11 @@ public class NetworkStressTest {
         // Set a security provider to allow key generation.
         Security.addProvider(new BouncyCastleProvider());
 
-        // Create the temporary directory.
-        tempDir = createTempDirectory();
+        // Create the test data directory.
+        testDataDir = createTestDataDirectory();
 
         // Create and start the seed node.
-        seedNode = new SeedNode(tempDir.toString());
+        seedNode = new SeedNode(testDataDir.toString());
         final NodeAddress seedNodeAddress = getSeedNodeAddress();
         final boolean useLocalhost = seedNodeAddress.hostName.equals("localhost");
         final Set<NodeAddress> seedNodes = new HashSet<>(1);
@@ -89,7 +89,7 @@ public class NetworkStressTest {
         }
         for (int p = 0; p < NPEERS; p++) {
             final int peerPort = Utils.findFreeSystemPort();
-            final File peerDir = new File(tempDir.toFile(), String.format("peer-%06d", p));
+            final File peerDir = new File(testDataDir.toFile(), String.format("peer-%06d", p));
             final File peerTorDir = new File(peerDir, "tor");
             final File peerStorageDir = new File(peerDir, "db");
             final File peerKeysDir = new File(peerDir, "keys");
@@ -140,9 +140,9 @@ public class NetworkStressTest {
         // Wait for concurrent tasks to finish.
         shutdownLatch.await();
 
-        // Cleanup temporary directory.
-        if (tempDir != null) {
-            deleteTempDirectory();
+        // Cleanup test data directory.
+        if (testDataDir != null) {
+            deleteTestDataDirectory();
         }
     }
 
@@ -156,7 +156,7 @@ public class NetworkStressTest {
                 bootstrapLatch.await(30, TimeUnit.SECONDS));
     }
 
-    private Path createTempDirectory() throws IOException {
+    private Path createTestDataDirectory() throws IOException {
         Path stressTestDirPath;
 
         String stressTestDir = System.getenv(TEST_DIR_ENVVAR);
@@ -173,11 +173,11 @@ public class NetworkStressTest {
         return stressTestDirPath;
     }
 
-    private void deleteTempDirectory() throws IOException {
+    private void deleteTestDataDirectory() throws IOException {
         // Based on <https://stackoverflow.com/a/27917071/6239236> by Tomasz Dzięcielewski.
         if (System.getenv(TEST_DIR_ENVVAR) != null)
             return;  // do not remove if given explicitly
-        Files.walkFileTree(tempDir, new SimpleFileVisitor<Path>() {
+        Files.walkFileTree(testDataDir, new SimpleFileVisitor<Path>() {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                 Files.delete(file);

From f100ef6384ddc2856e3a9a54a052669136102a02 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 27 Apr 2016 14:10:20 +0200
Subject: [PATCH 027/101] Only keep peer keys when reusing network stress test
 directory

---
 .../p2p/network/NetworkStressTest.java         | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 12402802ed..628abad4fb 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -173,20 +173,30 @@ public class NetworkStressTest {
         return stressTestDirPath;
     }
 
+    /**
+     * Delete the test data directory recursively, unless <code>STRESS_TEST_DIR</code> is defined,
+     * in which case peer node keys are kept.
+     *
+     * @throws IOException
+     */
     private void deleteTestDataDirectory() throws IOException {
         // Based on <https://stackoverflow.com/a/27917071/6239236> by Tomasz Dzięcielewski.
-        if (System.getenv(TEST_DIR_ENVVAR) != null)
-            return;  // do not remove if given explicitly
+        boolean keep = System.getenv(TEST_DIR_ENVVAR) != null;  // do not remove if given explicitly
         Files.walkFileTree(testDataDir, new SimpleFileVisitor<Path>() {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                Files.delete(file);
+                final String fileName = file.getFileName().toString();
+                if (!(keep && (fileName.equals("enc.key") || fileName.equals("sig.key"))))
+                    Files.delete(file);
                 return FileVisitResult.CONTINUE;
             }
 
             @Override
             public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
-                Files.delete(dir);
+                // ``dir`` is always a directory, I/O errors may still trigger ``NullPointerException``.
+                //noinspection ConstantConditions
+                if (!(keep && dir.toFile().listFiles().length > 0))
+                    Files.delete(dir);
                 return FileVisitResult.CONTINUE;
             }
         });

From d439058392e5130f351d455132666d2d4f8c650a Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 27 Apr 2016 14:26:47 +0200
Subject: [PATCH 028/101] Check that network stress test data directory
 variable is not empty

---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java   | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 628abad4fb..51c358a576 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -159,8 +159,8 @@ public class NetworkStressTest {
     private Path createTestDataDirectory() throws IOException {
         Path stressTestDirPath;
 
-        String stressTestDir = System.getenv(TEST_DIR_ENVVAR);
-        if (stressTestDir != null) {
+        final String stressTestDir = System.getenv(TEST_DIR_ENVVAR);
+        if ((stressTestDir != null) && !stressTestDir.equals("")) {
             // Test directory specified, use and create if missing.
             stressTestDirPath = Paths.get(stressTestDir);
             if (!Files.isDirectory(stressTestDirPath)) {
@@ -181,7 +181,8 @@ public class NetworkStressTest {
      */
     private void deleteTestDataDirectory() throws IOException {
         // Based on <https://stackoverflow.com/a/27917071/6239236> by Tomasz Dzięcielewski.
-        boolean keep = System.getenv(TEST_DIR_ENVVAR) != null;  // do not remove if given explicitly
+        final String stressTestDir = System.getenv(TEST_DIR_ENVVAR);
+        final boolean keep = (stressTestDir != null) && !stressTestDir.equals("");
         Files.walkFileTree(testDataDir, new SimpleFileVisitor<Path>() {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {

From bfbda2c38e57f37806d451b48e2aa9b544daee39 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 27 Apr 2016 14:38:14 +0200
Subject: [PATCH 029/101] Turn detailed logging of network stress test into a
 constant

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 51c358a576..af869e98bf 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -35,6 +35,8 @@ public class NetworkStressTest {
 
     /** Number of peer nodes to create. */
     private static final int NPEERS = 4;
+    /** Whether to log messages less important than warnings. */
+    private static final boolean USE_DETAILED_LOGGING = false;
 
     // Constants
 
@@ -77,7 +79,7 @@ public class NetworkStressTest {
         final Set<NodeAddress> seedNodes = new HashSet<>(1);
         seedNodes.add(seedNodeAddress);  // the only seed node in tests
         seedNode.createAndStartP2PService(seedNodeAddress, useLocalhost,
-                REGTEST_NETWORK_ID, true /*detailed logging*/, seedNodes,
+                REGTEST_NETWORK_ID, USE_DETAILED_LOGGING, seedNodes,
                 new SeedServiceListener(localServicesLatch, localServicesFailed));
 
         // Create and start peer nodes.

From a9973808b2e8535a5a01b674b59a8cf59cc479f9 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 27 Apr 2016 14:56:43 +0200
Subject: [PATCH 030/101] Make the number of peers in network stress test
 selectable with environment variable

---
 .../p2p/network/NetworkStressTest.java        | 25 +++++++++++++------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index af869e98bf..79cb5850ea 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -33,8 +33,6 @@ import java.util.concurrent.TimeUnit;
 public class NetworkStressTest {
     // Test parameters
 
-    /** Number of peer nodes to create. */
-    private static final int NPEERS = 4;
     /** Whether to log messages less important than warnings. */
     private static final boolean USE_DETAILED_LOGGING = false;
 
@@ -42,6 +40,10 @@ public class NetworkStressTest {
 
     /** Numeric identifier of the regtest Bitcoin network. */
     private static final int REGTEST_NETWORK_ID = 2;
+    /** Default number of peers in the test. */
+    private static final int NPEERS_DEFAULT = 4;
+    /** Environment variable to specify the number of peers in the test. */
+    private static final String NPEERS_ENVVAR = "STRESS_TEST_NPEERS";
     /** Environment variable to specify a persistent test data directory. */
     private static final String TEST_DIR_ENVVAR = "STRESS_TEST_DIR";
 
@@ -55,16 +57,25 @@ public class NetworkStressTest {
     private List<P2PService> peerNodes = new ArrayList<>();
 
     /** A barrier to wait for concurrent reception of preliminary data in peers. */
-    private CountDownLatch prelimDataLatch = new CountDownLatch(NPEERS);
+    private CountDownLatch prelimDataLatch;
     /** A barrier to wait for concurrent bootstrap of peers. */
-    private CountDownLatch bootstrapLatch = new CountDownLatch(NPEERS);
+    private CountDownLatch bootstrapLatch;
 
     @Before
     public void setUp() throws Exception {
+        /** Number of peer nodes to create. */
+        int nPeers = NPEERS_DEFAULT;
+
+        final String nPeersEnv = System.getenv(NPEERS_ENVVAR);
+        if (nPeersEnv != null && !nPeersEnv.equals(""))
+            nPeers = Integer.parseInt(nPeersEnv);
+        prelimDataLatch = new CountDownLatch(nPeers);
+        bootstrapLatch = new CountDownLatch(nPeers);
+
         /** A property where threads can indicate setup failure of local services (Tor node, hidden service). */
-        BooleanProperty localServicesFailed = new SimpleBooleanProperty(false);
+        final BooleanProperty localServicesFailed = new SimpleBooleanProperty(false);
         /** A barrier to wait for concurrent setup of local services (Tor node, hidden service). */
-        final CountDownLatch localServicesLatch = new CountDownLatch(1 /*seed node*/ + NPEERS);
+        final CountDownLatch localServicesLatch = new CountDownLatch(1 /*seed node*/ + nPeers);
 
         // Set a security provider to allow key generation.
         Security.addProvider(new BouncyCastleProvider());
@@ -89,7 +100,7 @@ public class NetworkStressTest {
         } else {
             seedNodesRepository.setTorSeedNodeAddresses(seedNodes);
         }
-        for (int p = 0; p < NPEERS; p++) {
+        for (int p = 0; p < nPeers; p++) {
             final int peerPort = Utils.findFreeSystemPort();
             final File peerDir = new File(testDataDir.toFile(), String.format("peer-%06d", p));
             final File peerTorDir = new File(peerDir, "tor");

From a99c0ffc0769c046c4374084baaf4b7395b3e731 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 28 Apr 2016 13:27:46 +0200
Subject: [PATCH 031/101] Keep Tor hidden service key when preserving network
 stress test data

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 79cb5850ea..964d240ba0 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -200,7 +200,7 @@ public class NetworkStressTest {
             @Override
             public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
                 final String fileName = file.getFileName().toString();
-                if (!(keep && (fileName.equals("enc.key") || fileName.equals("sig.key"))))
+                if (!(keep && (fileName.matches("enc\\.key|sig\\.key|private_key"))))  // peer and tor keys
                     Files.delete(file);
                 return FileVisitResult.CONTINUE;
             }

From 9f728c19c3495ecbca2b20f78dff0c3726309046 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 28 Apr 2016 13:47:04 +0200
Subject: [PATCH 032/101] Check minimum peers in network stress test

In preparation for direct send message test.
---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java     | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 964d240ba0..53e1934748 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -42,6 +42,8 @@ public class NetworkStressTest {
     private static final int REGTEST_NETWORK_ID = 2;
     /** Default number of peers in the test. */
     private static final int NPEERS_DEFAULT = 4;
+    /** Minimum number of peers for the test to work. */
+    private static final int NPEERS_MIN = 2;
     /** Environment variable to specify the number of peers in the test. */
     private static final String NPEERS_ENVVAR = "STRESS_TEST_NPEERS";
     /** Environment variable to specify a persistent test data directory. */
@@ -69,6 +71,9 @@ public class NetworkStressTest {
         final String nPeersEnv = System.getenv(NPEERS_ENVVAR);
         if (nPeersEnv != null && !nPeersEnv.equals(""))
             nPeers = Integer.parseInt(nPeersEnv);
+        if (nPeers < NPEERS_MIN)
+            throw new IllegalArgumentException(
+                    String.format("Test needs at least %d peer nodes to work: %d", NPEERS_MIN, nPeers));
         prelimDataLatch = new CountDownLatch(nPeers);
         bootstrapLatch = new CountDownLatch(nPeers);
 

From 2d9ed390f05748d569922fa4b077349e84a0bb74 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 28 Apr 2016 14:57:52 +0200
Subject: [PATCH 033/101] Partial direct send/receive test between two peers in
 network stress test

Still need to implement checks on the receiving side.
---
 .../p2p/network/NetworkStressTest.java        | 57 +++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 53e1934748..1f77534b1c 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -1,13 +1,17 @@
 package io.bitsquare.p2p.network;
 
+import io.bitsquare.app.Version;
 import io.bitsquare.common.Clock;
 import io.bitsquare.common.crypto.KeyRing;
 import io.bitsquare.common.crypto.KeyStorage;
+import io.bitsquare.common.crypto.PubKeyRing;
 import io.bitsquare.crypto.EncryptionService;
 import io.bitsquare.p2p.NodeAddress;
 import io.bitsquare.p2p.P2PService;
 import io.bitsquare.p2p.P2PServiceListener;
 import io.bitsquare.p2p.Utils;
+import io.bitsquare.p2p.messaging.DirectMessage;
+import io.bitsquare.p2p.messaging.SendDirectMessageListener;
 import io.bitsquare.p2p.seed.SeedNode;
 import io.bitsquare.p2p.seed.SeedNodesRepository;
 import javafx.beans.property.BooleanProperty;
@@ -57,6 +61,8 @@ public class NetworkStressTest {
     private SeedNode seedNode;
     /** A list of peer nodes represented as P2P services. */
     private List<P2PService> peerNodes = new ArrayList<>();
+    /** A list of peer node's public key rings. */
+    private List<PubKeyRing> peerPKRings = new ArrayList<>();
 
     /** A barrier to wait for concurrent reception of preliminary data in peers. */
     private CountDownLatch prelimDataLatch;
@@ -115,6 +121,7 @@ public class NetworkStressTest {
             peerKeysDir.mkdirs();  // needed for creating the key ring
             final KeyStorage peerKeyStorage = new KeyStorage(peerKeysDir);
             final KeyRing peerKeyRing = new KeyRing(peerKeyStorage);
+            peerPKRings.add(peerKeyRing.getPubKeyRing());
             final EncryptionService peerEncryptionService = new EncryptionService(peerKeyRing);
             final P2PService peer = new P2PService(seedNodesRepository, peerPort, peerTorDir, useLocalhost,
                     REGTEST_NETWORK_ID, peerStorageDir, new Clock(), peerEncryptionService, peerKeyRing);
@@ -172,6 +179,34 @@ public class NetworkStressTest {
         // Wait for peers to complete their bootstrapping.
         org.junit.Assert.assertTrue("timed out while waiting for bootstrap",
                 bootstrapLatch.await(30, TimeUnit.SECONDS));
+
+        // Test sending a direct message from peer #0 to peer #1.
+        BooleanProperty sentDirectFailed = new SimpleBooleanProperty(false);
+        final CountDownLatch sentDirectLatch = new CountDownLatch(1);
+        //final CountDownLatch receivedDirectLatch = new CountDownLatch(1);
+        final int srcPeerIdx = 0;
+        final int dstPeerIdx = 1;
+        final P2PService srcPeer = peerNodes.get(srcPeerIdx);
+        final P2PService dstPeer = peerNodes.get(dstPeerIdx);
+        srcPeer.sendEncryptedDirectMessage(dstPeer.getAddress(), peerPKRings.get(dstPeerIdx),
+                new StressTestDirectMessage("test-" + dstPeerIdx), new SendDirectMessageListener() {
+                    @Override
+                    public void onArrived() {
+                        sentDirectLatch.countDown();
+                    }
+
+                    @Override
+                    public void onFault() {
+                        sentDirectFailed.set(true);
+                        sentDirectLatch.countDown();
+                    }
+                });
+        // TODO: receiving data
+        // Wait for peer #0 to complete sending.
+        org.junit.Assert.assertTrue("timed out while sending direct message",
+                sentDirectLatch.await(30, TimeUnit.SECONDS));
+        org.junit.Assert.assertFalse("peer failed to send message", sentDirectFailed.get());
+        //TODO: wait for receiving data
     }
 
     private Path createTestDataDirectory() throws IOException {
@@ -305,3 +340,25 @@ public class NetworkStressTest {
         }
     }
 }
+
+// Message classes
+
+final class StressTestDirectMessage implements DirectMessage {
+    private static final long serialVersionUID = Version.P2P_NETWORK_VERSION;
+    private final int messageVersion = Version.getP2PMessageVersion();
+
+    private String data;
+
+    StressTestDirectMessage(String data) {
+        this.data = data;
+    }
+
+    @Override
+    public int getMessageVersion() {
+        return messageVersion;
+    }
+
+    public String getData() {
+        return data;
+    }
+}

From 30dd7f0f88f6afac0dab065cfe26fdb99745df8d Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Fri, 29 Apr 2016 12:27:11 +0200
Subject: [PATCH 034/101] Change content of network stress test messages

Otherwise the destination node can have a hard time checking its content.
---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 1f77534b1c..d5fffec245 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -189,7 +189,7 @@ public class NetworkStressTest {
         final P2PService srcPeer = peerNodes.get(srcPeerIdx);
         final P2PService dstPeer = peerNodes.get(dstPeerIdx);
         srcPeer.sendEncryptedDirectMessage(dstPeer.getAddress(), peerPKRings.get(dstPeerIdx),
-                new StressTestDirectMessage("test-" + dstPeerIdx), new SendDirectMessageListener() {
+                new StressTestDirectMessage("test/" + dstPeer.getAddress()), new SendDirectMessageListener() {
                     @Override
                     public void onArrived() {
                         sentDirectLatch.countDown();

From 97722a6ecdcc52d203eee6ad62cac4f12d6676d5 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Fri, 29 Apr 2016 12:52:19 +0200
Subject: [PATCH 035/101] Check direct message receiving side in network stress
 test

---
 .../p2p/network/NetworkStressTest.java        | 20 +++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index d5fffec245..a60cf8f323 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -5,11 +5,13 @@ import io.bitsquare.common.Clock;
 import io.bitsquare.common.crypto.KeyRing;
 import io.bitsquare.common.crypto.KeyStorage;
 import io.bitsquare.common.crypto.PubKeyRing;
+import io.bitsquare.crypto.DecryptedMsgWithPubKey;
 import io.bitsquare.crypto.EncryptionService;
 import io.bitsquare.p2p.NodeAddress;
 import io.bitsquare.p2p.P2PService;
 import io.bitsquare.p2p.P2PServiceListener;
 import io.bitsquare.p2p.Utils;
+import io.bitsquare.p2p.messaging.DecryptedDirectMessageListener;
 import io.bitsquare.p2p.messaging.DirectMessage;
 import io.bitsquare.p2p.messaging.SendDirectMessageListener;
 import io.bitsquare.p2p.seed.SeedNode;
@@ -183,7 +185,7 @@ public class NetworkStressTest {
         // Test sending a direct message from peer #0 to peer #1.
         BooleanProperty sentDirectFailed = new SimpleBooleanProperty(false);
         final CountDownLatch sentDirectLatch = new CountDownLatch(1);
-        //final CountDownLatch receivedDirectLatch = new CountDownLatch(1);
+        final CountDownLatch receivedDirectLatch = new CountDownLatch(1);
         final int srcPeerIdx = 0;
         final int dstPeerIdx = 1;
         final P2PService srcPeer = peerNodes.get(srcPeerIdx);
@@ -201,12 +203,22 @@ public class NetworkStressTest {
                         sentDirectLatch.countDown();
                     }
                 });
-        // TODO: receiving data
+        dstPeer.addDecryptedDirectMessageListener((decryptedMsgWithPubKey, peerNodeAddress) -> {
+            if (!(decryptedMsgWithPubKey.message instanceof StressTestDirectMessage))
+                return;
+            StressTestDirectMessage directMessage = (StressTestDirectMessage)(decryptedMsgWithPubKey.message);
+            if ((directMessage.getData().equals("test/" + dstPeer.getAddress())))
+                receivedDirectLatch.countDown();
+        });
+        // Since receiving is completed before sending is reported to be complete,
+        // all receiving checks should end before all sending checks to avoid deadlocking.
+        // Wait for peer #1 to complete receiving.
+        org.junit.Assert.assertTrue("timed out while receiving direct message",
+                receivedDirectLatch.await(30, TimeUnit.SECONDS));
         // Wait for peer #0 to complete sending.
         org.junit.Assert.assertTrue("timed out while sending direct message",
                 sentDirectLatch.await(30, TimeUnit.SECONDS));
         org.junit.Assert.assertFalse("peer failed to send message", sentDirectFailed.get());
-        //TODO: wait for receiving data
     }
 
     private Path createTestDataDirectory() throws IOException {
@@ -358,7 +370,7 @@ final class StressTestDirectMessage implements DirectMessage {
         return messageVersion;
     }
 
-    public String getData() {
+    String getData() {
         return data;
     }
 }

From b8d49e44b6512fb169544a12718ed7696e7b0320 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Fri, 29 Apr 2016 13:12:18 +0200
Subject: [PATCH 036/101] Send direct messages from all peers to random ones in
 network stress test

---
 .../p2p/network/NetworkStressTest.java        | 70 +++++++++----------
 1 file changed, 35 insertions(+), 35 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index a60cf8f323..0f22f7bc8e 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -29,10 +29,7 @@ import java.io.IOException;
 import java.nio.file.*;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.security.Security;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
 
@@ -182,43 +179,46 @@ public class NetworkStressTest {
         org.junit.Assert.assertTrue("timed out while waiting for bootstrap",
                 bootstrapLatch.await(30, TimeUnit.SECONDS));
 
-        // Test sending a direct message from peer #0 to peer #1.
+        // Test each peer sending a direct message to another random peer.
+        final int nPeers = peerNodes.size();
         BooleanProperty sentDirectFailed = new SimpleBooleanProperty(false);
-        final CountDownLatch sentDirectLatch = new CountDownLatch(1);
-        final CountDownLatch receivedDirectLatch = new CountDownLatch(1);
-        final int srcPeerIdx = 0;
-        final int dstPeerIdx = 1;
-        final P2PService srcPeer = peerNodes.get(srcPeerIdx);
-        final P2PService dstPeer = peerNodes.get(dstPeerIdx);
-        srcPeer.sendEncryptedDirectMessage(dstPeer.getAddress(), peerPKRings.get(dstPeerIdx),
-                new StressTestDirectMessage("test/" + dstPeer.getAddress()), new SendDirectMessageListener() {
-                    @Override
-                    public void onArrived() {
-                        sentDirectLatch.countDown();
-                    }
+        final CountDownLatch sentDirectLatch = new CountDownLatch(nPeers);
+        final CountDownLatch receivedDirectLatch = new CountDownLatch(nPeers);
+        for (final P2PService srcPeer : peerNodes) {
+            final int dstPeerIdx = (int) (Math.random() * nPeers);
+            final P2PService dstPeer = peerNodes.get(dstPeerIdx);
+            /*System.out.println(
+                    "Sending direct message from peer " + srcPeer.getAddress() + " to " + dstPeer.getAddress());*/
+            srcPeer.sendEncryptedDirectMessage(dstPeer.getAddress(), peerPKRings.get(dstPeerIdx),
+                    new StressTestDirectMessage("test/" + dstPeer.getAddress()), new SendDirectMessageListener() {
+                        @Override
+                        public void onArrived() {
+                            sentDirectLatch.countDown();
+                        }
 
-                    @Override
-                    public void onFault() {
-                        sentDirectFailed.set(true);
-                        sentDirectLatch.countDown();
-                    }
-                });
-        dstPeer.addDecryptedDirectMessageListener((decryptedMsgWithPubKey, peerNodeAddress) -> {
-            if (!(decryptedMsgWithPubKey.message instanceof StressTestDirectMessage))
-                return;
-            StressTestDirectMessage directMessage = (StressTestDirectMessage)(decryptedMsgWithPubKey.message);
-            if ((directMessage.getData().equals("test/" + dstPeer.getAddress())))
-                receivedDirectLatch.countDown();
-        });
+                        @Override
+                        public void onFault() {
+                            sentDirectFailed.set(true);
+                            sentDirectLatch.countDown();
+                        }
+                    });
+            dstPeer.addDecryptedDirectMessageListener((decryptedMsgWithPubKey, peerNodeAddress) -> {
+                if (!(decryptedMsgWithPubKey.message instanceof StressTestDirectMessage))
+                    return;
+                StressTestDirectMessage directMessage = (StressTestDirectMessage) (decryptedMsgWithPubKey.message);
+                if ((directMessage.getData().equals("test/" + dstPeer.getAddress())))
+                    receivedDirectLatch.countDown();
+            });
+        }
         // Since receiving is completed before sending is reported to be complete,
         // all receiving checks should end before all sending checks to avoid deadlocking.
-        // Wait for peer #1 to complete receiving.
-        org.junit.Assert.assertTrue("timed out while receiving direct message",
+        // Wait for peers to complete receiving.
+        org.junit.Assert.assertTrue("timed out while receiving direct messages",
                 receivedDirectLatch.await(30, TimeUnit.SECONDS));
-        // Wait for peer #0 to complete sending.
-        org.junit.Assert.assertTrue("timed out while sending direct message",
+        // Wait for peers to complete sending.
+        org.junit.Assert.assertTrue("timed out while sending direct messages",
                 sentDirectLatch.await(30, TimeUnit.SECONDS));
-        org.junit.Assert.assertFalse("peer failed to send message", sentDirectFailed.get());
+        org.junit.Assert.assertFalse("some peer(s) failed to send a direct message", sentDirectFailed.get());
     }
 
     private Path createTestDataDirectory() throws IOException {

From 9505a28ccc1a99793923bdb00bdb4f1757e3341b Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 3 May 2016 10:18:24 +0200
Subject: [PATCH 037/101] Print message receiving and sending times in network
 stress test

---
 .../bitsquare/p2p/network/NetworkStressTest.java   | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 0f22f7bc8e..2f3a68d876 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -5,13 +5,11 @@ import io.bitsquare.common.Clock;
 import io.bitsquare.common.crypto.KeyRing;
 import io.bitsquare.common.crypto.KeyStorage;
 import io.bitsquare.common.crypto.PubKeyRing;
-import io.bitsquare.crypto.DecryptedMsgWithPubKey;
 import io.bitsquare.crypto.EncryptionService;
 import io.bitsquare.p2p.NodeAddress;
 import io.bitsquare.p2p.P2PService;
 import io.bitsquare.p2p.P2PServiceListener;
 import io.bitsquare.p2p.Utils;
-import io.bitsquare.p2p.messaging.DecryptedDirectMessageListener;
 import io.bitsquare.p2p.messaging.DirectMessage;
 import io.bitsquare.p2p.messaging.SendDirectMessageListener;
 import io.bitsquare.p2p.seed.SeedNode;
@@ -29,6 +27,8 @@ import java.io.IOException;
 import java.nio.file.*;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.security.Security;
+import java.time.Duration;
+import java.time.Instant;
 import java.util.*;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -184,6 +184,7 @@ public class NetworkStressTest {
         BooleanProperty sentDirectFailed = new SimpleBooleanProperty(false);
         final CountDownLatch sentDirectLatch = new CountDownLatch(nPeers);
         final CountDownLatch receivedDirectLatch = new CountDownLatch(nPeers);
+        final Instant sendStart = Instant.now();
         for (final P2PService srcPeer : peerNodes) {
             final int dstPeerIdx = (int) (Math.random() * nPeers);
             final P2PService dstPeer = peerNodes.get(dstPeerIdx);
@@ -215,12 +216,21 @@ public class NetworkStressTest {
         // Wait for peers to complete receiving.
         org.junit.Assert.assertTrue("timed out while receiving direct messages",
                 receivedDirectLatch.await(30, TimeUnit.SECONDS));
+        print("receiving 1 direct message per node took %ss",
+                Duration.between(sendStart, Instant.now()).toMillis()/1000.0);
         // Wait for peers to complete sending.
         org.junit.Assert.assertTrue("timed out while sending direct messages",
                 sentDirectLatch.await(30, TimeUnit.SECONDS));
+        print("sending 1 direct message per node took %ss",
+                Duration.between(sendStart, Instant.now()).toMillis()/1000.0);
         org.junit.Assert.assertFalse("some peer(s) failed to send a direct message", sentDirectFailed.get());
     }
 
+    private void print(String message, Object... args) {
+        System.out.println(this.getClass().getSimpleName() + ": "
+            + String.format(message, args));
+    }
+
     private Path createTestDataDirectory() throws IOException {
         Path stressTestDirPath;
 

From 3e34d23b9ec35c0d75664afb3ddb682e9e18248f Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 3 May 2016 10:58:07 +0200
Subject: [PATCH 038/101] Convert print instruction to use helper method

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java  | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 2f3a68d876..903541b142 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -188,8 +188,7 @@ public class NetworkStressTest {
         for (final P2PService srcPeer : peerNodes) {
             final int dstPeerIdx = (int) (Math.random() * nPeers);
             final P2PService dstPeer = peerNodes.get(dstPeerIdx);
-            /*System.out.println(
-                    "Sending direct message from peer " + srcPeer.getAddress() + " to " + dstPeer.getAddress());*/
+            //print("sending direct message from peer %s to %s", srcPeer.getAddress(), dstPeer.getAddress());
             srcPeer.sendEncryptedDirectMessage(dstPeer.getAddress(), peerPKRings.get(dstPeerIdx),
                     new StressTestDirectMessage("test/" + dstPeer.getAddress()), new SendDirectMessageListener() {
                         @Override

From 53901ba79875a8ab6c06b60faa2f650007d11690 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 3 May 2016 11:06:38 +0200
Subject: [PATCH 039/101] Configure all peers once to receive direct messages
 in network stress test

Instead on adding a listener once per selected random destination peer.
---
 .../p2p/network/NetworkStressTest.java          | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 903541b142..f9612bdaa5 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -186,6 +186,16 @@ public class NetworkStressTest {
         final CountDownLatch receivedDirectLatch = new CountDownLatch(nPeers);
         final Instant sendStart = Instant.now();
         for (final P2PService srcPeer : peerNodes) {
+            // Make peer ready for receiving direct messages.
+            srcPeer.addDecryptedDirectMessageListener((decryptedMsgWithPubKey, peerNodeAddress) -> {
+                if (!(decryptedMsgWithPubKey.message instanceof StressTestDirectMessage))
+                    return;
+                StressTestDirectMessage directMessage = (StressTestDirectMessage) (decryptedMsgWithPubKey.message);
+                if ((directMessage.getData().equals("test/" + srcPeer.getAddress())))
+                    receivedDirectLatch.countDown();
+            });
+
+            // Select a random peer and send a direct message to it.
             final int dstPeerIdx = (int) (Math.random() * nPeers);
             final P2PService dstPeer = peerNodes.get(dstPeerIdx);
             //print("sending direct message from peer %s to %s", srcPeer.getAddress(), dstPeer.getAddress());
@@ -202,13 +212,6 @@ public class NetworkStressTest {
                             sentDirectLatch.countDown();
                         }
                     });
-            dstPeer.addDecryptedDirectMessageListener((decryptedMsgWithPubKey, peerNodeAddress) -> {
-                if (!(decryptedMsgWithPubKey.message instanceof StressTestDirectMessage))
-                    return;
-                StressTestDirectMessage directMessage = (StressTestDirectMessage) (decryptedMsgWithPubKey.message);
-                if ((directMessage.getData().equals("test/" + dstPeer.getAddress())))
-                    receivedDirectLatch.countDown();
-            });
         }
         // Since receiving is completed before sending is reported to be complete,
         // all receiving checks should end before all sending checks to avoid deadlocking.

From 109a71cd7f17fe0d1fd3d795f5954e4f69bccbe1 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 3 May 2016 11:08:10 +0200
Subject: [PATCH 040/101] Minor changes to comments and printed messages

---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index f9612bdaa5..8420110bf6 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -186,7 +186,7 @@ public class NetworkStressTest {
         final CountDownLatch receivedDirectLatch = new CountDownLatch(nPeers);
         final Instant sendStart = Instant.now();
         for (final P2PService srcPeer : peerNodes) {
-            // Make peer ready for receiving direct messages.
+            // Make the peer ready for receiving direct messages.
             srcPeer.addDecryptedDirectMessageListener((decryptedMsgWithPubKey, peerNodeAddress) -> {
                 if (!(decryptedMsgWithPubKey.message instanceof StressTestDirectMessage))
                     return;
@@ -218,12 +218,12 @@ public class NetworkStressTest {
         // Wait for peers to complete receiving.
         org.junit.Assert.assertTrue("timed out while receiving direct messages",
                 receivedDirectLatch.await(30, TimeUnit.SECONDS));
-        print("receiving 1 direct message per node took %ss",
+        print("receiving 1 direct message per peer took %ss",
                 Duration.between(sendStart, Instant.now()).toMillis()/1000.0);
         // Wait for peers to complete sending.
         org.junit.Assert.assertTrue("timed out while sending direct messages",
                 sentDirectLatch.await(30, TimeUnit.SECONDS));
-        print("sending 1 direct message per node took %ss",
+        print("sending 1 direct message per peer took %ss",
                 Duration.between(sendStart, Instant.now()).toMillis()/1000.0);
         org.junit.Assert.assertFalse("some peer(s) failed to send a direct message", sentDirectFailed.get());
     }

From da145435a136fb54c04ac8e7bb4ef79fb68ae2e0 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 3 May 2016 11:11:49 +0200
Subject: [PATCH 041/101] Keep destination peer address in a variable

---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java     | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 8420110bf6..3baf279454 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -198,9 +198,10 @@ public class NetworkStressTest {
             // Select a random peer and send a direct message to it.
             final int dstPeerIdx = (int) (Math.random() * nPeers);
             final P2PService dstPeer = peerNodes.get(dstPeerIdx);
+            final NodeAddress dstPeerAddress = dstPeer.getAddress();
             //print("sending direct message from peer %s to %s", srcPeer.getAddress(), dstPeer.getAddress());
-            srcPeer.sendEncryptedDirectMessage(dstPeer.getAddress(), peerPKRings.get(dstPeerIdx),
-                    new StressTestDirectMessage("test/" + dstPeer.getAddress()), new SendDirectMessageListener() {
+            srcPeer.sendEncryptedDirectMessage(dstPeerAddress, peerPKRings.get(dstPeerIdx),
+                    new StressTestDirectMessage("test/" + dstPeerAddress), new SendDirectMessageListener() {
                         @Override
                         public void onArrived() {
                             sentDirectLatch.countDown();

From c8e3c6d11ab386142c9f6e0d6126bed4c7c3853a Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 3 May 2016 11:54:34 +0200
Subject: [PATCH 042/101] Random delay when sending direct message in network
 stress test

The delay is always longer than the one that triggers throttling.  It makes not much sense for a single sending, but it will within a loop.

Also made some ``Connection`` throttle constants package-private so they can be accessed by tests.
---
 .../io/bitsquare/p2p/network/Connection.java     |  7 ++++---
 .../bitsquare/p2p/network/NetworkStressTest.java | 16 +++++++++++++---
 2 files changed, 17 insertions(+), 6 deletions(-)

diff --git a/network/src/main/java/io/bitsquare/p2p/network/Connection.java b/network/src/main/java/io/bitsquare/p2p/network/Connection.java
index f09e712895..2059b86cd8 100644
--- a/network/src/main/java/io/bitsquare/p2p/network/Connection.java
+++ b/network/src/main/java/io/bitsquare/p2p/network/Connection.java
@@ -62,10 +62,11 @@ public class Connection implements MessageListener {
     // Static
     ///////////////////////////////////////////////////////////////////////////////////////////
 
-    private static final int MAX_MSG_SIZE = 500 * 1024;              // 500 kb
+    // Leaving some constants package-private for tests to know limits.
+    static final int MAX_MSG_SIZE = 500 * 1024;              // 500 kb
     //TODO decrease limits again after testing
-    private static final int MSG_THROTTLE_PER_SEC = 70;              // With MAX_MSG_SIZE of 500kb results in bandwidth of 35 mbit/sec 
-    private static final int MSG_THROTTLE_PER_10_SEC = 500;           // With MAX_MSG_SIZE of 100kb results in bandwidth of 50 mbit/sec for 10 sec 
+    static final int MSG_THROTTLE_PER_SEC = 70;              // With MAX_MSG_SIZE of 500kb results in bandwidth of 35 mbit/sec
+    static final int MSG_THROTTLE_PER_10_SEC = 500;           // With MAX_MSG_SIZE of 100kb results in bandwidth of 50 mbit/sec for 10 sec
     private static final int SOCKET_TIMEOUT = (int) TimeUnit.SECONDS.toMillis(60);
 
     public static int getMaxMsgSize() {
diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 3baf279454..380834a0e0 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -2,6 +2,7 @@ package io.bitsquare.p2p.network;
 
 import io.bitsquare.app.Version;
 import io.bitsquare.common.Clock;
+import io.bitsquare.common.UserThread;
 import io.bitsquare.common.crypto.KeyRing;
 import io.bitsquare.common.crypto.KeyStorage;
 import io.bitsquare.common.crypto.PubKeyRing;
@@ -43,6 +44,7 @@ public class NetworkStressTest {
 
     /** Numeric identifier of the regtest Bitcoin network. */
     private static final int REGTEST_NETWORK_ID = 2;
+
     /** Default number of peers in the test. */
     private static final int NPEERS_DEFAULT = 4;
     /** Minimum number of peers for the test to work. */
@@ -52,6 +54,11 @@ public class NetworkStressTest {
     /** Environment variable to specify a persistent test data directory. */
     private static final String TEST_DIR_ENVVAR = "STRESS_TEST_DIR";
 
+    /** Minimum delay between direct messages in milliseconds, 25% larger than throttle limit. */
+    private static long MIN_DIRECT_DELAY_MILLIS = Math.round(1.25 * (1.0 / Connection.MSG_THROTTLE_PER_SEC) * 1000);
+    /** Maximum delay between direct messages in milliseconds, 10 times larger than minimum. */
+    private static long MAX_DIRECT_DELAY_MILLIS = 10 * MIN_DIRECT_DELAY_MILLIS;
+
     // Instance fields
 
     /** A directory to (temporarily) hold seed and normal nodes' configuration and state files. */
@@ -195,12 +202,14 @@ public class NetworkStressTest {
                     receivedDirectLatch.countDown();
             });
 
-            // Select a random peer and send a direct message to it.
+            // Select a random peer and send a direct message to it after a random delay
+            // not shorter than throttle limits.
             final int dstPeerIdx = (int) (Math.random() * nPeers);
             final P2PService dstPeer = peerNodes.get(dstPeerIdx);
             final NodeAddress dstPeerAddress = dstPeer.getAddress();
             //print("sending direct message from peer %s to %s", srcPeer.getAddress(), dstPeer.getAddress());
-            srcPeer.sendEncryptedDirectMessage(dstPeerAddress, peerPKRings.get(dstPeerIdx),
+            UserThread.runAfterRandomDelay(() -> srcPeer.sendEncryptedDirectMessage(
+                    dstPeerAddress, peerPKRings.get(dstPeerIdx),
                     new StressTestDirectMessage("test/" + dstPeerAddress), new SendDirectMessageListener() {
                         @Override
                         public void onArrived() {
@@ -212,7 +221,8 @@ public class NetworkStressTest {
                             sentDirectFailed.set(true);
                             sentDirectLatch.countDown();
                         }
-                    });
+                    }), MIN_DIRECT_DELAY_MILLIS, MAX_DIRECT_DELAY_MILLIS, TimeUnit.MILLISECONDS
+            );
         }
         // Since receiving is completed before sending is reported to be complete,
         // all receiving checks should end before all sending checks to avoid deadlocking.

From 6c188e35fa37a2bde82434df57411575fd0a10c0 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 3 May 2016 12:07:39 +0200
Subject: [PATCH 043/101] Send direct messages in network stress test in a loop

Actually each peer runs a loop to setup messages to be sent in the future with a random delay.

This is an actual stress test for the first time. Maybe the timeouts should be adjusted accordingly.
---
 .../p2p/network/NetworkStressTest.java        | 52 ++++++++++---------
 1 file changed, 28 insertions(+), 24 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 380834a0e0..73e30d3be2 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -54,6 +54,8 @@ public class NetworkStressTest {
     /** Environment variable to specify a persistent test data directory. */
     private static final String TEST_DIR_ENVVAR = "STRESS_TEST_DIR";
 
+    /** Number of direct messages to be sent by each peer. */
+    private static int DIRECT_COUNT = 100;
     /** Minimum delay between direct messages in milliseconds, 25% larger than throttle limit. */
     private static long MIN_DIRECT_DELAY_MILLIS = Math.round(1.25 * (1.0 / Connection.MSG_THROTTLE_PER_SEC) * 1000);
     /** Maximum delay between direct messages in milliseconds, 10 times larger than minimum. */
@@ -189,8 +191,8 @@ public class NetworkStressTest {
         // Test each peer sending a direct message to another random peer.
         final int nPeers = peerNodes.size();
         BooleanProperty sentDirectFailed = new SimpleBooleanProperty(false);
-        final CountDownLatch sentDirectLatch = new CountDownLatch(nPeers);
-        final CountDownLatch receivedDirectLatch = new CountDownLatch(nPeers);
+        final CountDownLatch sentDirectLatch = new CountDownLatch(DIRECT_COUNT * nPeers);
+        final CountDownLatch receivedDirectLatch = new CountDownLatch(DIRECT_COUNT * nPeers);
         final Instant sendStart = Instant.now();
         for (final P2PService srcPeer : peerNodes) {
             // Make the peer ready for receiving direct messages.
@@ -202,39 +204,41 @@ public class NetworkStressTest {
                     receivedDirectLatch.countDown();
             });
 
-            // Select a random peer and send a direct message to it after a random delay
-            // not shorter than throttle limits.
-            final int dstPeerIdx = (int) (Math.random() * nPeers);
-            final P2PService dstPeer = peerNodes.get(dstPeerIdx);
-            final NodeAddress dstPeerAddress = dstPeer.getAddress();
-            //print("sending direct message from peer %s to %s", srcPeer.getAddress(), dstPeer.getAddress());
-            UserThread.runAfterRandomDelay(() -> srcPeer.sendEncryptedDirectMessage(
-                    dstPeerAddress, peerPKRings.get(dstPeerIdx),
-                    new StressTestDirectMessage("test/" + dstPeerAddress), new SendDirectMessageListener() {
-                        @Override
-                        public void onArrived() {
-                            sentDirectLatch.countDown();
-                        }
+            for (int i = 0; i < DIRECT_COUNT; i++) {
+                // Select a random peer and send a direct message to it after a random delay
+                // not shorter than throttle limits.
+                final int dstPeerIdx = (int) (Math.random() * nPeers);
+                final P2PService dstPeer = peerNodes.get(dstPeerIdx);
+                final NodeAddress dstPeerAddress = dstPeer.getAddress();
+                //print("sending direct message from peer %s to %s", srcPeer.getAddress(), dstPeer.getAddress());
+                UserThread.runAfterRandomDelay(() -> srcPeer.sendEncryptedDirectMessage(
+                        dstPeerAddress, peerPKRings.get(dstPeerIdx),
+                        new StressTestDirectMessage("test/" + dstPeerAddress), new SendDirectMessageListener() {
+                            @Override
+                            public void onArrived() {
+                                sentDirectLatch.countDown();
+                            }
 
-                        @Override
-                        public void onFault() {
-                            sentDirectFailed.set(true);
-                            sentDirectLatch.countDown();
-                        }
-                    }), MIN_DIRECT_DELAY_MILLIS, MAX_DIRECT_DELAY_MILLIS, TimeUnit.MILLISECONDS
-            );
+                            @Override
+                            public void onFault() {
+                                sentDirectFailed.set(true);
+                                sentDirectLatch.countDown();
+                            }
+                        }), MIN_DIRECT_DELAY_MILLIS, MAX_DIRECT_DELAY_MILLIS, TimeUnit.MILLISECONDS
+                );
+            }
         }
         // Since receiving is completed before sending is reported to be complete,
         // all receiving checks should end before all sending checks to avoid deadlocking.
         // Wait for peers to complete receiving.
         org.junit.Assert.assertTrue("timed out while receiving direct messages",
                 receivedDirectLatch.await(30, TimeUnit.SECONDS));
-        print("receiving 1 direct message per peer took %ss",
+        print("receiving %d direct messages per peer took %ss", DIRECT_COUNT,
                 Duration.between(sendStart, Instant.now()).toMillis()/1000.0);
         // Wait for peers to complete sending.
         org.junit.Assert.assertTrue("timed out while sending direct messages",
                 sentDirectLatch.await(30, TimeUnit.SECONDS));
-        print("sending 1 direct message per peer took %ss",
+        print("sending %d direct messages per peer took %ss", DIRECT_COUNT,
                 Duration.between(sendStart, Instant.now()).toMillis()/1000.0);
         org.junit.Assert.assertFalse("some peer(s) failed to send a direct message", sentDirectFailed.get());
     }

From ec216a8702d837b5cb705c73e782746b6fe93a29 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 3 May 2016 14:01:43 +0200
Subject: [PATCH 044/101] Fix direct message loop in network stress test

The logic was broken and all sends where packed in the same (min delay, max delay) period. Now a variable ensures that the random delay gets added to the previous time.
---
 .../bitsquare/p2p/network/NetworkStressTest.java   | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 73e30d3be2..034725e643 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -204,14 +204,18 @@ public class NetworkStressTest {
                     receivedDirectLatch.countDown();
             });
 
+            long nextSendMillis = System.currentTimeMillis();
             for (int i = 0; i < DIRECT_COUNT; i++) {
-                // Select a random peer and send a direct message to it after a random delay
-                // not shorter than throttle limits.
+                // Select a random peer and send a direct message to it...
                 final int dstPeerIdx = (int) (Math.random() * nPeers);
                 final P2PService dstPeer = peerNodes.get(dstPeerIdx);
                 final NodeAddress dstPeerAddress = dstPeer.getAddress();
-                //print("sending direct message from peer %s to %s", srcPeer.getAddress(), dstPeer.getAddress());
-                UserThread.runAfterRandomDelay(() -> srcPeer.sendEncryptedDirectMessage(
+                // ...after a random delay not shorter than throttle limits.
+                nextSendMillis += Math.round(Math.random() * (MAX_DIRECT_DELAY_MILLIS - MIN_DIRECT_DELAY_MILLIS));
+                final long sendAfterMillis = nextSendMillis - System.currentTimeMillis();
+                /*print("sending direct message from peer %s to %s in %sms",
+                        srcPeer.getAddress(), dstPeer.getAddress(), sendAfterMillis);*/
+                UserThread.runAfter(() -> srcPeer.sendEncryptedDirectMessage(
                         dstPeerAddress, peerPKRings.get(dstPeerIdx),
                         new StressTestDirectMessage("test/" + dstPeerAddress), new SendDirectMessageListener() {
                             @Override
@@ -224,7 +228,7 @@ public class NetworkStressTest {
                                 sentDirectFailed.set(true);
                                 sentDirectLatch.countDown();
                             }
-                        }), MIN_DIRECT_DELAY_MILLIS, MAX_DIRECT_DELAY_MILLIS, TimeUnit.MILLISECONDS
+                        }), sendAfterMillis, TimeUnit.MILLISECONDS
                 );
             }
         }

From 987b28e4e2bf4ae595a14bae9969bc5fca6ceef2 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 3 May 2016 14:16:22 +0200
Subject: [PATCH 045/101] Use parameter-based timeouts for direct messages in
 network stress test

---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 034725e643..a47b58227d 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -234,14 +234,17 @@ public class NetworkStressTest {
         }
         // Since receiving is completed before sending is reported to be complete,
         // all receiving checks should end before all sending checks to avoid deadlocking.
-        // Wait for peers to complete receiving.
+        /** Time to transmit all messages in the worst random case, and with no computation delays. */
+        final long idealMaxDirectDelay = MAX_DIRECT_DELAY_MILLIS * DIRECT_COUNT;
+        // Wait for peers to complete receiving.  We are generous here.
         org.junit.Assert.assertTrue("timed out while receiving direct messages",
-                receivedDirectLatch.await(30, TimeUnit.SECONDS));
+                receivedDirectLatch.await(2 * idealMaxDirectDelay, TimeUnit.MILLISECONDS));
         print("receiving %d direct messages per peer took %ss", DIRECT_COUNT,
                 Duration.between(sendStart, Instant.now()).toMillis()/1000.0);
         // Wait for peers to complete sending.
+        // This should be nearly instantaneous after waiting for reception is completed.
         org.junit.Assert.assertTrue("timed out while sending direct messages",
-                sentDirectLatch.await(30, TimeUnit.SECONDS));
+                sentDirectLatch.await(idealMaxDirectDelay / 10, TimeUnit.MILLISECONDS));
         print("sending %d direct messages per peer took %ss", DIRECT_COUNT,
                 Duration.between(sendStart, Instant.now()).toMillis()/1000.0);
         org.junit.Assert.assertFalse("some peer(s) failed to send a direct message", sentDirectFailed.get());

From e7e45f497e997d291e125a32f68932c9f1c58a53 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 3 May 2016 14:22:24 +0200
Subject: [PATCH 046/101] Use plain system milliseconds instead of instants and
 durations

For consistency with the rest of the stress test.
---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java  | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index a47b58227d..d1d02fadd6 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -28,8 +28,6 @@ import java.io.IOException;
 import java.nio.file.*;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.security.Security;
-import java.time.Duration;
-import java.time.Instant;
 import java.util.*;
 import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.TimeUnit;
@@ -193,7 +191,7 @@ public class NetworkStressTest {
         BooleanProperty sentDirectFailed = new SimpleBooleanProperty(false);
         final CountDownLatch sentDirectLatch = new CountDownLatch(DIRECT_COUNT * nPeers);
         final CountDownLatch receivedDirectLatch = new CountDownLatch(DIRECT_COUNT * nPeers);
-        final Instant sendStart = Instant.now();
+        final long sendStartMillis = System.currentTimeMillis();
         for (final P2PService srcPeer : peerNodes) {
             // Make the peer ready for receiving direct messages.
             srcPeer.addDecryptedDirectMessageListener((decryptedMsgWithPubKey, peerNodeAddress) -> {
@@ -240,13 +238,13 @@ public class NetworkStressTest {
         org.junit.Assert.assertTrue("timed out while receiving direct messages",
                 receivedDirectLatch.await(2 * idealMaxDirectDelay, TimeUnit.MILLISECONDS));
         print("receiving %d direct messages per peer took %ss", DIRECT_COUNT,
-                Duration.between(sendStart, Instant.now()).toMillis()/1000.0);
+                (System.currentTimeMillis() - sendStartMillis)/1000.0);
         // Wait for peers to complete sending.
         // This should be nearly instantaneous after waiting for reception is completed.
         org.junit.Assert.assertTrue("timed out while sending direct messages",
                 sentDirectLatch.await(idealMaxDirectDelay / 10, TimeUnit.MILLISECONDS));
         print("sending %d direct messages per peer took %ss", DIRECT_COUNT,
-                Duration.between(sendStart, Instant.now()).toMillis()/1000.0);
+                (System.currentTimeMillis() - sendStartMillis)/1000.0);
         org.junit.Assert.assertFalse("some peer(s) failed to send a direct message", sentDirectFailed.get());
     }
 

From 97e6b942d31699f77f7c9247677f903b724de934 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 3 May 2016 14:43:55 +0200
Subject: [PATCH 047/101] Customize direct message count per peer from
 environment variable

---
 .../p2p/network/NetworkStressTest.java        | 45 +++++++++++++------
 1 file changed, 31 insertions(+), 14 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index d1d02fadd6..20ef6b7ac3 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -40,6 +40,13 @@ public class NetworkStressTest {
 
     // Constants
 
+    /** Environment variable to specify the number of peers in the test. */
+    private static final String NPEERS_ENVVAR = "STRESS_TEST_NPEERS";
+    /** Environment variable to specify a persistent test data directory. */
+    private static final String TEST_DIR_ENVVAR = "STRESS_TEST_DIR";
+    /** Environment variable to specify the number of direct messages sent per peer. */
+    private static final String DIRECT_COUNT_ENVVAR = "STRESS_TEST_NDIRECT";
+
     /** Numeric identifier of the regtest Bitcoin network. */
     private static final int REGTEST_NETWORK_ID = 2;
 
@@ -47,13 +54,9 @@ public class NetworkStressTest {
     private static final int NPEERS_DEFAULT = 4;
     /** Minimum number of peers for the test to work. */
     private static final int NPEERS_MIN = 2;
-    /** Environment variable to specify the number of peers in the test. */
-    private static final String NPEERS_ENVVAR = "STRESS_TEST_NPEERS";
-    /** Environment variable to specify a persistent test data directory. */
-    private static final String TEST_DIR_ENVVAR = "STRESS_TEST_DIR";
+    /** Default number of direct messages to be sent by each peer. */
+    private static final int DIRECT_COUNT_DEFAULT = 100;
 
-    /** Number of direct messages to be sent by each peer. */
-    private static int DIRECT_COUNT = 100;
     /** Minimum delay between direct messages in milliseconds, 25% larger than throttle limit. */
     private static long MIN_DIRECT_DELAY_MILLIS = Math.round(1.25 * (1.0 / Connection.MSG_THROTTLE_PER_SEC) * 1000);
     /** Maximum delay between direct messages in milliseconds, 10 times larger than minimum. */
@@ -75,17 +78,31 @@ public class NetworkStressTest {
     /** A barrier to wait for concurrent bootstrap of peers. */
     private CountDownLatch bootstrapLatch;
 
+    /** Number of direct messages to be sent by each peer. */
+    private int directCount = DIRECT_COUNT_DEFAULT;
+
     @Before
     public void setUp() throws Exception {
+        // Parse test parameter environment variables.
+
         /** Number of peer nodes to create. */
         int nPeers = NPEERS_DEFAULT;
-
         final String nPeersEnv = System.getenv(NPEERS_ENVVAR);
         if (nPeersEnv != null && !nPeersEnv.equals(""))
             nPeers = Integer.parseInt(nPeersEnv);
         if (nPeers < NPEERS_MIN)
             throw new IllegalArgumentException(
-                    String.format("Test needs at least %d peer nodes to work: %d", NPEERS_MIN, nPeers));
+                    String.format("Test needs at least %d peer nodes to work: %d", NPEERS_MIN, nPeers)
+            );
+
+        final String nDirectEnv = System.getenv(DIRECT_COUNT_ENVVAR);
+        if (nDirectEnv != null && !nDirectEnv.equals(""))
+            directCount = Integer.parseInt(nDirectEnv);
+        if (directCount < 0)
+            throw new IllegalArgumentException(
+                    String.format("Direct messages sent per peer must not be negative: %d", directCount)
+            );
+
         prelimDataLatch = new CountDownLatch(nPeers);
         bootstrapLatch = new CountDownLatch(nPeers);
 
@@ -189,8 +206,8 @@ public class NetworkStressTest {
         // Test each peer sending a direct message to another random peer.
         final int nPeers = peerNodes.size();
         BooleanProperty sentDirectFailed = new SimpleBooleanProperty(false);
-        final CountDownLatch sentDirectLatch = new CountDownLatch(DIRECT_COUNT * nPeers);
-        final CountDownLatch receivedDirectLatch = new CountDownLatch(DIRECT_COUNT * nPeers);
+        final CountDownLatch sentDirectLatch = new CountDownLatch(directCount * nPeers);
+        final CountDownLatch receivedDirectLatch = new CountDownLatch(directCount * nPeers);
         final long sendStartMillis = System.currentTimeMillis();
         for (final P2PService srcPeer : peerNodes) {
             // Make the peer ready for receiving direct messages.
@@ -203,7 +220,7 @@ public class NetworkStressTest {
             });
 
             long nextSendMillis = System.currentTimeMillis();
-            for (int i = 0; i < DIRECT_COUNT; i++) {
+            for (int i = 0; i < directCount; i++) {
                 // Select a random peer and send a direct message to it...
                 final int dstPeerIdx = (int) (Math.random() * nPeers);
                 final P2PService dstPeer = peerNodes.get(dstPeerIdx);
@@ -233,17 +250,17 @@ public class NetworkStressTest {
         // Since receiving is completed before sending is reported to be complete,
         // all receiving checks should end before all sending checks to avoid deadlocking.
         /** Time to transmit all messages in the worst random case, and with no computation delays. */
-        final long idealMaxDirectDelay = MAX_DIRECT_DELAY_MILLIS * DIRECT_COUNT;
+        final long idealMaxDirectDelay = MAX_DIRECT_DELAY_MILLIS * directCount;
         // Wait for peers to complete receiving.  We are generous here.
         org.junit.Assert.assertTrue("timed out while receiving direct messages",
                 receivedDirectLatch.await(2 * idealMaxDirectDelay, TimeUnit.MILLISECONDS));
-        print("receiving %d direct messages per peer took %ss", DIRECT_COUNT,
+        print("receiving %d direct messages per peer took %ss", directCount,
                 (System.currentTimeMillis() - sendStartMillis)/1000.0);
         // Wait for peers to complete sending.
         // This should be nearly instantaneous after waiting for reception is completed.
         org.junit.Assert.assertTrue("timed out while sending direct messages",
                 sentDirectLatch.await(idealMaxDirectDelay / 10, TimeUnit.MILLISECONDS));
-        print("sending %d direct messages per peer took %ss", DIRECT_COUNT,
+        print("sending %d direct messages per peer took %ss", directCount,
                 (System.currentTimeMillis() - sendStartMillis)/1000.0);
         org.junit.Assert.assertFalse("some peer(s) failed to send a direct message", sentDirectFailed.get());
     }

From 8eefecdc9c80a0a1933b366c53bad4c8636ae8ea Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 4 May 2016 11:11:20 +0200
Subject: [PATCH 048/101] Report pending latch counts on network stress test
 timeouts

---
 .../p2p/network/NetworkStressTest.java        | 22 ++++++++++++-------
 1 file changed, 14 insertions(+), 8 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 20ef6b7ac3..789f6f07bd 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -197,11 +197,11 @@ public class NetworkStressTest {
     @Test
     public void test() throws InterruptedException {
         // Wait for peers to get their preliminary data.
-        org.junit.Assert.assertTrue("timed out while waiting for preliminary data",
-                prelimDataLatch.await(30, TimeUnit.SECONDS));
+        assertLatch("timed out while waiting for preliminary data",
+                prelimDataLatch, 30, TimeUnit.SECONDS);
         // Wait for peers to complete their bootstrapping.
-        org.junit.Assert.assertTrue("timed out while waiting for bootstrap",
-                bootstrapLatch.await(30, TimeUnit.SECONDS));
+        assertLatch("timed out while waiting for bootstrap",
+                bootstrapLatch, 30, TimeUnit.SECONDS);
 
         // Test each peer sending a direct message to another random peer.
         final int nPeers = peerNodes.size();
@@ -252,14 +252,14 @@ public class NetworkStressTest {
         /** Time to transmit all messages in the worst random case, and with no computation delays. */
         final long idealMaxDirectDelay = MAX_DIRECT_DELAY_MILLIS * directCount;
         // Wait for peers to complete receiving.  We are generous here.
-        org.junit.Assert.assertTrue("timed out while receiving direct messages",
-                receivedDirectLatch.await(2 * idealMaxDirectDelay, TimeUnit.MILLISECONDS));
+        assertLatch("timed out while receiving direct messages",
+                receivedDirectLatch, 2 * idealMaxDirectDelay, TimeUnit.MILLISECONDS);
         print("receiving %d direct messages per peer took %ss", directCount,
                 (System.currentTimeMillis() - sendStartMillis)/1000.0);
         // Wait for peers to complete sending.
         // This should be nearly instantaneous after waiting for reception is completed.
-        org.junit.Assert.assertTrue("timed out while sending direct messages",
-                sentDirectLatch.await(idealMaxDirectDelay / 10, TimeUnit.MILLISECONDS));
+        assertLatch("timed out while sending direct messages",
+                sentDirectLatch, idealMaxDirectDelay / 10, TimeUnit.MILLISECONDS);
         print("sending %d direct messages per peer took %ss", directCount,
                 (System.currentTimeMillis() - sendStartMillis)/1000.0);
         org.junit.Assert.assertFalse("some peer(s) failed to send a direct message", sentDirectFailed.get());
@@ -270,6 +270,12 @@ public class NetworkStressTest {
             + String.format(message, args));
     }
 
+    private static void assertLatch(String message, CountDownLatch latch, long timeout, TimeUnit unit)
+            throws InterruptedException {
+        if (!latch.await(timeout, unit))
+            org.junit.Assert.fail(String.format("%s (%d pending in latch)", message, latch.getCount()));
+    }
+
     private Path createTestDataDirectory() throws IOException {
         Path stressTestDirPath;
 

From a9efa4ec156bc75cbb7d1a9ee4e7c5c48c4302ce Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 4 May 2016 12:29:54 +0200
Subject: [PATCH 049/101] Print network stress test steps to help locating the
 source of warnings

---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 789f6f07bd..9fc0d0260a 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -116,6 +116,7 @@ public class NetworkStressTest {
 
         // Create the test data directory.
         testDataDir = createTestDataDirectory();
+        print("test data directory: " + testDataDir);
 
         // Create and start the seed node.
         seedNode = new SeedNode(testDataDir.toString());
@@ -126,6 +127,7 @@ public class NetworkStressTest {
         seedNode.createAndStartP2PService(seedNodeAddress, useLocalhost,
                 REGTEST_NETWORK_ID, USE_DETAILED_LOGGING, seedNodes,
                 new SeedServiceListener(localServicesLatch, localServicesFailed));
+        print("created seed node");
 
         // Create and start peer nodes.
         SeedNodesRepository seedNodesRepository = new SeedNodesRepository();
@@ -151,6 +153,7 @@ public class NetworkStressTest {
             peerNodes.add(peer);
             peer.start(new PeerServiceListener(localServicesLatch, localServicesFailed));
         }
+        print("created peer nodes");
 
         // Wait for concurrent tasks to finish.
         localServicesLatch.await();
@@ -159,6 +162,8 @@ public class NetworkStressTest {
         if (localServicesFailed.get()) {
             throw new Exception("nodes failed to start");
         }
+
+        print("all local nodes started");
     }
 
     @NotNull
@@ -187,6 +192,7 @@ public class NetworkStressTest {
         }
         // Wait for concurrent tasks to finish.
         shutdownLatch.await();
+        print("all local nodes stopped");
 
         // Cleanup test data directory.
         if (testDataDir != null) {
@@ -199,9 +205,11 @@ public class NetworkStressTest {
         // Wait for peers to get their preliminary data.
         assertLatch("timed out while waiting for preliminary data",
                 prelimDataLatch, 30, TimeUnit.SECONDS);
+        print("preliminary data received");
         // Wait for peers to complete their bootstrapping.
         assertLatch("timed out while waiting for bootstrap",
                 bootstrapLatch, 30, TimeUnit.SECONDS);
+        print("bootstrap complete");
 
         // Test each peer sending a direct message to another random peer.
         final int nPeers = peerNodes.size();
@@ -247,6 +255,7 @@ public class NetworkStressTest {
                 );
             }
         }
+        print("%d direct messages scheduled to be sent by each of %d peers", directCount, nPeers);
         // Since receiving is completed before sending is reported to be complete,
         // all receiving checks should end before all sending checks to avoid deadlocking.
         /** Time to transmit all messages in the worst random case, and with no computation delays. */

From acb31003edaf6c79f6c60fbe628fe73b9b2a601f Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Fri, 6 May 2016 13:09:09 +0200
Subject: [PATCH 050/101] Set maximum connections to seed node in network
 stress test

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 9fc0d0260a..9cb5e69ba2 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -124,7 +124,7 @@ public class NetworkStressTest {
         final boolean useLocalhost = seedNodeAddress.hostName.equals("localhost");
         final Set<NodeAddress> seedNodes = new HashSet<>(1);
         seedNodes.add(seedNodeAddress);  // the only seed node in tests
-        seedNode.createAndStartP2PService(seedNodeAddress, useLocalhost,
+        seedNode.createAndStartP2PService(seedNodeAddress, SeedNode.MAX_CONNECTIONS_DEFAULT, useLocalhost,
                 REGTEST_NETWORK_ID, USE_DETAILED_LOGGING, seedNodes,
                 new SeedServiceListener(localServicesLatch, localServicesFailed));
         print("created seed node");

From 9e61f1f3dbcbe5e5b165f2c8c20a7db96102b2dc Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Mon, 9 May 2016 09:14:30 +0200
Subject: [PATCH 051/101] Some comments on peer creation in network stress test

---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 9cb5e69ba2..ca708161e0 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -129,7 +129,7 @@ public class NetworkStressTest {
                 new SeedServiceListener(localServicesLatch, localServicesFailed));
         print("created seed node");
 
-        // Create and start peer nodes.
+        // Create and start peer nodes, all connecting to the seed node above.
         SeedNodesRepository seedNodesRepository = new SeedNodesRepository();
         if (useLocalhost) {
             seedNodesRepository.setLocalhostSeedNodeAddresses(seedNodes);
@@ -137,17 +137,24 @@ public class NetworkStressTest {
             seedNodesRepository.setTorSeedNodeAddresses(seedNodes);
         }
         for (int p = 0; p < nPeers; p++) {
+            // peer network port
             final int peerPort = Utils.findFreeSystemPort();
+
+            // peer data directories
             final File peerDir = new File(testDataDir.toFile(), String.format("peer-%06d", p));
             final File peerTorDir = new File(peerDir, "tor");
             final File peerStorageDir = new File(peerDir, "db");
             final File peerKeysDir = new File(peerDir, "keys");
             //noinspection ResultOfMethodCallIgnored
             peerKeysDir.mkdirs();  // needed for creating the key ring
+
+            // peer keys
             final KeyStorage peerKeyStorage = new KeyStorage(peerKeysDir);
             final KeyRing peerKeyRing = new KeyRing(peerKeyStorage);
             peerPKRings.add(peerKeyRing.getPubKeyRing());
             final EncryptionService peerEncryptionService = new EncryptionService(peerKeyRing);
+
+            // create, save and start peer
             final P2PService peer = new P2PService(seedNodesRepository, peerPort, peerTorDir, useLocalhost,
                     REGTEST_NETWORK_ID, peerStorageDir, new Clock(), peerEncryptionService, peerKeyRing);
             peerNodes.add(peer);

From d9f00e6627167d8388982d46055c6810cd8c3cbd Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Mon, 9 May 2016 10:10:52 +0200
Subject: [PATCH 052/101] Wait more for message reception in network stress
 test

If we wait for too little there are lots of connection refused errors when stopping nodes.
---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index ca708161e0..8c21e8835e 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -269,7 +269,7 @@ public class NetworkStressTest {
         final long idealMaxDirectDelay = MAX_DIRECT_DELAY_MILLIS * directCount;
         // Wait for peers to complete receiving.  We are generous here.
         assertLatch("timed out while receiving direct messages",
-                receivedDirectLatch, 2 * idealMaxDirectDelay, TimeUnit.MILLISECONDS);
+                receivedDirectLatch, 10 * idealMaxDirectDelay, TimeUnit.MILLISECONDS);
         print("receiving %d direct messages per peer took %ss", directCount,
                 (System.currentTimeMillis() - sendStartMillis)/1000.0);
         // Wait for peers to complete sending.

From 6ab5daf1c12b2966cfa81f1550930173d3f5b055 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Mon, 9 May 2016 10:54:35 +0200
Subject: [PATCH 053/101] Report ratio to max receive timeout in network stress
 test

My computer is able to handle 50 messages per each of 20 peers with a delay ratio around 3.  With 100 messages or 50 peers, receiving times out even if we wait 10 times the ideal max timeout.
---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java     | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 8c21e8835e..8cff7d5b18 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -270,8 +270,9 @@ public class NetworkStressTest {
         // Wait for peers to complete receiving.  We are generous here.
         assertLatch("timed out while receiving direct messages",
                 receivedDirectLatch, 10 * idealMaxDirectDelay, TimeUnit.MILLISECONDS);
-        print("receiving %d direct messages per peer took %ss", directCount,
-                (System.currentTimeMillis() - sendStartMillis)/1000.0);
+        final long recvMillis = System.currentTimeMillis() - sendStartMillis;
+        print("receiving %d direct messages per peer took %ss (%.2f x ideal max)",
+                directCount, recvMillis/1000.0, recvMillis/(float)idealMaxDirectDelay);
         // Wait for peers to complete sending.
         // This should be nearly instantaneous after waiting for reception is completed.
         assertLatch("timed out while sending direct messages",

From 0599c820beabf43dd3ef989c9eccc7d4c30a6a30 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Mon, 9 May 2016 12:05:48 +0200
Subject: [PATCH 054/101] Show min/max/avg send send delay in network stress
 test

---
 .../p2p/network/NetworkStressTest.java        | 52 +++++++++++++------
 1 file changed, 37 insertions(+), 15 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 8cff7d5b18..35f0d82e0b 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -6,6 +6,7 @@ import io.bitsquare.common.UserThread;
 import io.bitsquare.common.crypto.KeyRing;
 import io.bitsquare.common.crypto.KeyStorage;
 import io.bitsquare.common.crypto.PubKeyRing;
+import io.bitsquare.common.util.Tuple3;
 import io.bitsquare.crypto.EncryptionService;
 import io.bitsquare.p2p.NodeAddress;
 import io.bitsquare.p2p.P2PService;
@@ -221,6 +222,7 @@ public class NetworkStressTest {
         // Test each peer sending a direct message to another random peer.
         final int nPeers = peerNodes.size();
         BooleanProperty sentDirectFailed = new SimpleBooleanProperty(false);
+        final List<Long> sentDelays = new Vector<>(nPeers * directCount);
         final CountDownLatch sentDirectLatch = new CountDownLatch(directCount * nPeers);
         final CountDownLatch receivedDirectLatch = new CountDownLatch(directCount * nPeers);
         final long sendStartMillis = System.currentTimeMillis();
@@ -245,20 +247,24 @@ public class NetworkStressTest {
                 final long sendAfterMillis = nextSendMillis - System.currentTimeMillis();
                 /*print("sending direct message from peer %s to %s in %sms",
                         srcPeer.getAddress(), dstPeer.getAddress(), sendAfterMillis);*/
-                UserThread.runAfter(() -> srcPeer.sendEncryptedDirectMessage(
-                        dstPeerAddress, peerPKRings.get(dstPeerIdx),
-                        new StressTestDirectMessage("test/" + dstPeerAddress), new SendDirectMessageListener() {
-                            @Override
-                            public void onArrived() {
-                                sentDirectLatch.countDown();
-                            }
+                UserThread.runAfter(() -> {
+                            final long sendMillis = System.currentTimeMillis();
+                            srcPeer.sendEncryptedDirectMessage(
+                                    dstPeerAddress, peerPKRings.get(dstPeerIdx),
+                                    new StressTestDirectMessage("test/" + dstPeerAddress), new SendDirectMessageListener() {
+                                        @Override
+                                        public void onArrived() {
+                                            sentDelays.add(System.currentTimeMillis() - sendMillis);
+                                            sentDirectLatch.countDown();
+                                        }
 
-                            @Override
-                            public void onFault() {
-                                sentDirectFailed.set(true);
-                                sentDirectLatch.countDown();
-                            }
-                        }), sendAfterMillis, TimeUnit.MILLISECONDS
+                                        @Override
+                                        public void onFault() {
+                                            sentDirectFailed.set(true);
+                                            sentDirectLatch.countDown();
+                                        }
+                                    });
+                        }, sendAfterMillis, TimeUnit.MILLISECONDS
                 );
             }
         }
@@ -277,8 +283,10 @@ public class NetworkStressTest {
         // This should be nearly instantaneous after waiting for reception is completed.
         assertLatch("timed out while sending direct messages",
                 sentDirectLatch, idealMaxDirectDelay / 10, TimeUnit.MILLISECONDS);
-        print("sending %d direct messages per peer took %ss", directCount,
-                (System.currentTimeMillis() - sendStartMillis)/1000.0);
+        Tuple3<Long, Long, Long> mma = minMaxAvg(sentDelays);
+        print("sending %d direct messages per peer took %ss (min/max/avg %s/%s/%s ms)",
+                directCount, (System.currentTimeMillis() - sendStartMillis)/1000.0,
+                mma.first, mma.second, mma.third);
         org.junit.Assert.assertFalse("some peer(s) failed to send a direct message", sentDirectFailed.get());
     }
 
@@ -340,6 +348,20 @@ public class NetworkStressTest {
         });
     }
 
+    private Tuple3<Long, Long, Long> minMaxAvg(List<Long> l) {
+        long min = Long.MAX_VALUE;
+        long max = Long.MIN_VALUE;
+        long sum = 0;
+        for (long e : l) {
+            if (e < min)
+                min = e;
+            if (e > max)
+                max = e;
+            sum += e;
+        }
+        return new Tuple3<>(min, max, sum / l.size());
+    }
+
     // P2P service listener classes
 
     private class TestSetupListener implements SetupListener {

From 9ad4f371b20bcd41e11be11051488d85567d6121 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 10 May 2016 11:13:02 +0200
Subject: [PATCH 055/101] Split long line

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java  | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 35f0d82e0b..68768d62d7 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -251,7 +251,8 @@ public class NetworkStressTest {
                             final long sendMillis = System.currentTimeMillis();
                             srcPeer.sendEncryptedDirectMessage(
                                     dstPeerAddress, peerPKRings.get(dstPeerIdx),
-                                    new StressTestDirectMessage("test/" + dstPeerAddress), new SendDirectMessageListener() {
+                                    new StressTestDirectMessage("test/" + dstPeerAddress),
+                                    new SendDirectMessageListener() {
                                         @Override
                                         public void onArrived() {
                                             sentDelays.add(System.currentTimeMillis() - sendMillis);

From f74d88cb338662c4624555051384037ed4431103 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 10 May 2016 12:00:18 +0200
Subject: [PATCH 056/101] Factor out peer node creation in network stress test

This will be useful later when stopping and restarting peer nodes in mailbox send test.  I added a getter to ``P2PService`` to get its public keyring.
---
 .../java/io/bitsquare/p2p/P2PService.java     |  5 +++
 .../p2p/network/NetworkStressTest.java        | 38 ++++++++++---------
 2 files changed, 26 insertions(+), 17 deletions(-)

diff --git a/network/src/main/java/io/bitsquare/p2p/P2PService.java b/network/src/main/java/io/bitsquare/p2p/P2PService.java
index 45e63c82b1..13663007ca 100644
--- a/network/src/main/java/io/bitsquare/p2p/P2PService.java
+++ b/network/src/main/java/io/bitsquare/p2p/P2PService.java
@@ -775,6 +775,11 @@ public class P2PService implements SetupListener, MessageListener, ConnectionLis
         return peerManager;
     }
 
+    @VisibleForTesting
+    public KeyRing getKeyRing() {
+        return optionalKeyRing.get();
+    }
+
     ///////////////////////////////////////////////////////////////////////////////////////////
     // Private
     ///////////////////////////////////////////////////////////////////////////////////////////
diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 68768d62d7..6538cb4f5e 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -140,24 +140,9 @@ public class NetworkStressTest {
         for (int p = 0; p < nPeers; p++) {
             // peer network port
             final int peerPort = Utils.findFreeSystemPort();
-
-            // peer data directories
-            final File peerDir = new File(testDataDir.toFile(), String.format("peer-%06d", p));
-            final File peerTorDir = new File(peerDir, "tor");
-            final File peerStorageDir = new File(peerDir, "db");
-            final File peerKeysDir = new File(peerDir, "keys");
-            //noinspection ResultOfMethodCallIgnored
-            peerKeysDir.mkdirs();  // needed for creating the key ring
-
-            // peer keys
-            final KeyStorage peerKeyStorage = new KeyStorage(peerKeysDir);
-            final KeyRing peerKeyRing = new KeyRing(peerKeyStorage);
-            peerPKRings.add(peerKeyRing.getPubKeyRing());
-            final EncryptionService peerEncryptionService = new EncryptionService(peerKeyRing);
-
             // create, save and start peer
-            final P2PService peer = new P2PService(seedNodesRepository, peerPort, peerTorDir, useLocalhost,
-                    REGTEST_NETWORK_ID, peerStorageDir, new Clock(), peerEncryptionService, peerKeyRing);
+            final P2PService peer = createPeerNode(p, peerPort, useLocalhost, seedNodesRepository);
+            peerPKRings.add(peer.getKeyRing().getPubKeyRing());
             peerNodes.add(peer);
             peer.start(new PeerServiceListener(localServicesLatch, localServicesFailed));
         }
@@ -185,6 +170,25 @@ public class NetworkStressTest {
         return new NodeAddress("localhost", port);
     }
 
+    @NotNull
+    private P2PService createPeerNode(int n, int port, boolean useLocalhost, SeedNodesRepository seedNodesRepository) {
+        // peer data directories
+        final File peerDir = new File(testDataDir.toFile(), String.format("peer-%06d", n));
+        final File peerTorDir = new File(peerDir, "tor");
+        final File peerStorageDir = new File(peerDir, "db");
+        final File peerKeysDir = new File(peerDir, "keys");
+        //noinspection ResultOfMethodCallIgnored
+        peerKeysDir.mkdirs();  // needed for creating the key ring
+
+        // peer keys
+        final KeyStorage peerKeyStorage = new KeyStorage(peerKeysDir);
+        final KeyRing peerKeyRing = new KeyRing(peerKeyStorage);
+        final EncryptionService peerEncryptionService = new EncryptionService(peerKeyRing);
+
+        return new P2PService(seedNodesRepository, port, peerTorDir, useLocalhost,
+                REGTEST_NETWORK_ID, peerStorageDir, new Clock(), peerEncryptionService, peerKeyRing);
+    }
+
     @After
     public void tearDown() throws InterruptedException, IOException {
         /** A barrier to wait for concurrent shutdown of services. */

From ab4491d9565b0f876a97dd5747bc7d1e07c3c6d2 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 10 May 2016 12:05:10 +0200
Subject: [PATCH 057/101] Draft and describe loop for mailbox send test

---
 .../p2p/network/NetworkStressTest.java         | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 6538cb4f5e..4e0f84ee04 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -53,8 +53,8 @@ public class NetworkStressTest {
 
     /** Default number of peers in the test. */
     private static final int NPEERS_DEFAULT = 4;
-    /** Minimum number of peers for the test to work. */
-    private static final int NPEERS_MIN = 2;
+    /** Minimum number of peers for the test to work (2 for direct messages, 3 for mailbox messages). */
+    private static final int NPEERS_MIN = 3;
     /** Default number of direct messages to be sent by each peer. */
     private static final int DIRECT_COUNT_DEFAULT = 100;
 
@@ -293,6 +293,20 @@ public class NetworkStressTest {
                 directCount, (System.currentTimeMillis() - sendStartMillis)/1000.0,
                 mma.first, mma.second, mma.third);
         org.junit.Assert.assertFalse("some peer(s) failed to send a direct message", sentDirectFailed.get());
+
+        // Test sending and receiving mailbox messages.
+        // We start by putting the first half of peers online and the second one offline.
+        // Then the first online peer sends a number of messages to random peers (regardless of their state),
+        // so that some messages are delivered directly and others into a mailbox.
+        // Then the first online peer is put offline and the last offline peer is put online
+        // (so it can get its mailbox messages),
+        // and the new first online node sends messages.
+        // This is repeated until all nodes have been online and offline.
+        for (int firstOnline = 0, firstOffline = (int)Math.ceil(nPeers/2.0);
+                firstOnline < nPeers;
+                firstOnline++, firstOffline = ++firstOffline%nPeers) {
+            System.out.println("firstOnline "+firstOnline+" firstOffline "+firstOffline);
+        }
     }
 
     private void print(String message, Object... args) {

From 7351363830a5c65dc85d6eaa89039b526d37b925 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 10 May 2016 14:49:01 +0200
Subject: [PATCH 058/101] Check optional value in keyring getter, hint nullable

---
 network/src/main/java/io/bitsquare/p2p/P2PService.java | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/network/src/main/java/io/bitsquare/p2p/P2PService.java b/network/src/main/java/io/bitsquare/p2p/P2PService.java
index 13663007ca..66a89ec552 100644
--- a/network/src/main/java/io/bitsquare/p2p/P2PService.java
+++ b/network/src/main/java/io/bitsquare/p2p/P2PService.java
@@ -776,8 +776,9 @@ public class P2PService implements SetupListener, MessageListener, ConnectionLis
     }
 
     @VisibleForTesting
+    @Nullable
     public KeyRing getKeyRing() {
-        return optionalKeyRing.get();
+        return optionalKeyRing.isPresent()? optionalKeyRing.get() : null;
     }
 
     ///////////////////////////////////////////////////////////////////////////////////////////

From 5a4886ffa4cceacd55cf5bc4a340a45e585917a3 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 10 May 2016 15:09:48 +0200
Subject: [PATCH 059/101] Keep track of peer ports and seed nodes repo

It will be useful when recreating nodes for mailbox send test.
---
 .../io/bitsquare/p2p/network/NetworkStressTest.java   | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 4e0f84ee04..ab5376e341 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -69,8 +69,12 @@ public class NetworkStressTest {
     private Path testDataDir;
     /** A single seed node that other nodes will contact to request initial data. */
     private SeedNode seedNode;
+    /** The repository of seed nodes used in the test. */
+    private SeedNodesRepository seedNodesRepository = new SeedNodesRepository();
     /** A list of peer nodes represented as P2P services. */
     private List<P2PService> peerNodes = new ArrayList<>();
+    /** A list of peer node's service ports. */
+    private List<Integer> peerPorts = new ArrayList<>();
     /** A list of peer node's public key rings. */
     private List<PubKeyRing> peerPKRings = new ArrayList<>();
 
@@ -131,7 +135,6 @@ public class NetworkStressTest {
         print("created seed node");
 
         // Create and start peer nodes, all connecting to the seed node above.
-        SeedNodesRepository seedNodesRepository = new SeedNodesRepository();
         if (useLocalhost) {
             seedNodesRepository.setLocalhostSeedNodeAddresses(seedNodes);
         } else {
@@ -140,8 +143,10 @@ public class NetworkStressTest {
         for (int p = 0; p < nPeers; p++) {
             // peer network port
             final int peerPort = Utils.findFreeSystemPort();
+            peerPorts.add(peerPort);
             // create, save and start peer
-            final P2PService peer = createPeerNode(p, peerPort, useLocalhost, seedNodesRepository);
+            final P2PService peer = createPeerNode(p, peerPort, useLocalhost);
+            //noinspection ConstantConditions
             peerPKRings.add(peer.getKeyRing().getPubKeyRing());
             peerNodes.add(peer);
             peer.start(new PeerServiceListener(localServicesLatch, localServicesFailed));
@@ -171,7 +176,7 @@ public class NetworkStressTest {
     }
 
     @NotNull
-    private P2PService createPeerNode(int n, int port, boolean useLocalhost, SeedNodesRepository seedNodesRepository) {
+    private P2PService createPeerNode(int n, int port, boolean useLocalhost) {
         // peer data directories
         final File peerDir = new File(testDataDir.toFile(), String.format("peer-%06d", n));
         final File peerTorDir = new File(peerDir, "tor");

From 39c951267357a9c9fbefa46f517744ee702c5638 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 10 May 2016 15:23:22 +0200
Subject: [PATCH 060/101] Rename method to get seed node address for network
 stress test

This one better conveys the intention of providing a new address each time it is called.
---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index ab5376e341..eb1b219c21 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -125,7 +125,7 @@ public class NetworkStressTest {
 
         // Create and start the seed node.
         seedNode = new SeedNode(testDataDir.toString());
-        final NodeAddress seedNodeAddress = getSeedNodeAddress();
+        final NodeAddress seedNodeAddress = newSeedNodeAddress();
         final boolean useLocalhost = seedNodeAddress.hostName.equals("localhost");
         final Set<NodeAddress> seedNodes = new HashSet<>(1);
         seedNodes.add(seedNodeAddress);  // the only seed node in tests
@@ -165,7 +165,7 @@ public class NetworkStressTest {
     }
 
     @NotNull
-    private static NodeAddress getSeedNodeAddress() {
+    private static NodeAddress newSeedNodeAddress() {
         // The address is only considered by ``SeedNodesRepository`` if
         // it ends in the digit matching the network identifier.
         int port;

From 4a264dbdc73875cc480a63ecbd2f065a017f5e27 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 10 May 2016 15:30:55 +0200
Subject: [PATCH 061/101] Make use localhost a field in network stress test

It will help with the recreation of peers for mailbox send test.
---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java  | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index eb1b219c21..7374ca4922 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -67,6 +67,8 @@ public class NetworkStressTest {
 
     /** A directory to (temporarily) hold seed and normal nodes' configuration and state files. */
     private Path testDataDir;
+    /** Whether to use localhost addresses instead of Tor hidden services. */
+    private boolean useLocalhost;
     /** A single seed node that other nodes will contact to request initial data. */
     private SeedNode seedNode;
     /** The repository of seed nodes used in the test. */
@@ -126,7 +128,7 @@ public class NetworkStressTest {
         // Create and start the seed node.
         seedNode = new SeedNode(testDataDir.toString());
         final NodeAddress seedNodeAddress = newSeedNodeAddress();
-        final boolean useLocalhost = seedNodeAddress.hostName.equals("localhost");
+        useLocalhost = seedNodeAddress.hostName.equals("localhost");
         final Set<NodeAddress> seedNodes = new HashSet<>(1);
         seedNodes.add(seedNodeAddress);  // the only seed node in tests
         seedNode.createAndStartP2PService(seedNodeAddress, SeedNode.MAX_CONNECTIONS_DEFAULT, useLocalhost,
@@ -145,7 +147,7 @@ public class NetworkStressTest {
             final int peerPort = Utils.findFreeSystemPort();
             peerPorts.add(peerPort);
             // create, save and start peer
-            final P2PService peer = createPeerNode(p, peerPort, useLocalhost);
+            final P2PService peer = createPeerNode(p, peerPort);
             //noinspection ConstantConditions
             peerPKRings.add(peer.getKeyRing().getPubKeyRing());
             peerNodes.add(peer);
@@ -176,7 +178,7 @@ public class NetworkStressTest {
     }
 
     @NotNull
-    private P2PService createPeerNode(int n, int port, boolean useLocalhost) {
+    private P2PService createPeerNode(int n, int port) {
         // peer data directories
         final File peerDir = new File(testDataDir.toFile(), String.format("peer-%06d", n));
         final File peerTorDir = new File(peerDir, "tor");

From 444f9d90b86c56e164eb474ac4cdfbe50052f097 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 10 May 2016 15:35:47 +0200
Subject: [PATCH 062/101] Example restarting of peer in network stress test

This mechanism will be used for the mailbox send test.
---
 .../p2p/network/NetworkStressTest.java        | 46 +++++++++++++++++++
 1 file changed, 46 insertions(+)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 7374ca4922..80727663e1 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -309,6 +309,52 @@ public class NetworkStressTest {
         // (so it can get its mailbox messages),
         // and the new first online node sends messages.
         // This is repeated until all nodes have been online and offline.
+        final CountDownLatch shutDownLatch = new CountDownLatch(1);
+        P2PService peer = peerNodes.get(0);
+        peer.shutDown(shutDownLatch::countDown);
+        shutDownLatch.await();
+
+        peer = createPeerNode(0, peerPorts.get(0));
+        peerNodes.set(0, peer);
+        final CountDownLatch bootLatch = new CountDownLatch(1);
+        peer.start(new P2PServiceListener() {
+            @Override
+            public void onRequestingDataCompleted() {
+
+            }
+
+            @Override
+            public void onNoSeedNodeAvailable() {
+
+            }
+
+            @Override
+            public void onNoPeersAvailable() {
+
+            }
+
+            @Override
+            public void onBootstrapComplete() {
+                bootLatch.countDown();
+            }
+
+            @Override
+            public void onTorNodeReady() {
+
+            }
+
+            @Override
+            public void onHiddenServicePublished() {
+
+            }
+
+            @Override
+            public void onSetupFailed(Throwable throwable) {
+
+            }
+        });
+        bootLatch.await();
+
         for (int firstOnline = 0, firstOffline = (int)Math.ceil(nPeers/2.0);
                 firstOnline < nPeers;
                 firstOnline++, firstOffline = ++firstOffline%nPeers) {

From ad6c0599f602cfbbaeaf540f5e1869b9395f580c Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 10 May 2016 17:31:58 +0200
Subject: [PATCH 063/101] Wait for preliminary data and bootstrap to setup of
 network stress tests

---
 .../p2p/network/NetworkStressTest.java        | 19 ++++++++++---------
 1 file changed, 10 insertions(+), 9 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 80727663e1..a91fba09fe 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -164,6 +164,16 @@ public class NetworkStressTest {
         }
 
         print("all local nodes started");
+
+        // Wait for peers to get their preliminary data.
+        assertLatch("timed out while waiting for preliminary data",
+                prelimDataLatch, 30, TimeUnit.SECONDS);
+        print("preliminary data received");
+
+        // Wait for peers to complete their bootstrapping.
+        assertLatch("timed out while waiting for bootstrap",
+                bootstrapLatch, 30, TimeUnit.SECONDS);
+        print("bootstrap complete");
     }
 
     @NotNull
@@ -221,15 +231,6 @@ public class NetworkStressTest {
 
     @Test
     public void test() throws InterruptedException {
-        // Wait for peers to get their preliminary data.
-        assertLatch("timed out while waiting for preliminary data",
-                prelimDataLatch, 30, TimeUnit.SECONDS);
-        print("preliminary data received");
-        // Wait for peers to complete their bootstrapping.
-        assertLatch("timed out while waiting for bootstrap",
-                bootstrapLatch, 30, TimeUnit.SECONDS);
-        print("bootstrap complete");
-
         // Test each peer sending a direct message to another random peer.
         final int nPeers = peerNodes.size();
         BooleanProperty sentDirectFailed = new SimpleBooleanProperty(false);

From c67f724e0f9384429d0c0a1f6aef87956ad3b09d Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 10 May 2016 17:40:54 +0200
Subject: [PATCH 064/101] Wait for preliminary data and bootstrap while setting
 up network stress tests

This should be common for the different tests.
---
 .../p2p/network/NetworkStressTest.java        | 29 ++++++++++---------
 1 file changed, 16 insertions(+), 13 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index a91fba09fe..f3d211fde5 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -80,11 +80,6 @@ public class NetworkStressTest {
     /** A list of peer node's public key rings. */
     private List<PubKeyRing> peerPKRings = new ArrayList<>();
 
-    /** A barrier to wait for concurrent reception of preliminary data in peers. */
-    private CountDownLatch prelimDataLatch;
-    /** A barrier to wait for concurrent bootstrap of peers. */
-    private CountDownLatch bootstrapLatch;
-
     /** Number of direct messages to be sent by each peer. */
     private int directCount = DIRECT_COUNT_DEFAULT;
 
@@ -109,14 +104,15 @@ public class NetworkStressTest {
             throw new IllegalArgumentException(
                     String.format("Direct messages sent per peer must not be negative: %d", directCount)
             );
-
-        prelimDataLatch = new CountDownLatch(nPeers);
-        bootstrapLatch = new CountDownLatch(nPeers);
-
         /** A property where threads can indicate setup failure of local services (Tor node, hidden service). */
         final BooleanProperty localServicesFailed = new SimpleBooleanProperty(false);
         /** A barrier to wait for concurrent setup of local services (Tor node, hidden service). */
         final CountDownLatch localServicesLatch = new CountDownLatch(1 /*seed node*/ + nPeers);
+        /* A barrier to wait for concurrent reception of preliminary data in peers. */
+        final CountDownLatch prelimDataLatch = new CountDownLatch(nPeers);
+        /* A barrier to wait for concurrent bootstrap of peers. */
+        final CountDownLatch bootstrapLatch = new CountDownLatch(nPeers);
+
 
         // Set a security provider to allow key generation.
         Security.addProvider(new BouncyCastleProvider());
@@ -151,7 +147,8 @@ public class NetworkStressTest {
             //noinspection ConstantConditions
             peerPKRings.add(peer.getKeyRing().getPubKeyRing());
             peerNodes.add(peer);
-            peer.start(new PeerServiceListener(localServicesLatch, localServicesFailed));
+            peer.start(new PeerServiceListener(
+                    localServicesLatch, localServicesFailed, prelimDataLatch, bootstrapLatch));
         }
         print("created peer nodes");
 
@@ -492,14 +489,20 @@ public class NetworkStressTest {
     }
 
     private class PeerServiceListener extends TestSetupListener implements P2PServiceListener {
-        PeerServiceListener(CountDownLatch localServicesLatch, BooleanProperty localServicesFailed) {
+        private final CountDownLatch prelimDataLatch;
+        private final CountDownLatch bootstrapLatch;
+
+        PeerServiceListener(CountDownLatch localServicesLatch, BooleanProperty localServicesFailed,
+                            CountDownLatch prelimDataLatch, CountDownLatch bootstrapLatch) {
             super(localServicesLatch, localServicesFailed);
+            this.prelimDataLatch = prelimDataLatch;
+            this.bootstrapLatch = bootstrapLatch;
         }
 
         @Override
         public void onRequestingDataCompleted() {
             // preliminary data received
-            NetworkStressTest.this.prelimDataLatch.countDown();
+            prelimDataLatch.countDown();
         }
 
         @Override
@@ -515,7 +518,7 @@ public class NetworkStressTest {
         @Override
         public void onBootstrapComplete() {
             // peer bootstrapped
-            NetworkStressTest.this.bootstrapLatch.countDown();
+            bootstrapLatch.countDown();
         }
     }
 }

From 4b75235f339dadafa78e783bb65ccc8a5606d93f Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 10 May 2016 18:42:28 +0200
Subject: [PATCH 065/101] Split network stress tests in direct and mailbox

Now they can be run independently.
---
 .../bitsquare/p2p/network/NetworkStressTest.java   | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index f3d211fde5..a3e56421c8 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -226,9 +226,9 @@ public class NetworkStressTest {
         }
     }
 
+    /** Test each peer sending a direct message to another random peer. */
     @Test
-    public void test() throws InterruptedException {
-        // Test each peer sending a direct message to another random peer.
+    public void test_direct() throws InterruptedException {
         final int nPeers = peerNodes.size();
         BooleanProperty sentDirectFailed = new SimpleBooleanProperty(false);
         final List<Long> sentDelays = new Vector<>(nPeers * directCount);
@@ -288,18 +288,21 @@ public class NetworkStressTest {
                 receivedDirectLatch, 10 * idealMaxDirectDelay, TimeUnit.MILLISECONDS);
         final long recvMillis = System.currentTimeMillis() - sendStartMillis;
         print("receiving %d direct messages per peer took %ss (%.2f x ideal max)",
-                directCount, recvMillis/1000.0, recvMillis/(float)idealMaxDirectDelay);
+                directCount, recvMillis / 1000.0, recvMillis / (float) idealMaxDirectDelay);
         // Wait for peers to complete sending.
         // This should be nearly instantaneous after waiting for reception is completed.
         assertLatch("timed out while sending direct messages",
                 sentDirectLatch, idealMaxDirectDelay / 10, TimeUnit.MILLISECONDS);
         Tuple3<Long, Long, Long> mma = minMaxAvg(sentDelays);
         print("sending %d direct messages per peer took %ss (min/max/avg %s/%s/%s ms)",
-                directCount, (System.currentTimeMillis() - sendStartMillis)/1000.0,
+                directCount, (System.currentTimeMillis() - sendStartMillis) / 1000.0,
                 mma.first, mma.second, mma.third);
         org.junit.Assert.assertFalse("some peer(s) failed to send a direct message", sentDirectFailed.get());
+    }
 
-        // Test sending and receiving mailbox messages.
+    /** Test sending and receiving mailbox messages. */
+    @Test
+    public void test_mailbox() throws InterruptedException {
         // We start by putting the first half of peers online and the second one offline.
         // Then the first online peer sends a number of messages to random peers (regardless of their state),
         // so that some messages are delivered directly and others into a mailbox.
@@ -353,6 +356,7 @@ public class NetworkStressTest {
         });
         bootLatch.await();
 
+        final int nPeers = peerNodes.size();
         for (int firstOnline = 0, firstOffline = (int)Math.ceil(nPeers/2.0);
                 firstOnline < nPeers;
                 firstOnline++, firstOffline = ++firstOffline%nPeers) {

From 0fc8a1b5c0c4d3ab936c8c619e66f938ed127d95 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 10 May 2016 18:45:13 +0200
Subject: [PATCH 066/101] Add main function to run an individual network stress
 test

---
 .../io/bitsquare/p2p/network/NetworkStressTest.java | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index a3e56421c8..7fbd61f745 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -23,6 +23,9 @@ import org.jetbrains.annotations.NotNull;
 import org.junit.After;
 import org.junit.Before;
 import org.junit.Test;
+import org.junit.runner.JUnitCore;
+import org.junit.runner.Request;
+import org.junit.runner.Result;
 
 import java.io.File;
 import java.io.IOException;
@@ -83,6 +86,16 @@ public class NetworkStressTest {
     /** Number of direct messages to be sent by each peer. */
     private int directCount = DIRECT_COUNT_DEFAULT;
 
+    // Inspired by <https://stackoverflow.com/a/9288513> by Marc Peters.
+    public static void main(String[] args) {
+        Request request = (args.length == 0)
+                ? Request.aClass(NetworkStressTest.class)
+                : Request.method(NetworkStressTest.class, args[0]);
+
+        Result result = new JUnitCore().run(request);
+        System.exit(result.wasSuccessful() ? 0 : 1);
+    }
+
     @Before
     public void setUp() throws Exception {
         // Parse test parameter environment variables.

From c4bb180dbf346bb9c5e7e5da17d482a8be189fcf Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 11 May 2016 18:25:47 +0200
Subject: [PATCH 067/101] Show exceptions after network stress test run as main
 class

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java  | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 7fbd61f745..d22397e098 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -26,6 +26,7 @@ import org.junit.Test;
 import org.junit.runner.JUnitCore;
 import org.junit.runner.Request;
 import org.junit.runner.Result;
+import org.junit.runner.notification.Failure;
 
 import java.io.File;
 import java.io.IOException;
@@ -93,6 +94,8 @@ public class NetworkStressTest {
                 : Request.method(NetworkStressTest.class, args[0]);
 
         Result result = new JUnitCore().run(request);
+        for (Failure f : result.getFailures())
+            System.err.printf("\n%s\n%s", f, f.getTrace());
         System.exit(result.wasSuccessful() ? 0 : 1);
     }
 

From fa461f23e8f116a8950a755b63b41283851e06cc Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 11 May 2016 18:34:51 +0200
Subject: [PATCH 068/101] Wait 2 seconds per peer for preliminary data and
 bootstrap

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index d22397e098..280d1957c5 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -180,12 +180,12 @@ public class NetworkStressTest {
 
         // Wait for peers to get their preliminary data.
         assertLatch("timed out while waiting for preliminary data",
-                prelimDataLatch, 30, TimeUnit.SECONDS);
+                prelimDataLatch, 2 * nPeers, TimeUnit.SECONDS);
         print("preliminary data received");
 
         // Wait for peers to complete their bootstrapping.
         assertLatch("timed out while waiting for bootstrap",
-                bootstrapLatch, 30, TimeUnit.SECONDS);
+                bootstrapLatch, 2 * nPeers, TimeUnit.SECONDS);
         print("bootstrap complete");
     }
 

From da5b59b06b18ef6c387550bbbfcd3a81eb68eb28 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 11 May 2016 20:29:52 +0200
Subject: [PATCH 069/101] Set a ``UserThread`` executor in network stress test

That makes it more stable against ``ConcurrentModificationException`` (see #443), but may need to allow more open files for the process.
---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java   | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 280d1957c5..60e3918c57 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -35,6 +35,7 @@ import java.nio.file.attribute.BasicFileAttributes;
 import java.security.Security;
 import java.util.*;
 import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
 public class NetworkStressTest {
@@ -137,6 +138,12 @@ public class NetworkStressTest {
         testDataDir = createTestDataDirectory();
         print("test data directory: " + testDataDir);
 
+        // Setting the executor seems to make tests more stable against ``ConcurrentModificationException``
+        // (see #443).  However it make it use more open files, so you may need to use ``ulimit -n NUMBER``
+        // or run ``prlimit -nNUMBER -pPID`` (as root) on your shell's PID if you get too many open files errors.
+        // NUMBER=16384 seems to be enough for 100 peers in Debian GNU/Linux.
+        UserThread.setExecutor(Executors.newSingleThreadExecutor());
+
         // Create and start the seed node.
         seedNode = new SeedNode(testDataDir.toString());
         final NodeAddress seedNodeAddress = newSeedNodeAddress();

From 2881a3dd63acb7133825481bb1d7e44a23464454 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 11 May 2016 20:33:31 +0200
Subject: [PATCH 070/101] Indicate when nodes are being stopped in the network
 stress test

It may help distinguish errors derived from forcibly stopping nodes (e.g. after a timeout).
---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java   | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 60e3918c57..8d3d063929 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -228,6 +228,8 @@ public class NetworkStressTest {
 
     @After
     public void tearDown() throws InterruptedException, IOException {
+        print("stopping all local nodes");
+
         /** A barrier to wait for concurrent shutdown of services. */
         final CountDownLatch shutdownLatch = new CountDownLatch((seedNode != null? 1 : 0) + peerNodes.size());
 

From 84035ae6589702fc3f03db70716e18182e931b13 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 11 May 2016 20:46:40 +0200
Subject: [PATCH 071/101] Stop seed node after peer nodes in network stress
 test

Probably not really needed.
---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java  | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 8d3d063929..ee6eecf022 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -233,14 +233,14 @@ public class NetworkStressTest {
         /** A barrier to wait for concurrent shutdown of services. */
         final CountDownLatch shutdownLatch = new CountDownLatch((seedNode != null? 1 : 0) + peerNodes.size());
 
-        // Stop the seed node.
-        if (seedNode != null) {
-            seedNode.shutDown(shutdownLatch::countDown);
-        }
         // Stop peer nodes.
         for (P2PService peer : peerNodes) {
             peer.shutDown(shutdownLatch::countDown);
         }
+        // Stop the seed node.
+        if (seedNode != null) {
+            seedNode.shutDown(shutdownLatch::countDown);
+        }
         // Wait for concurrent tasks to finish.
         shutdownLatch.await();
         print("all local nodes stopped");

From ac3dbe9160e8d0f4ffe9fd9374f9a140ef68733d Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 11 May 2016 20:57:48 +0200
Subject: [PATCH 072/101] Wait longer for direct receive in network stress test

Using the new executor made thing a little slower.
---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index ee6eecf022..cd1051d919 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -310,7 +310,7 @@ public class NetworkStressTest {
         final long idealMaxDirectDelay = MAX_DIRECT_DELAY_MILLIS * directCount;
         // Wait for peers to complete receiving.  We are generous here.
         assertLatch("timed out while receiving direct messages",
-                receivedDirectLatch, 10 * idealMaxDirectDelay, TimeUnit.MILLISECONDS);
+                receivedDirectLatch, 25 * idealMaxDirectDelay, TimeUnit.MILLISECONDS);
         final long recvMillis = System.currentTimeMillis() - sendStartMillis;
         print("receiving %d direct messages per peer took %ss (%.2f x ideal max)",
                 directCount, recvMillis / 1000.0, recvMillis / (float) idealMaxDirectDelay);

From 663cd82432480803fc59af2fa44aa1ea9eec64aa Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 08:48:22 +0200
Subject: [PATCH 073/101] Show progress of network stress test operations

---
 .../p2p/network/NetworkStressTest.java        | 26 ++++++++++++++++---
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index cd1051d919..01bea548ab 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -100,6 +100,24 @@ public class NetworkStressTest {
         System.exit(result.wasSuccessful() ? 0 : 1);
     }
 
+    /** Return a string of a character repeated a number of times. */
+    private static String nChars(char c, int n) {
+        String ret = "";
+        for (int i = 0; i < n; i++)
+            ret += c;
+        return ret;
+    }
+
+    /** Decrease latch count and print pending as a progress bar based on the given character. */
+    private void countDownAndPrint(CountDownLatch latch, char c) {
+        latch.countDown();
+        int remaining = (int)latch.getCount();
+        if (remaining > 0)
+            System.err.print(String.format("\r%s: %s ", this.getClass().getSimpleName(), nChars(c, remaining)));
+        if (remaining == 1)
+            System.err.println();
+    }
+
     @Before
     public void setUp() throws Exception {
         // Parse test parameter environment variables.
@@ -235,11 +253,11 @@ public class NetworkStressTest {
 
         // Stop peer nodes.
         for (P2PService peer : peerNodes) {
-            peer.shutDown(shutdownLatch::countDown);
+            peer.shutDown(() -> countDownAndPrint(shutdownLatch, '.'));
         }
         // Stop the seed node.
         if (seedNode != null) {
-            seedNode.shutDown(shutdownLatch::countDown);
+            seedNode.shutDown(() -> countDownAndPrint(shutdownLatch, '.'));
         }
         // Wait for concurrent tasks to finish.
         shutdownLatch.await();
@@ -531,7 +549,7 @@ public class NetworkStressTest {
         @Override
         public void onRequestingDataCompleted() {
             // preliminary data received
-            prelimDataLatch.countDown();
+            countDownAndPrint(prelimDataLatch, 'p');
         }
 
         @Override
@@ -547,7 +565,7 @@ public class NetworkStressTest {
         @Override
         public void onBootstrapComplete() {
             // peer bootstrapped
-            bootstrapLatch.countDown();
+            countDownAndPrint(bootstrapLatch, 'b');
         }
     }
 }

From 0e9bcff2c894ad75a0d44cfcea287341a27e5558 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 08:56:14 +0200
Subject: [PATCH 074/101] Better progress reporting in network stress test
 shutdown

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 01bea548ab..723ef56f9d 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -246,11 +246,10 @@ public class NetworkStressTest {
 
     @After
     public void tearDown() throws InterruptedException, IOException {
-        print("stopping all local nodes");
-
         /** A barrier to wait for concurrent shutdown of services. */
         final CountDownLatch shutdownLatch = new CountDownLatch((seedNode != null? 1 : 0) + peerNodes.size());
 
+        print("stopping all local nodes");
         // Stop peer nodes.
         for (P2PService peer : peerNodes) {
             peer.shutDown(() -> countDownAndPrint(shutdownLatch, '.'));
@@ -264,6 +263,7 @@ public class NetworkStressTest {
         print("all local nodes stopped");
 
         // Cleanup test data directory.
+        print("cleaning up test data directory");
         if (testDataDir != null) {
             deleteTestDataDirectory();
         }

From 7b0834e3b87b08ea541e1910a4a8b43a02f9da79 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 09:01:41 +0200
Subject: [PATCH 075/101] Change format of progress report in network stress
 test

Printing on stdout with a different format makes progress bars easier to filter out.
---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java     | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 723ef56f9d..b6fac59d56 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -113,9 +113,10 @@ public class NetworkStressTest {
         latch.countDown();
         int remaining = (int)latch.getCount();
         if (remaining > 0)
-            System.err.print(String.format("\r%s: %s ", this.getClass().getSimpleName(), nChars(c, remaining)));
+            System.out.print(String.format("\r%s> %s ", this.getClass().getSimpleName(), nChars(c, remaining)));
         if (remaining == 1)
-            System.err.println();
+            System.out.print('\n');
+        System.out.flush();
     }
 
     @Before

From 3a9b2dd9323da3c8e8467d47a78c1d9fbbc55595 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 09:13:48 +0200
Subject: [PATCH 076/101] Show progress of direct sent messages in network
 stress test

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index b6fac59d56..e636129fa3 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -309,13 +309,13 @@ public class NetworkStressTest {
                                         @Override
                                         public void onArrived() {
                                             sentDelays.add(System.currentTimeMillis() - sendMillis);
-                                            sentDirectLatch.countDown();
+                                            countDownAndPrint(sentDirectLatch, 'd');
                                         }
 
                                         @Override
                                         public void onFault() {
                                             sentDirectFailed.set(true);
-                                            sentDirectLatch.countDown();
+                                            countDownAndPrint(sentDirectLatch, 'd');
                                         }
                                     });
                         }, sendAfterMillis, TimeUnit.MILLISECONDS

From e7fd9d9ed79f86816c56a7085d3092e44a60f878 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 09:40:04 +0200
Subject: [PATCH 077/101] Shorten progress bar when it exceeds terminal width

---
 .../bitsquare/p2p/network/NetworkStressTest.java | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index e636129fa3..2e31a1eacb 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -70,6 +70,8 @@ public class NetworkStressTest {
 
     // Instance fields
 
+    /** Number of columns in terminal (for progress reporting). */
+    private int terminalColumns = 80;
     /** A directory to (temporarily) hold seed and normal nodes' configuration and state files. */
     private Path testDataDir;
     /** Whether to use localhost addresses instead of Tor hidden services. */
@@ -112,8 +114,13 @@ public class NetworkStressTest {
     private void countDownAndPrint(CountDownLatch latch, char c) {
         latch.countDown();
         int remaining = (int)latch.getCount();
-        if (remaining > 0)
-            System.out.print(String.format("\r%s> %s ", this.getClass().getSimpleName(), nChars(c, remaining)));
+        if (remaining > 0) {
+            String hdr = String.format("\r%s> ", this.getClass().getSimpleName());
+            String msg = (hdr.length() - 1/*carriage return*/ + remaining + 1/*final space*/ > terminalColumns)
+                    ? String.format("%c*%d", c, remaining)
+                    : nChars(c, remaining);
+            System.out.print(hdr + msg + ' ');
+        }
         if (remaining == 1)
             System.out.print('\n');
         System.out.flush();
@@ -140,6 +147,11 @@ public class NetworkStressTest {
             throw new IllegalArgumentException(
                     String.format("Direct messages sent per peer must not be negative: %d", directCount)
             );
+
+        final String terminalColumnsEnv = System.getenv("COLUMNS");
+        if (terminalColumnsEnv != null && !terminalColumnsEnv.equals(""))
+            terminalColumns = Integer.parseInt(terminalColumnsEnv);
+
         /** A property where threads can indicate setup failure of local services (Tor node, hidden service). */
         final BooleanProperty localServicesFailed = new SimpleBooleanProperty(false);
         /** A barrier to wait for concurrent setup of local services (Tor node, hidden service). */

From dfaf6302b9da2b8ab80a66063fd17e7b52fdb5fd Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 09:47:34 +0200
Subject: [PATCH 078/101] Throttle progress printing in network stress test

---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java    | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 2e31a1eacb..11c58fb840 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -72,6 +72,8 @@ public class NetworkStressTest {
 
     /** Number of columns in terminal (for progress reporting). */
     private int terminalColumns = 80;
+    /** The last time a progress bar update was printed (to throttle message printing). */
+    private long lastProgressUpdateMillis = 0;
     /** A directory to (temporarily) hold seed and normal nodes' configuration and state files. */
     private Path testDataDir;
     /** Whether to use localhost addresses instead of Tor hidden services. */
@@ -114,6 +116,10 @@ public class NetworkStressTest {
     private void countDownAndPrint(CountDownLatch latch, char c) {
         latch.countDown();
         int remaining = (int)latch.getCount();
+        long now = System.currentTimeMillis();
+        if (now - lastProgressUpdateMillis < 500)
+            return;  // throttle message printing below half a second
+        lastProgressUpdateMillis = now;
         if (remaining > 0) {
             String hdr = String.format("\r%s> ", this.getClass().getSimpleName());
             String msg = (hdr.length() - 1/*carriage return*/ + remaining + 1/*final space*/ > terminalColumns)

From 53019a88f2c4ea4413e9a01bc121dd43e7354662 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 09:51:56 +0200
Subject: [PATCH 079/101] Split latch count down and progress bar printing
 methods

---
 .../p2p/network/NetworkStressTest.java        | 22 +++++++++++--------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 11c58fb840..49e90678fe 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -112,26 +112,30 @@ public class NetworkStressTest {
         return ret;
     }
 
-    /** Decrease latch count and print pending as a progress bar based on the given character. */
-    private void countDownAndPrint(CountDownLatch latch, char c) {
-        latch.countDown();
-        int remaining = (int)latch.getCount();
+    /** Print a progress bar based on the given character. */
+    private void printProgress(char c, int n) {
         long now = System.currentTimeMillis();
         if (now - lastProgressUpdateMillis < 500)
             return;  // throttle message printing below half a second
         lastProgressUpdateMillis = now;
-        if (remaining > 0) {
+        if (n > 0) {
             String hdr = String.format("\r%s> ", this.getClass().getSimpleName());
-            String msg = (hdr.length() - 1/*carriage return*/ + remaining + 1/*final space*/ > terminalColumns)
-                    ? String.format("%c*%d", c, remaining)
-                    : nChars(c, remaining);
+            String msg = (hdr.length() - 1/*carriage return*/ + n + 1/*final space*/ > terminalColumns)
+                    ? String.format("%c*%d", c, n)
+                    : nChars(c, n);
             System.out.print(hdr + msg + ' ');
         }
-        if (remaining == 1)
+        if (n == 1)
             System.out.print('\n');
         System.out.flush();
     }
 
+    /** Decrease latch count and print pending as a progress bar based on the given character. */
+    private void countDownAndPrint(CountDownLatch latch, char c) {
+        latch.countDown();
+        printProgress(c, (int)latch.getCount());
+    }
+
     @Before
     public void setUp() throws Exception {
         // Parse test parameter environment variables.

From 7f1204e09ec749ca29c3f430a40096cbd7eb11db Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 10:04:22 +0200
Subject: [PATCH 080/101] Simplify test progress reporting, force printing on
 last entry

---
 .../p2p/network/NetworkStressTest.java        | 20 ++++++++++---------
 1 file changed, 11 insertions(+), 9 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 49e90678fe..650dbcbcae 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -114,17 +114,19 @@ public class NetworkStressTest {
 
     /** Print a progress bar based on the given character. */
     private void printProgress(char c, int n) {
+        if (n < 1)
+            return;  // nothing to represent
+
         long now = System.currentTimeMillis();
-        if (now - lastProgressUpdateMillis < 500)
-            return;  // throttle message printing below half a second
+        if ((n != 1) && ((now - lastProgressUpdateMillis) < 500))
+            return;  // throttle message printing below half a second (but last one is always printed)
         lastProgressUpdateMillis = now;
-        if (n > 0) {
-            String hdr = String.format("\r%s> ", this.getClass().getSimpleName());
-            String msg = (hdr.length() - 1/*carriage return*/ + n + 1/*final space*/ > terminalColumns)
-                    ? String.format("%c*%d", c, n)
-                    : nChars(c, n);
-            System.out.print(hdr + msg + ' ');
-        }
+
+        String hdr = String.format("\r%s> ", this.getClass().getSimpleName());
+        String msg = (hdr.length() - 1/*carriage return*/ + n + 1/*final space*/ > terminalColumns)
+                ? String.format("%c*%d", c, n)
+                : nChars(c, n);
+        System.out.print(hdr + msg + ' ');
         if (n == 1)
             System.out.print('\n');
         System.out.flush();

From 16e2e03a98bb48578b677d65a71d327fd587f56e Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 10:28:43 +0200
Subject: [PATCH 081/101] Only use numbers in test progress reporting, simpler
 and cleaner

---
 .../p2p/network/NetworkStressTest.java         | 18 +++---------------
 1 file changed, 3 insertions(+), 15 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 650dbcbcae..1505e8d133 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -70,8 +70,6 @@ public class NetworkStressTest {
 
     // Instance fields
 
-    /** Number of columns in terminal (for progress reporting). */
-    private int terminalColumns = 80;
     /** The last time a progress bar update was printed (to throttle message printing). */
     private long lastProgressUpdateMillis = 0;
     /** A directory to (temporarily) hold seed and normal nodes' configuration and state files. */
@@ -118,17 +116,11 @@ public class NetworkStressTest {
             return;  // nothing to represent
 
         long now = System.currentTimeMillis();
-        if ((n != 1) && ((now - lastProgressUpdateMillis) < 500))
-            return;  // throttle message printing below half a second (but last one is always printed)
+        if ((now - lastProgressUpdateMillis) < 500)
+            return;  // throttle message printing below half a second
         lastProgressUpdateMillis = now;
 
-        String hdr = String.format("\r%s> ", this.getClass().getSimpleName());
-        String msg = (hdr.length() - 1/*carriage return*/ + n + 1/*final space*/ > terminalColumns)
-                ? String.format("%c*%d", c, n)
-                : nChars(c, n);
-        System.out.print(hdr + msg + ' ');
-        if (n == 1)
-            System.out.print('\n');
+        System.out.print(String.format("\r%s> %c*%-6d ", this.getClass().getSimpleName(), c, n));
         System.out.flush();
     }
 
@@ -160,10 +152,6 @@ public class NetworkStressTest {
                     String.format("Direct messages sent per peer must not be negative: %d", directCount)
             );
 
-        final String terminalColumnsEnv = System.getenv("COLUMNS");
-        if (terminalColumnsEnv != null && !terminalColumnsEnv.equals(""))
-            terminalColumns = Integer.parseInt(terminalColumnsEnv);
-
         /** A property where threads can indicate setup failure of local services (Tor node, hidden service). */
         final BooleanProperty localServicesFailed = new SimpleBooleanProperty(false);
         /** A barrier to wait for concurrent setup of local services (Tor node, hidden service). */

From 123928a7a2d70be6f9ee4c79b07afce12022c976 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 10:30:12 +0200
Subject: [PATCH 082/101] Remove unused function for test progress reporting

---
 .../io/bitsquare/p2p/network/NetworkStressTest.java  | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 1505e8d133..1ccc478c58 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -102,15 +102,7 @@ public class NetworkStressTest {
         System.exit(result.wasSuccessful() ? 0 : 1);
     }
 
-    /** Return a string of a character repeated a number of times. */
-    private static String nChars(char c, int n) {
-        String ret = "";
-        for (int i = 0; i < n; i++)
-            ret += c;
-        return ret;
-    }
-
-    /** Print a progress bar based on the given character. */
+    /** Print a progress indicator based on the given character. */
     private void printProgress(char c, int n) {
         if (n < 1)
             return;  // nothing to represent
@@ -124,7 +116,7 @@ public class NetworkStressTest {
         System.out.flush();
     }
 
-    /** Decrease latch count and print pending as a progress bar based on the given character. */
+    /** Decrease latch count and print a progress indicator based on the given character. */
     private void countDownAndPrint(CountDownLatch latch, char c) {
         latch.countDown();
         printProgress(c, (int)latch.getCount());

From c27aabda2e166ddbc5c308aaed5c3870d042d8c1 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 10:33:59 +0200
Subject: [PATCH 083/101] Some comments on test progress indicators

---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java    | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 1ccc478c58..9cdcabdd99 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -105,13 +105,15 @@ public class NetworkStressTest {
     /** Print a progress indicator based on the given character. */
     private void printProgress(char c, int n) {
         if (n < 1)
-            return;  // nothing to represent
+            return;  // completed tasks are not shown
 
+        // Do not print the indicator if the last one was shown less than half second ago.
         long now = System.currentTimeMillis();
         if ((now - lastProgressUpdateMillis) < 500)
-            return;  // throttle message printing below half a second
+            return;
         lastProgressUpdateMillis = now;
 
+        // Keep a fixed length so that indicators do not overwrite partially.
         System.out.print(String.format("\r%s> %c*%-6d ", this.getClass().getSimpleName(), c, n));
         System.out.flush();
     }

From 3ab270f5c5f0f9f68bf3d7fe1b23d5947fa7031b Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 12:08:30 +0200
Subject: [PATCH 084/101] Skeleton of mailbox send stress test

---
 .../p2p/network/NetworkStressTest.java        | 118 ++++++++++--------
 1 file changed, 69 insertions(+), 49 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 9cdcabdd99..cd98a0fc6f 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -360,58 +360,41 @@ public class NetworkStressTest {
         // (so it can get its mailbox messages),
         // and the new first online node sends messages.
         // This is repeated until all nodes have been online and offline.
-        final CountDownLatch shutDownLatch = new CountDownLatch(1);
-        P2PService peer = peerNodes.get(0);
-        peer.shutDown(shutDownLatch::countDown);
-        shutDownLatch.await();
-
-        peer = createPeerNode(0, peerPorts.get(0));
-        peerNodes.set(0, peer);
-        final CountDownLatch bootLatch = new CountDownLatch(1);
-        peer.start(new P2PServiceListener() {
-            @Override
-            public void onRequestingDataCompleted() {
-
-            }
-
-            @Override
-            public void onNoSeedNodeAvailable() {
-
-            }
-
-            @Override
-            public void onNoPeersAvailable() {
-
-            }
-
-            @Override
-            public void onBootstrapComplete() {
-                bootLatch.countDown();
-            }
-
-            @Override
-            public void onTorNodeReady() {
-
-            }
-
-            @Override
-            public void onHiddenServicePublished() {
-
-            }
-
-            @Override
-            public void onSetupFailed(Throwable throwable) {
-
-            }
-        });
-        bootLatch.await();
 
+        // Put the second half of peers offline.
         final int nPeers = peerNodes.size();
-        for (int firstOnline = 0, firstOffline = (int)Math.ceil(nPeers/2.0);
-                firstOnline < nPeers;
-                firstOnline++, firstOffline = ++firstOffline%nPeers) {
-            System.out.println("firstOnline "+firstOnline+" firstOffline "+firstOffline);
+        final CountDownLatch halfShutDown = new CountDownLatch(nPeers / 2);
+        int firstPeerDown = (int)Math.ceil(nPeers / 2.0);
+        for (P2PService peer : peerNodes.subList(firstPeerDown, nPeers)) {
+            peer.shutDown(halfShutDown::countDown);
         }
+        assertLatch("timed out while stopping a half of the peers",
+                halfShutDown, 10, TimeUnit.SECONDS);
+        print("stopped a half of the peers for mailbox test");
+
+        // Cycle through peers sending to others, stopping the peer
+        // and starting one of the stopped peers.
+        for (int firstOnline = 0, firstOffline = firstPeerDown;
+                firstOnline < nPeers;
+                firstOnline++, firstOffline = ++firstOffline % nPeers) {
+            // TODO: Make first online node send messages to other nodes.
+
+            // When done, put first online peer offline.
+            final CountDownLatch stopLatch = new CountDownLatch(1);
+            peerNodes.get(firstOnline).shutDown(stopLatch::countDown);
+            stopLatch.await(10, TimeUnit.SECONDS);
+            print("put peer %d offline", firstOnline);
+
+            // When done, put first offline peer online.
+            final CountDownLatch startLatch = new CountDownLatch(1);
+            final P2PService startedPeer = createPeerNode(firstOffline, peerPorts.get(firstOffline));
+            // TODO: Setup message listeners.
+            peerNodes.set(firstOffline, startedPeer);
+            startedPeer.start(new MailboxStartListener(startLatch));
+            startLatch.await(10, TimeUnit.SECONDS);
+            print("put peer %d online", firstOffline);
+        }
+        // TODO: Wait for nodes to receive messages.
     }
 
     private void print(String message, Object... args) {
@@ -575,6 +558,43 @@ public class NetworkStressTest {
             countDownAndPrint(bootstrapLatch, 'b');
         }
     }
+
+    private class MailboxStartListener implements P2PServiceListener {
+        private final CountDownLatch startLatch;
+
+        MailboxStartListener(CountDownLatch startLatch) {
+            this.startLatch = startLatch;
+        }
+
+        @Override
+        public void onRequestingDataCompleted() {
+        }
+
+        @Override
+        public void onNoSeedNodeAvailable() {
+        }
+
+        @Override
+        public void onNoPeersAvailable() {
+        }
+
+        @Override
+        public void onBootstrapComplete() {
+            startLatch.countDown();
+        }
+
+        @Override
+        public void onTorNodeReady() {
+        }
+
+        @Override
+        public void onHiddenServicePublished() {
+        }
+
+        @Override
+        public void onSetupFailed(Throwable throwable) {
+        }
+    }
 }
 
 // Message classes

From d0a54a476c1c12ce449feab212c6c383f7aba051 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 12:27:14 +0200
Subject: [PATCH 085/101] Get number of mailbox messages from environment,
 create properly sized latch

---
 .../p2p/network/NetworkStressTest.java        | 22 ++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index cd98a0fc6f..3ab27d6ad4 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -52,6 +52,8 @@ public class NetworkStressTest {
     private static final String TEST_DIR_ENVVAR = "STRESS_TEST_DIR";
     /** Environment variable to specify the number of direct messages sent per peer. */
     private static final String DIRECT_COUNT_ENVVAR = "STRESS_TEST_NDIRECT";
+    /** Environment variable to specify the number of mailbox messages sent per peer. */
+    private static final String MAILBOX_COUNT_ENVVAR = "STRESS_TEST_NMAILBOX";
 
     /** Numeric identifier of the regtest Bitcoin network. */
     private static final int REGTEST_NETWORK_ID = 2;
@@ -62,6 +64,8 @@ public class NetworkStressTest {
     private static final int NPEERS_MIN = 3;
     /** Default number of direct messages to be sent by each peer. */
     private static final int DIRECT_COUNT_DEFAULT = 100;
+    /** Default number of mailbox messages to be sent by each peer. */
+    private static final int MAILBOX_COUNT_DEFAULT = 100;
 
     /** Minimum delay between direct messages in milliseconds, 25% larger than throttle limit. */
     private static long MIN_DIRECT_DELAY_MILLIS = Math.round(1.25 * (1.0 / Connection.MSG_THROTTLE_PER_SEC) * 1000);
@@ -89,6 +93,8 @@ public class NetworkStressTest {
 
     /** Number of direct messages to be sent by each peer. */
     private int directCount = DIRECT_COUNT_DEFAULT;
+    /** Number of mailbox messages to be sent by each peer. */
+    private int mailboxCount = MAILBOX_COUNT_DEFAULT;
 
     // Inspired by <https://stackoverflow.com/a/9288513> by Marc Peters.
     public static void main(String[] args) {
@@ -146,6 +152,14 @@ public class NetworkStressTest {
                     String.format("Direct messages sent per peer must not be negative: %d", directCount)
             );
 
+        final String nMailboxEnv = System.getenv(MAILBOX_COUNT_ENVVAR);
+        if (nMailboxEnv != null && !nMailboxEnv.equals(""))
+            mailboxCount = Integer.parseInt(nMailboxEnv);
+        if (mailboxCount < 0)
+            throw new IllegalArgumentException(
+                    String.format("Mailbox messages sent per peer must not be negative: %d", mailboxCount)
+            );
+
         /** A property where threads can indicate setup failure of local services (Tor node, hidden service). */
         final BooleanProperty localServicesFailed = new SimpleBooleanProperty(false);
         /** A barrier to wait for concurrent setup of local services (Tor node, hidden service). */
@@ -374,6 +388,10 @@ public class NetworkStressTest {
 
         // Cycle through peers sending to others, stopping the peer
         // and starting one of the stopped peers.
+        //
+        // No sent latch here since the order of events is different
+        // depending on whether the message goes direct or via mailbox.
+        final CountDownLatch receivedMailboxLatch = new CountDownLatch(mailboxCount * nPeers);
         for (int firstOnline = 0, firstOffline = firstPeerDown;
                 firstOnline < nPeers;
                 firstOnline++, firstOffline = ++firstOffline % nPeers) {
@@ -394,7 +412,9 @@ public class NetworkStressTest {
             startLatch.await(10, TimeUnit.SECONDS);
             print("put peer %d online", firstOffline);
         }
-        // TODO: Wait for nodes to receive messages.
+        // TODO: Wait for nodes to receive messages, use meaningful timeout.
+        assertLatch("timed out while receiving mailbox messages",
+                receivedMailboxLatch, 120, TimeUnit.SECONDS);
     }
 
     private void print(String message, Object... args) {

From 554652c6fef2369efc987eba78f55939253cc772 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 12:39:30 +0200
Subject: [PATCH 086/101] Factor out environment integer parsing in network
 stress test

---
 .../p2p/network/NetworkStressTest.java        | 40 ++++++++-----------
 1 file changed, 16 insertions(+), 24 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 3ab27d6ad4..aad99920f8 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -130,35 +130,27 @@ public class NetworkStressTest {
         printProgress(c, (int)latch.getCount());
     }
 
+    /** Parse an integer value from the given environment variable, with default and minimum values. */
+    private int parseEnvInt(String envVar, int defValue, int minValue) {
+        int value = defValue;
+        final String envValue = System.getenv(envVar);
+        if (envValue != null && !envValue.equals(""))
+            value = Integer.parseInt(envValue);
+        if (value < minValue)
+            throw new IllegalArgumentException(
+                    String.format("%s must be at least %d: %d", envVar, minValue, value)
+            );
+        return value;
+    }
+
     @Before
     public void setUp() throws Exception {
         // Parse test parameter environment variables.
 
         /** Number of peer nodes to create. */
-        int nPeers = NPEERS_DEFAULT;
-        final String nPeersEnv = System.getenv(NPEERS_ENVVAR);
-        if (nPeersEnv != null && !nPeersEnv.equals(""))
-            nPeers = Integer.parseInt(nPeersEnv);
-        if (nPeers < NPEERS_MIN)
-            throw new IllegalArgumentException(
-                    String.format("Test needs at least %d peer nodes to work: %d", NPEERS_MIN, nPeers)
-            );
-
-        final String nDirectEnv = System.getenv(DIRECT_COUNT_ENVVAR);
-        if (nDirectEnv != null && !nDirectEnv.equals(""))
-            directCount = Integer.parseInt(nDirectEnv);
-        if (directCount < 0)
-            throw new IllegalArgumentException(
-                    String.format("Direct messages sent per peer must not be negative: %d", directCount)
-            );
-
-        final String nMailboxEnv = System.getenv(MAILBOX_COUNT_ENVVAR);
-        if (nMailboxEnv != null && !nMailboxEnv.equals(""))
-            mailboxCount = Integer.parseInt(nMailboxEnv);
-        if (mailboxCount < 0)
-            throw new IllegalArgumentException(
-                    String.format("Mailbox messages sent per peer must not be negative: %d", mailboxCount)
-            );
+        final int nPeers = parseEnvInt(NPEERS_ENVVAR, NPEERS_DEFAULT, NPEERS_MIN);
+        directCount = parseEnvInt(DIRECT_COUNT_ENVVAR, DIRECT_COUNT_DEFAULT, 0);
+        mailboxCount = parseEnvInt(MAILBOX_COUNT_ENVVAR, MAILBOX_COUNT_DEFAULT, 0);
 
         /** A property where threads can indicate setup failure of local services (Tor node, hidden service). */
         final BooleanProperty localServicesFailed = new SimpleBooleanProperty(false);

From 1f6ea10bd619875b3a57faa88847cacd01c7e29e Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Thu, 12 May 2016 12:46:00 +0200
Subject: [PATCH 087/101] Check for timeout when starting and stopping nodes in
 mailbox stress test

---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java  | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index aad99920f8..b53b95bc33 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -392,7 +392,8 @@ public class NetworkStressTest {
             // When done, put first online peer offline.
             final CountDownLatch stopLatch = new CountDownLatch(1);
             peerNodes.get(firstOnline).shutDown(stopLatch::countDown);
-            stopLatch.await(10, TimeUnit.SECONDS);
+            assertLatch("timed out while stopping peer " + firstOnline,
+                    stopLatch, 10, TimeUnit.SECONDS);
             print("put peer %d offline", firstOnline);
 
             // When done, put first offline peer online.
@@ -401,10 +402,11 @@ public class NetworkStressTest {
             // TODO: Setup message listeners.
             peerNodes.set(firstOffline, startedPeer);
             startedPeer.start(new MailboxStartListener(startLatch));
-            startLatch.await(10, TimeUnit.SECONDS);
+            assertLatch("timed out while starting peer " + firstOffline,
+                    startLatch, 10, TimeUnit.SECONDS);
             print("put peer %d online", firstOffline);
         }
-        // TODO: Wait for nodes to receive messages, use meaningful timeout.
+        // TODO: Use meaningful timeout.
         assertLatch("timed out while receiving mailbox messages",
                 receivedMailboxLatch, 120, TimeUnit.SECONDS);
     }

From bad681d8e87f04a2173aa9666cbc931087a44aa3 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 17 May 2016 10:33:02 +0200
Subject: [PATCH 088/101] Remind not to send direct message to oneself in
 network stress test

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index b53b95bc33..7d2e10c45d 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -304,6 +304,7 @@ public class NetworkStressTest {
             long nextSendMillis = System.currentTimeMillis();
             for (int i = 0; i < directCount; i++) {
                 // Select a random peer and send a direct message to it...
+                // TODO: Do not send the message to oneself.
                 final int dstPeerIdx = (int) (Math.random() * nPeers);
                 final P2PService dstPeer = peerNodes.get(dstPeerIdx);
                 final NodeAddress dstPeerAddress = dstPeer.getAddress();

From 14374fed6d2e6c295b4fe5b302e637a152972e56 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 17 May 2016 11:27:46 +0200
Subject: [PATCH 089/101] Do not send direct message to oneself in network
 stress test

---
 .../p2p/network/NetworkStressTest.java         | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 7d2e10c45d..bd42ef3556 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -292,22 +292,28 @@ public class NetworkStressTest {
         final CountDownLatch receivedDirectLatch = new CountDownLatch(directCount * nPeers);
         final long sendStartMillis = System.currentTimeMillis();
         for (final P2PService srcPeer : peerNodes) {
+            final NodeAddress srcPeerAddress = srcPeer.getAddress();
+
             // Make the peer ready for receiving direct messages.
             srcPeer.addDecryptedDirectMessageListener((decryptedMsgWithPubKey, peerNodeAddress) -> {
                 if (!(decryptedMsgWithPubKey.message instanceof StressTestDirectMessage))
                     return;
                 StressTestDirectMessage directMessage = (StressTestDirectMessage) (decryptedMsgWithPubKey.message);
-                if ((directMessage.getData().equals("test/" + srcPeer.getAddress())))
+                if ((directMessage.getData().equals("test/" + srcPeerAddress)))
                     receivedDirectLatch.countDown();
             });
 
             long nextSendMillis = System.currentTimeMillis();
             for (int i = 0; i < directCount; i++) {
-                // Select a random peer and send a direct message to it...
-                // TODO: Do not send the message to oneself.
-                final int dstPeerIdx = (int) (Math.random() * nPeers);
-                final P2PService dstPeer = peerNodes.get(dstPeerIdx);
-                final NodeAddress dstPeerAddress = dstPeer.getAddress();
+                // Select a random peer (different than source one) and send a direct message to it...
+                int peerIdx;
+                NodeAddress peerAddr;
+                do {
+                    peerIdx = (int) (Math.random() * nPeers);
+                    peerAddr = peerNodes.get(peerIdx).getAddress();
+                } while (srcPeerAddress.equals(peerAddr));
+                final int dstPeerIdx = peerIdx;
+                final NodeAddress dstPeerAddress = peerAddr;
                 // ...after a random delay not shorter than throttle limits.
                 nextSendMillis += Math.round(Math.random() * (MAX_DIRECT_DELAY_MILLIS - MIN_DIRECT_DELAY_MILLIS));
                 final long sendAfterMillis = nextSendMillis - System.currentTimeMillis();

From 976d9cb333a376cf79064c0b7a92a84472d41266 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Tue, 17 May 2016 14:06:07 +0200
Subject: [PATCH 090/101] Implement sending and receiving mailbox messages in
 network stress test

Not working thoug, lots of "Message not broadcasted because we have stopped the handler already.".
---
 .../p2p/network/NetworkStressTest.java        | 122 ++++++++++++++++--
 1 file changed, 110 insertions(+), 12 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index bd42ef3556..e73a9a7b37 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -7,13 +7,13 @@ import io.bitsquare.common.crypto.KeyRing;
 import io.bitsquare.common.crypto.KeyStorage;
 import io.bitsquare.common.crypto.PubKeyRing;
 import io.bitsquare.common.util.Tuple3;
+import io.bitsquare.crypto.DecryptedMsgWithPubKey;
 import io.bitsquare.crypto.EncryptionService;
 import io.bitsquare.p2p.NodeAddress;
 import io.bitsquare.p2p.P2PService;
 import io.bitsquare.p2p.P2PServiceListener;
 import io.bitsquare.p2p.Utils;
-import io.bitsquare.p2p.messaging.DirectMessage;
-import io.bitsquare.p2p.messaging.SendDirectMessageListener;
+import io.bitsquare.p2p.messaging.*;
 import io.bitsquare.p2p.seed.SeedNode;
 import io.bitsquare.p2p.seed.SeedNodesRepository;
 import javafx.beans.property.BooleanProperty;
@@ -374,10 +374,18 @@ public class NetworkStressTest {
         // and the new first online node sends messages.
         // This is repeated until all nodes have been online and offline.
 
-        // Put the second half of peers offline.
         final int nPeers = peerNodes.size();
-        final CountDownLatch halfShutDown = new CountDownLatch(nPeers / 2);
+        // No sent latch here since the order of events is different
+        // depending on whether the message goes direct or via mailbox.
+        final CountDownLatch receivedMailboxLatch = new CountDownLatch(mailboxCount * nPeers);
+
+        // Configure the first half of peers to receive messages...
         int firstPeerDown = (int)Math.ceil(nPeers / 2.0);
+        for (P2PService peer : peerNodes.subList(0, firstPeerDown)) {
+            addMailboxListeners(peer, receivedMailboxLatch);
+        }
+        // ...and put the second half offline.
+        final CountDownLatch halfShutDown = new CountDownLatch(nPeers / 2);
         for (P2PService peer : peerNodes.subList(firstPeerDown, nPeers)) {
             peer.shutDown(halfShutDown::countDown);
         }
@@ -387,26 +395,53 @@ public class NetworkStressTest {
 
         // Cycle through peers sending to others, stopping the peer
         // and starting one of the stopped peers.
-        //
-        // No sent latch here since the order of events is different
-        // depending on whether the message goes direct or via mailbox.
-        final CountDownLatch receivedMailboxLatch = new CountDownLatch(mailboxCount * nPeers);
+        BooleanProperty sentMailboxFailed = new SimpleBooleanProperty(false);
         for (int firstOnline = 0, firstOffline = firstPeerDown;
                 firstOnline < nPeers;
                 firstOnline++, firstOffline = ++firstOffline % nPeers) {
-            // TODO: Make first online node send messages to other nodes.
+            // The first online peer sends messages to random other peers.
+            final P2PService onlinePeer = peerNodes.get(firstOnline);
+            final NodeAddress onlinePeerAddress = onlinePeer.getAddress();
+            for (int i = 0; i < mailboxCount; i++) {
+                // Select a random peer (different than source one)...
+                int peerIdx;
+                NodeAddress peerAddr;
+                do {
+                    peerIdx = (int) (Math.random() * nPeers);
+                    peerAddr = peerNodes.get(peerIdx).getAddress();
+                } while (onlinePeerAddress.equals(peerAddr));
+                final int dstPeerIdx = peerIdx;
+                final NodeAddress dstPeerAddress = peerAddr;
+                // ...and send a message to it.
+                onlinePeer.sendEncryptedMailboxMessage(dstPeerAddress, peerPKRings.get(dstPeerIdx),
+                        new StressTestMailboxMessage(onlinePeerAddress, "test/" + dstPeerAddress),
+                        new SendMailboxMessageListener() {  // checked in receiver
+                            @Override
+                            public void onArrived() {
+                            }
+
+                            @Override
+                            public void onStoredInMailbox() {
+                            }
+
+                            @Override
+                            public void onFault(String errorMessage) {
+                                sentMailboxFailed.set(true);
+                            }
+                        });
+            }
 
             // When done, put first online peer offline.
             final CountDownLatch stopLatch = new CountDownLatch(1);
-            peerNodes.get(firstOnline).shutDown(stopLatch::countDown);
+            onlinePeer.shutDown(stopLatch::countDown);
             assertLatch("timed out while stopping peer " + firstOnline,
                     stopLatch, 10, TimeUnit.SECONDS);
             print("put peer %d offline", firstOnline);
 
-            // When done, put first offline peer online.
+            // When done, put first offline peer online and setup message listeners.
             final CountDownLatch startLatch = new CountDownLatch(1);
             final P2PService startedPeer = createPeerNode(firstOffline, peerPorts.get(firstOffline));
-            // TODO: Setup message listeners.
+            addMailboxListeners(startedPeer, receivedMailboxLatch);
             peerNodes.set(firstOffline, startedPeer);
             startedPeer.start(new MailboxStartListener(startLatch));
             assertLatch("timed out while starting peer " + firstOffline,
@@ -418,6 +453,36 @@ public class NetworkStressTest {
                 receivedMailboxLatch, 120, TimeUnit.SECONDS);
     }
 
+    /** Configure the peer to decrease the latch on receipt of mailbox message (direct or via mailbox). */
+    private void addMailboxListeners(P2PService peer, CountDownLatch receivedMailboxLatch) {
+        class MailboxMessageListener implements DecryptedDirectMessageListener, DecryptedMailboxListener {
+            private void handle(DecryptedMsgWithPubKey decryptedMsgWithPubKey) {
+                if (!(decryptedMsgWithPubKey.message instanceof StressTestMailboxMessage))
+                    return;
+                StressTestMailboxMessage msg = (StressTestMailboxMessage) (decryptedMsgWithPubKey.message);
+                if ((msg.getData().equals("test/" + peer.getAddress())))
+                    receivedMailboxLatch.countDown();
+            }
+
+            @Override
+            public void onDirectMessage(
+                    DecryptedMsgWithPubKey decryptedMsgWithPubKey, NodeAddress srcNodeAddress) {
+                handle(decryptedMsgWithPubKey);
+            }
+
+            @Override
+            public void onMailboxMessageAdded(
+                    DecryptedMsgWithPubKey decryptedMsgWithPubKey, NodeAddress srcNodeAddress) {
+                handle(decryptedMsgWithPubKey);
+            }
+        }
+
+        final MailboxMessageListener listener = new MailboxMessageListener();
+        peer.addDecryptedDirectMessageListener(listener);
+        peer.addDecryptedMailboxListener(listener);
+    }
+
+
     private void print(String message, Object... args) {
         System.out.println(this.getClass().getSimpleName() + ": "
             + String.format(message, args));
@@ -639,3 +704,36 @@ final class StressTestDirectMessage implements DirectMessage {
         return data;
     }
 }
+
+final class StressTestMailboxMessage implements MailboxMessage {
+    private static final long serialVersionUID = Version.P2P_NETWORK_VERSION;
+    private final int messageVersion = Version.getP2PMessageVersion();
+
+    private final String uid = UUID.randomUUID().toString();
+    private NodeAddress senderNodeAddress;
+    private String data;
+
+    StressTestMailboxMessage(NodeAddress sender, String data) {
+        this.senderNodeAddress = sender;
+        this.data = data;
+    }
+
+    @Override
+    public int getMessageVersion() {
+        return messageVersion;
+    }
+
+    @Override
+    public NodeAddress getSenderNodeAddress() {
+        return senderNodeAddress;
+    }
+
+    @Override
+    public String getUID() {
+        return uid;
+    }
+
+    String getData() {
+        return data;
+    }
+}

From 1a5960124c9e20b36cb4bcd99d6119c1df3f1714 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 18 May 2016 10:16:36 +0200
Subject: [PATCH 091/101] Wait a little more for node bootstrap in network
 stress test

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index e73a9a7b37..e2583ce162 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -218,12 +218,12 @@ public class NetworkStressTest {
 
         // Wait for peers to get their preliminary data.
         assertLatch("timed out while waiting for preliminary data",
-                prelimDataLatch, 2 * nPeers, TimeUnit.SECONDS);
+                prelimDataLatch, 5 * nPeers, TimeUnit.SECONDS);
         print("preliminary data received");
 
         // Wait for peers to complete their bootstrapping.
         assertLatch("timed out while waiting for bootstrap",
-                bootstrapLatch, 2 * nPeers, TimeUnit.SECONDS);
+                bootstrapLatch, 5 * nPeers, TimeUnit.SECONDS);
         print("bootstrap complete");
     }
 

From ea35ace8db9482726494f75af9efb4ce6de41c20 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 18 May 2016 10:19:14 +0200
Subject: [PATCH 092/101] Report failed mailbox messages in network stress test

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java    | 1 +
 1 file changed, 1 insertion(+)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index e2583ce162..e4911cb71b 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -451,6 +451,7 @@ public class NetworkStressTest {
         // TODO: Use meaningful timeout.
         assertLatch("timed out while receiving mailbox messages",
                 receivedMailboxLatch, 120, TimeUnit.SECONDS);
+        org.junit.Assert.assertFalse("some peer(s) failed to send a message", sentMailboxFailed.get());
     }
 
     /** Configure the peer to decrease the latch on receipt of mailbox message (direct or via mailbox). */

From 1a496e09639782bef9c78f5145026b5cea8ec9e0 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 18 May 2016 10:39:07 +0200
Subject: [PATCH 093/101] Wait for mailbox send before stopping peer in network
 stress test

Otherwise the peer may be already stopped before sending.
---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java   | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index e4911cb71b..f5a7158f1f 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -402,6 +402,7 @@ public class NetworkStressTest {
             // The first online peer sends messages to random other peers.
             final P2PService onlinePeer = peerNodes.get(firstOnline);
             final NodeAddress onlinePeerAddress = onlinePeer.getAddress();
+            final CountDownLatch sendLatch = new CountDownLatch(mailboxCount);
             for (int i = 0; i < mailboxCount; i++) {
                 // Select a random peer (different than source one)...
                 int peerIdx;
@@ -418,18 +419,24 @@ public class NetworkStressTest {
                         new SendMailboxMessageListener() {  // checked in receiver
                             @Override
                             public void onArrived() {
+                                sendLatch.countDown();
                             }
 
                             @Override
                             public void onStoredInMailbox() {
+                                sendLatch.countDown();
                             }
 
                             @Override
                             public void onFault(String errorMessage) {
                                 sentMailboxFailed.set(true);
+                                sendLatch.countDown();
                             }
                         });
             }
+            // TODO: Use meaningful timeout.
+            assertLatch("timed out while sending from peer " + firstOnline,
+                    sendLatch, 2 * mailboxCount, TimeUnit.SECONDS);
 
             // When done, put first online peer offline.
             final CountDownLatch stopLatch = new CountDownLatch(1);

From cb14a7961be0449dc29b414584b110c2ac31c5bb Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 18 May 2016 10:41:47 +0200
Subject: [PATCH 094/101] Show progress in mailbox network stress test

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index f5a7158f1f..e22b093c2e 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -469,7 +469,7 @@ public class NetworkStressTest {
                     return;
                 StressTestMailboxMessage msg = (StressTestMailboxMessage) (decryptedMsgWithPubKey.message);
                 if ((msg.getData().equals("test/" + peer.getAddress())))
-                    receivedMailboxLatch.countDown();
+                    countDownAndPrint(receivedMailboxLatch, 'm');
             }
 
             @Override

From 8c8fbd1947953e590186fac37f0f9cf83919a4fa Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 18 May 2016 10:47:03 +0200
Subject: [PATCH 095/101] Comment out some debug prints in mailbox network
 stress test

---
 .../java/io/bitsquare/p2p/network/NetworkStressTest.java    | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index e22b093c2e..26a66c3c71 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -391,7 +391,7 @@ public class NetworkStressTest {
         }
         assertLatch("timed out while stopping a half of the peers",
                 halfShutDown, 10, TimeUnit.SECONDS);
-        print("stopped a half of the peers for mailbox test");
+        //print("stopped a half of the peers for mailbox test");
 
         // Cycle through peers sending to others, stopping the peer
         // and starting one of the stopped peers.
@@ -443,7 +443,7 @@ public class NetworkStressTest {
             onlinePeer.shutDown(stopLatch::countDown);
             assertLatch("timed out while stopping peer " + firstOnline,
                     stopLatch, 10, TimeUnit.SECONDS);
-            print("put peer %d offline", firstOnline);
+            //print("put peer %d offline", firstOnline);
 
             // When done, put first offline peer online and setup message listeners.
             final CountDownLatch startLatch = new CountDownLatch(1);
@@ -453,7 +453,7 @@ public class NetworkStressTest {
             startedPeer.start(new MailboxStartListener(startLatch));
             assertLatch("timed out while starting peer " + firstOffline,
                     startLatch, 10, TimeUnit.SECONDS);
-            print("put peer %d online", firstOffline);
+            //print("put peer %d online", firstOffline);
         }
         // TODO: Use meaningful timeout.
         assertLatch("timed out while receiving mailbox messages",

From d1c6d4ad9c307af6be0d89c493117565e351f30c Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 18 May 2016 11:23:21 +0200
Subject: [PATCH 096/101] Rearrange network stress test code for clarity

No actual changes to code.
---
 .../p2p/network/NetworkStressTest.java        | 377 +++++++++---------
 1 file changed, 197 insertions(+), 180 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 26a66c3c71..6c8bfb9ee6 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -96,6 +96,9 @@ public class NetworkStressTest {
     /** Number of mailbox messages to be sent by each peer. */
     private int mailboxCount = MAILBOX_COUNT_DEFAULT;
 
+
+    // # MAIN ENTRY POINT
+
     // Inspired by <https://stackoverflow.com/a/9288513> by Marc Peters.
     public static void main(String[] args) {
         Request request = (args.length == 0)
@@ -108,6 +111,20 @@ public class NetworkStressTest {
         System.exit(result.wasSuccessful() ? 0 : 1);
     }
 
+
+    // # COMMON UTILITIES
+
+    private void print(String message, Object... args) {
+        System.out.println(this.getClass().getSimpleName() + ": "
+                + String.format(message, args));
+    }
+
+    /** Decrease latch count and print a progress indicator based on the given character. */
+    private void countDownAndPrint(CountDownLatch latch, char c) {
+        latch.countDown();
+        printProgress(c, (int)latch.getCount());
+    }
+
     /** Print a progress indicator based on the given character. */
     private void printProgress(char c, int n) {
         if (n < 1)
@@ -124,25 +141,29 @@ public class NetworkStressTest {
         System.out.flush();
     }
 
-    /** Decrease latch count and print a progress indicator based on the given character. */
-    private void countDownAndPrint(CountDownLatch latch, char c) {
-        latch.countDown();
-        printProgress(c, (int)latch.getCount());
+    private static void assertLatch(String message, CountDownLatch latch, long timeout, TimeUnit unit)
+            throws InterruptedException {
+        if (!latch.await(timeout, unit))
+            org.junit.Assert.fail(String.format("%s (%d pending in latch)", message, latch.getCount()));
     }
 
-    /** Parse an integer value from the given environment variable, with default and minimum values. */
-    private int parseEnvInt(String envVar, int defValue, int minValue) {
-        int value = defValue;
-        final String envValue = System.getenv(envVar);
-        if (envValue != null && !envValue.equals(""))
-            value = Integer.parseInt(envValue);
-        if (value < minValue)
-            throw new IllegalArgumentException(
-                    String.format("%s must be at least %d: %d", envVar, minValue, value)
-            );
-        return value;
+    private Tuple3<Long, Long, Long> minMaxAvg(List<Long> l) {
+        long min = Long.MAX_VALUE;
+        long max = Long.MIN_VALUE;
+        long sum = 0;
+        for (long e : l) {
+            if (e < min)
+                min = e;
+            if (e > max)
+                max = e;
+            sum += e;
+        }
+        return new Tuple3<>(min, max, sum / l.size());
     }
 
+
+    // # TEST SETUP
+
     @Before
     public void setUp() throws Exception {
         // Parse test parameter environment variables.
@@ -161,7 +182,6 @@ public class NetworkStressTest {
         /* A barrier to wait for concurrent bootstrap of peers. */
         final CountDownLatch bootstrapLatch = new CountDownLatch(nPeers);
 
-
         // Set a security provider to allow key generation.
         Security.addProvider(new BouncyCastleProvider());
 
@@ -227,6 +247,36 @@ public class NetworkStressTest {
         print("bootstrap complete");
     }
 
+    /** Parse an integer value from the given environment variable, with default and minimum values. */
+    private int parseEnvInt(String envVar, int defValue, int minValue) {
+        int value = defValue;
+        final String envValue = System.getenv(envVar);
+        if (envValue != null && !envValue.equals(""))
+            value = Integer.parseInt(envValue);
+        if (value < minValue)
+            throw new IllegalArgumentException(
+                    String.format("%s must be at least %d: %d", envVar, minValue, value)
+            );
+        return value;
+    }
+
+    private Path createTestDataDirectory() throws IOException {
+        Path stressTestDirPath;
+
+        final String stressTestDir = System.getenv(TEST_DIR_ENVVAR);
+        if ((stressTestDir != null) && !stressTestDir.equals("")) {
+            // Test directory specified, use and create if missing.
+            stressTestDirPath = Paths.get(stressTestDir);
+            if (!Files.isDirectory(stressTestDirPath)) {
+                //noinspection ResultOfMethodCallIgnored
+                stressTestDirPath.toFile().mkdirs();
+            }
+        } else {
+            stressTestDirPath = Files.createTempDirectory("bsq" + this.getClass().getSimpleName());
+        }
+        return stressTestDirPath;
+    }
+
     @NotNull
     private static NodeAddress newSeedNodeAddress() {
         // The address is only considered by ``SeedNodesRepository`` if
@@ -257,6 +307,99 @@ public class NetworkStressTest {
                 REGTEST_NETWORK_ID, peerStorageDir, new Clock(), peerEncryptionService, peerKeyRing);
     }
 
+    // ## TEST SETUP: P2P service listener classes
+
+    private class TestSetupListener implements SetupListener {
+        private final CountDownLatch localServicesLatch;
+        private final BooleanProperty localServicesFailed;
+
+        TestSetupListener(CountDownLatch localServicesLatch, BooleanProperty localServicesFailed) {
+            this.localServicesLatch = localServicesLatch;
+            this.localServicesFailed = localServicesFailed;
+        }
+
+        @Override
+        public void onTorNodeReady() {
+            // do nothing
+        }
+
+        @Override
+        public void onHiddenServicePublished() {
+            // successful result
+            localServicesLatch.countDown();
+        }
+
+        @Override
+        public void onSetupFailed(Throwable throwable) {
+            // failed result
+            localServicesFailed.set(true);
+            localServicesLatch.countDown();
+        }
+    }
+
+    private class SeedServiceListener extends TestSetupListener implements P2PServiceListener {
+        SeedServiceListener(CountDownLatch localServicesLatch, BooleanProperty localServicesFailed) {
+            super(localServicesLatch, localServicesFailed);
+        }
+
+        @Override
+        public void onRequestingDataCompleted() {
+            // preliminary data not used in single seed node
+        }
+
+        @Override
+        public void onNoSeedNodeAvailable() {
+            // expected in single seed node
+        }
+
+        @Override
+        public void onNoPeersAvailable() {
+            // expected in single seed node
+        }
+
+        @Override
+        public void onBootstrapComplete() {
+            // not used in single seed node
+        }
+    }
+
+    private class PeerServiceListener extends TestSetupListener implements P2PServiceListener {
+        private final CountDownLatch prelimDataLatch;
+        private final CountDownLatch bootstrapLatch;
+
+        PeerServiceListener(CountDownLatch localServicesLatch, BooleanProperty localServicesFailed,
+                            CountDownLatch prelimDataLatch, CountDownLatch bootstrapLatch) {
+            super(localServicesLatch, localServicesFailed);
+            this.prelimDataLatch = prelimDataLatch;
+            this.bootstrapLatch = bootstrapLatch;
+        }
+
+        @Override
+        public void onRequestingDataCompleted() {
+            // preliminary data received
+            countDownAndPrint(prelimDataLatch, 'p');
+        }
+
+        @Override
+        public void onNoSeedNodeAvailable() {
+            // do nothing
+        }
+
+        @Override
+        public void onNoPeersAvailable() {
+            // do nothing
+        }
+
+        @Override
+        public void onBootstrapComplete() {
+            // peer bootstrapped
+            countDownAndPrint(bootstrapLatch, 'b');
+        }
+    }
+
+
+    // # TEST CLEANUP
+
     @After
     public void tearDown() throws InterruptedException, IOException {
         /** A barrier to wait for concurrent shutdown of services. */
@@ -282,6 +425,39 @@ public class NetworkStressTest {
         }
     }
 
+    /**
+     * Delete the test data directory recursively, unless <code>STRESS_TEST_DIR</code> is defined,
+     * in which case peer node keys are kept.
+     *
+     * @throws IOException
+     */
+    private void deleteTestDataDirectory() throws IOException {
+        // Based on <https://stackoverflow.com/a/27917071/6239236> by Tomasz Dzięcielewski.
+        final String stressTestDir = System.getenv(TEST_DIR_ENVVAR);
+        final boolean keep = (stressTestDir != null) && !stressTestDir.equals("");
+        Files.walkFileTree(testDataDir, new SimpleFileVisitor<Path>() {
+            @Override
+            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
+                final String fileName = file.getFileName().toString();
+                if (!(keep && (fileName.matches("enc\\.key|sig\\.key|private_key"))))  // peer and tor keys
+                    Files.delete(file);
+                return FileVisitResult.CONTINUE;
+            }
+
+            @Override
+            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
+                // ``dir`` is always a directory, I/O errors may still trigger ``NullPointerException``.
+                //noinspection ConstantConditions
+                if (!(keep && dir.toFile().listFiles().length > 0))
+                    Files.delete(dir);
+                return FileVisitResult.CONTINUE;
+            }
+        });
+    }
+
+
+    // # DIRECT SENDING AND RECEIVING
+
     /** Test each peer sending a direct message to another random peer. */
     @Test
     public void test_direct() throws InterruptedException {
@@ -363,6 +539,9 @@ public class NetworkStressTest {
         org.junit.Assert.assertFalse("some peer(s) failed to send a direct message", sentDirectFailed.get());
     }
 
+
+    // # DIRECT + MAILBOX SENDING AND RECEIVING
+
     /** Test sending and receiving mailbox messages. */
     @Test
     public void test_mailbox() throws InterruptedException {
@@ -490,169 +669,6 @@ public class NetworkStressTest {
         peer.addDecryptedMailboxListener(listener);
     }
 
-
-    private void print(String message, Object... args) {
-        System.out.println(this.getClass().getSimpleName() + ": "
-            + String.format(message, args));
-    }
-
-    private static void assertLatch(String message, CountDownLatch latch, long timeout, TimeUnit unit)
-            throws InterruptedException {
-        if (!latch.await(timeout, unit))
-            org.junit.Assert.fail(String.format("%s (%d pending in latch)", message, latch.getCount()));
-    }
-
-    private Path createTestDataDirectory() throws IOException {
-        Path stressTestDirPath;
-
-        final String stressTestDir = System.getenv(TEST_DIR_ENVVAR);
-        if ((stressTestDir != null) && !stressTestDir.equals("")) {
-            // Test directory specified, use and create if missing.
-            stressTestDirPath = Paths.get(stressTestDir);
-            if (!Files.isDirectory(stressTestDirPath)) {
-                //noinspection ResultOfMethodCallIgnored
-                stressTestDirPath.toFile().mkdirs();
-            }
-        } else {
-            stressTestDirPath = Files.createTempDirectory("bsq" + this.getClass().getSimpleName());
-        }
-        return stressTestDirPath;
-    }
-
-    /**
-     * Delete the test data directory recursively, unless <code>STRESS_TEST_DIR</code> is defined,
-     * in which case peer node keys are kept.
-     *
-     * @throws IOException
-     */
-    private void deleteTestDataDirectory() throws IOException {
-        // Based on <https://stackoverflow.com/a/27917071/6239236> by Tomasz Dzięcielewski.
-        final String stressTestDir = System.getenv(TEST_DIR_ENVVAR);
-        final boolean keep = (stressTestDir != null) && !stressTestDir.equals("");
-        Files.walkFileTree(testDataDir, new SimpleFileVisitor<Path>() {
-            @Override
-            public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
-                final String fileName = file.getFileName().toString();
-                if (!(keep && (fileName.matches("enc\\.key|sig\\.key|private_key"))))  // peer and tor keys
-                    Files.delete(file);
-                return FileVisitResult.CONTINUE;
-            }
-
-            @Override
-            public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
-                // ``dir`` is always a directory, I/O errors may still trigger ``NullPointerException``.
-                //noinspection ConstantConditions
-                if (!(keep && dir.toFile().listFiles().length > 0))
-                    Files.delete(dir);
-                return FileVisitResult.CONTINUE;
-            }
-        });
-    }
-
-    private Tuple3<Long, Long, Long> minMaxAvg(List<Long> l) {
-        long min = Long.MAX_VALUE;
-        long max = Long.MIN_VALUE;
-        long sum = 0;
-        for (long e : l) {
-            if (e < min)
-                min = e;
-            if (e > max)
-                max = e;
-            sum += e;
-        }
-        return new Tuple3<>(min, max, sum / l.size());
-    }
-
-    // P2P service listener classes
-
-    private class TestSetupListener implements SetupListener {
-        private final CountDownLatch localServicesLatch;
-        private final BooleanProperty localServicesFailed;
-
-        TestSetupListener(CountDownLatch localServicesLatch, BooleanProperty localServicesFailed) {
-            this.localServicesLatch = localServicesLatch;
-            this.localServicesFailed = localServicesFailed;
-        }
-
-        @Override
-        public void onTorNodeReady() {
-            // do nothing
-        }
-
-        @Override
-        public void onHiddenServicePublished() {
-            // successful result
-            localServicesLatch.countDown();
-        }
-
-        @Override
-        public void onSetupFailed(Throwable throwable) {
-            // failed result
-            localServicesFailed.set(true);
-            localServicesLatch.countDown();
-        }
-    }
-
-    private class SeedServiceListener extends TestSetupListener implements P2PServiceListener {
-        SeedServiceListener(CountDownLatch localServicesLatch, BooleanProperty localServicesFailed) {
-            super(localServicesLatch, localServicesFailed);
-        }
-
-        @Override
-        public void onRequestingDataCompleted() {
-            // preliminary data not used in single seed node
-        }
-
-        @Override
-        public void onNoSeedNodeAvailable() {
-            // expected in single seed node
-        }
-
-        @Override
-        public void onNoPeersAvailable() {
-            // expected in single seed node
-        }
-
-        @Override
-        public void onBootstrapComplete() {
-            // not used in single seed node
-        }
-    }
-
-    private class PeerServiceListener extends TestSetupListener implements P2PServiceListener {
-        private final CountDownLatch prelimDataLatch;
-        private final CountDownLatch bootstrapLatch;
-
-        PeerServiceListener(CountDownLatch localServicesLatch, BooleanProperty localServicesFailed,
-                            CountDownLatch prelimDataLatch, CountDownLatch bootstrapLatch) {
-            super(localServicesLatch, localServicesFailed);
-            this.prelimDataLatch = prelimDataLatch;
-            this.bootstrapLatch = bootstrapLatch;
-        }
-
-        @Override
-        public void onRequestingDataCompleted() {
-            // preliminary data received
-            countDownAndPrint(prelimDataLatch, 'p');
-        }
-
-        @Override
-        public void onNoSeedNodeAvailable() {
-            // do nothing
-        }
-
-        @Override
-        public void onNoPeersAvailable() {
-            // do nothing
-        }
-
-        @Override
-        public void onBootstrapComplete() {
-            // peer bootstrapped
-            countDownAndPrint(bootstrapLatch, 'b');
-        }
-    }
-
     private class MailboxStartListener implements P2PServiceListener {
         private final CountDownLatch startLatch;
 
@@ -691,7 +707,8 @@ public class NetworkStressTest {
     }
 }
 
-// Message classes
+
+// # MESSAGE CLASSES
 
 final class StressTestDirectMessage implements DirectMessage {
     private static final long serialVersionUID = Version.P2P_NETWORK_VERSION;

From 2a24ce94a304c668eb9a685333b8e0fe49ff8183 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 18 May 2016 12:40:13 +0200
Subject: [PATCH 097/101] Constant for mailbox message delay in network stress
 test

---
 .../io/bitsquare/p2p/network/NetworkStressTest.java    | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 6c8bfb9ee6..de487e4284 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -71,6 +71,8 @@ public class NetworkStressTest {
     private static long MIN_DIRECT_DELAY_MILLIS = Math.round(1.25 * (1.0 / Connection.MSG_THROTTLE_PER_SEC) * 1000);
     /** Maximum delay between direct messages in milliseconds, 10 times larger than minimum. */
     private static long MAX_DIRECT_DELAY_MILLIS = 10 * MIN_DIRECT_DELAY_MILLIS;
+    /** Estimated delay in seconds to send or receive a mailbox message. */
+    private static long MAILBOX_DELAY_SECS = 2;
 
     // Instance fields
 
@@ -613,9 +615,8 @@ public class NetworkStressTest {
                             }
                         });
             }
-            // TODO: Use meaningful timeout.
             assertLatch("timed out while sending from peer " + firstOnline,
-                    sendLatch, 2 * mailboxCount, TimeUnit.SECONDS);
+                    sendLatch, MAILBOX_DELAY_SECS * mailboxCount, TimeUnit.SECONDS);
 
             // When done, put first online peer offline.
             final CountDownLatch stopLatch = new CountDownLatch(1);
@@ -631,12 +632,11 @@ public class NetworkStressTest {
             peerNodes.set(firstOffline, startedPeer);
             startedPeer.start(new MailboxStartListener(startLatch));
             assertLatch("timed out while starting peer " + firstOffline,
-                    startLatch, 10, TimeUnit.SECONDS);
+                    startLatch, 10 + MAILBOX_DELAY_SECS * nPeers, TimeUnit.SECONDS);
             //print("put peer %d online", firstOffline);
         }
-        // TODO: Use meaningful timeout.
         assertLatch("timed out while receiving mailbox messages",
-                receivedMailboxLatch, 120, TimeUnit.SECONDS);
+                receivedMailboxLatch, 2 * (MAILBOX_DELAY_SECS + 1) * nPeers * mailboxCount, TimeUnit.SECONDS);
         org.junit.Assert.assertFalse("some peer(s) failed to send a message", sentMailboxFailed.get());
     }
 

From b4b40714363f84c88fd41c096d5edbcea0a81062 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 18 May 2016 12:55:26 +0200
Subject: [PATCH 098/101] Use meaningful timeouts and constants in network
 stress test

---
 .../p2p/network/NetworkStressTest.java        | 25 +++++++++++++------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index de487e4284..567feb3803 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -67,6 +67,12 @@ public class NetworkStressTest {
     /** Default number of mailbox messages to be sent by each peer. */
     private static final int MAILBOX_COUNT_DEFAULT = 100;
 
+    /** Maximum delay in seconds for a node to receive preliminary data. */
+    private static long MAX_PRELIMINARY_DELAY_SECS = 5;
+    /** Maximum delay in seconds for a node to bootstrap after receiving preliminary data. */
+    private static long MAX_BOOTSTRAP_DELAY_SECS = 5;
+    /** Maximum delay in seconds for a node to shutdown. */
+    private static long MAX_SHUTDOWN_DELAY_SECS = 2;
     /** Minimum delay between direct messages in milliseconds, 25% larger than throttle limit. */
     private static long MIN_DIRECT_DELAY_MILLIS = Math.round(1.25 * (1.0 / Connection.MSG_THROTTLE_PER_SEC) * 1000);
     /** Maximum delay between direct messages in milliseconds, 10 times larger than minimum. */
@@ -240,12 +246,12 @@ public class NetworkStressTest {
 
         // Wait for peers to get their preliminary data.
         assertLatch("timed out while waiting for preliminary data",
-                prelimDataLatch, 5 * nPeers, TimeUnit.SECONDS);
+                prelimDataLatch, MAX_PRELIMINARY_DELAY_SECS * nPeers, TimeUnit.SECONDS);
         print("preliminary data received");
 
         // Wait for peers to complete their bootstrapping.
         assertLatch("timed out while waiting for bootstrap",
-                bootstrapLatch, 5 * nPeers, TimeUnit.SECONDS);
+                bootstrapLatch, MAX_BOOTSTRAP_DELAY_SECS * nPeers, TimeUnit.SECONDS);
         print("bootstrap complete");
     }
 
@@ -405,7 +411,9 @@ public class NetworkStressTest {
     @After
     public void tearDown() throws InterruptedException, IOException {
         /** A barrier to wait for concurrent shutdown of services. */
-        final CountDownLatch shutdownLatch = new CountDownLatch((seedNode != null? 1 : 0) + peerNodes.size());
+
+        final int nNodes = (seedNode != null ? 1 : 0) + peerNodes.size();
+        final CountDownLatch shutdownLatch = new CountDownLatch(nNodes);
 
         print("stopping all local nodes");
         // Stop peer nodes.
@@ -417,7 +425,8 @@ public class NetworkStressTest {
             seedNode.shutDown(() -> countDownAndPrint(shutdownLatch, '.'));
         }
         // Wait for concurrent tasks to finish.
-        shutdownLatch.await();
+        assertLatch("timed out while stopping nodes",
+                shutdownLatch, MAX_SHUTDOWN_DELAY_SECS * nNodes, TimeUnit.SECONDS);
         print("all local nodes stopped");
 
         // Cleanup test data directory.
@@ -571,7 +580,7 @@ public class NetworkStressTest {
             peer.shutDown(halfShutDown::countDown);
         }
         assertLatch("timed out while stopping a half of the peers",
-                halfShutDown, 10, TimeUnit.SECONDS);
+                halfShutDown, MAX_SHUTDOWN_DELAY_SECS * nPeers, TimeUnit.SECONDS);
         //print("stopped a half of the peers for mailbox test");
 
         // Cycle through peers sending to others, stopping the peer
@@ -622,7 +631,7 @@ public class NetworkStressTest {
             final CountDownLatch stopLatch = new CountDownLatch(1);
             onlinePeer.shutDown(stopLatch::countDown);
             assertLatch("timed out while stopping peer " + firstOnline,
-                    stopLatch, 10, TimeUnit.SECONDS);
+                    stopLatch, MAX_SHUTDOWN_DELAY_SECS, TimeUnit.SECONDS);
             //print("put peer %d offline", firstOnline);
 
             // When done, put first offline peer online and setup message listeners.
@@ -632,7 +641,9 @@ public class NetworkStressTest {
             peerNodes.set(firstOffline, startedPeer);
             startedPeer.start(new MailboxStartListener(startLatch));
             assertLatch("timed out while starting peer " + firstOffline,
-                    startLatch, 10 + MAILBOX_DELAY_SECS * nPeers, TimeUnit.SECONDS);
+                    startLatch,
+                    (MAX_PRELIMINARY_DELAY_SECS + MAX_BOOTSTRAP_DELAY_SECS) + MAILBOX_DELAY_SECS * nPeers,
+                    TimeUnit.SECONDS);
             //print("put peer %d online", firstOffline);
         }
         assertLatch("timed out while receiving mailbox messages",

From 47af7fca703f8504c3d4e2e091510f9af0425306 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 18 May 2016 13:11:44 +0200
Subject: [PATCH 099/101] Print time needed to complete mailbox stress test

---
 .../io/bitsquare/p2p/network/NetworkStressTest.java    | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 567feb3803..b1dd1d166d 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -585,7 +585,9 @@ public class NetworkStressTest {
 
         // Cycle through peers sending to others, stopping the peer
         // and starting one of the stopped peers.
+        print("%d mailbox messages to be sent by each of %d peers", mailboxCount, nPeers);
         BooleanProperty sentMailboxFailed = new SimpleBooleanProperty(false);
+        final long sendStartMillis = System.currentTimeMillis();
         for (int firstOnline = 0, firstOffline = firstPeerDown;
                 firstOnline < nPeers;
                 firstOnline++, firstOffline = ++firstOffline % nPeers) {
@@ -642,12 +644,18 @@ public class NetworkStressTest {
             startedPeer.start(new MailboxStartListener(startLatch));
             assertLatch("timed out while starting peer " + firstOffline,
                     startLatch,
+                    // this assumes some delay per received mailbox message
                     (MAX_PRELIMINARY_DELAY_SECS + MAX_BOOTSTRAP_DELAY_SECS) + MAILBOX_DELAY_SECS * nPeers,
                     TimeUnit.SECONDS);
             //print("put peer %d online", firstOffline);
         }
+        /** Time to transmit all messages with the estimated per-message delay, with 1 second computation delay. */
+        final long idealMaxMailboxDelay = 2 * (MAILBOX_DELAY_SECS + 1) * 1000 * nPeers * mailboxCount;
         assertLatch("timed out while receiving mailbox messages",
-                receivedMailboxLatch, 2 * (MAILBOX_DELAY_SECS + 1) * nPeers * mailboxCount, TimeUnit.SECONDS);
+                receivedMailboxLatch, idealMaxMailboxDelay, TimeUnit.MILLISECONDS);
+        final long recvMillis = System.currentTimeMillis() - sendStartMillis;
+        print("receiving %d mailbox messages per peer took %ss (%.2f x ideal max)",
+                mailboxCount, recvMillis / 1000.0, recvMillis / (float) idealMaxMailboxDelay);
         org.junit.Assert.assertFalse("some peer(s) failed to send a message", sentMailboxFailed.get());
     }
 

From 43fa580e8d88d821bfabf2fa1b9583178f170790 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 18 May 2016 13:15:03 +0200
Subject: [PATCH 100/101] Remove computation delay from ideal max delay of
 mailbox stress test

---
 .../test/java/io/bitsquare/p2p/network/NetworkStressTest.java | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index b1dd1d166d..7f85ddacdc 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -649,8 +649,8 @@ public class NetworkStressTest {
                     TimeUnit.SECONDS);
             //print("put peer %d online", firstOffline);
         }
-        /** Time to transmit all messages with the estimated per-message delay, with 1 second computation delay. */
-        final long idealMaxMailboxDelay = 2 * (MAILBOX_DELAY_SECS + 1) * 1000 * nPeers * mailboxCount;
+        /** Time to transmit all messages with the estimated per-message delay, with no computation delays. */
+        final long idealMaxMailboxDelay = 2 * MAILBOX_DELAY_SECS * 1000 * nPeers * mailboxCount;
         assertLatch("timed out while receiving mailbox messages",
                 receivedMailboxLatch, idealMaxMailboxDelay, TimeUnit.MILLISECONDS);
         final long recvMillis = System.currentTimeMillis() - sendStartMillis;

From dc6676d98f40618a6799cef3e021f632be7ffd97 Mon Sep 17 00:00:00 2001
From: Ivan Vilata-i-Balaguer <ivan@selidor.net>
Date: Wed, 18 May 2016 13:33:07 +0200
Subject: [PATCH 101/101] Document network stress test class for its invocation

---
 .../io/bitsquare/p2p/network/NetworkStressTest.java   | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
index 7f85ddacdc..fae8cf1284 100644
--- a/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
+++ b/network/src/test/java/io/bitsquare/p2p/network/NetworkStressTest.java
@@ -38,6 +38,17 @@ import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
 
+/**
+ * Bitsquare network stress tests.
+ *
+ * You can invoke this class directly from the command line.
+ * If the name of a single test is given as an argument, only that test is run.
+ * Otherwise all tests in the class are run.
+ *
+ * You can also set some {@code STRESS_TEST_*} environment variables to
+ * customize the execution of tests.
+ * See the {@code *_ENVVAR} constants for the names of these variables.
+ */
 public class NetworkStressTest {
     // Test parameters