From c63cf2f0a08c7f690166fee304d61112ee55d5c1 Mon Sep 17 00:00:00 2001 From: woodser Date: Sun, 28 Jul 2024 09:20:50 -0400 Subject: [PATCH] remove duplicate historical trade stats after fuzzing --- .../trade/statistics/TradeStatistics3.java | 8 +-- .../statistics/TradeStatisticsManager.java | 55 +++++++++++++------ 2 files changed, 41 insertions(+), 22 deletions(-) diff --git a/core/src/main/java/haveno/core/trade/statistics/TradeStatistics3.java b/core/src/main/java/haveno/core/trade/statistics/TradeStatistics3.java index c60903c9bd..209fad38cf 100644 --- a/core/src/main/java/haveno/core/trade/statistics/TradeStatistics3.java +++ b/core/src/main/java/haveno/core/trade/statistics/TradeStatistics3.java @@ -69,6 +69,8 @@ public final class TradeStatistics3 implements ProcessOncePersistableNetworkPayl @JsonExclude private transient static final ZoneId ZONE_ID = ZoneId.systemDefault(); + private static final double FUZZ_AMOUNT_PCT = 0.05; + private static final int FUZZ_DATE_HOURS = 24; public static TradeStatistics3 from(Trade trade, @Nullable String referralId, @@ -102,8 +104,7 @@ public final class TradeStatistics3 implements ProcessOncePersistableNetworkPayl long originalTimestamp = trade.getTakeOfferDate().getTime(); long exactAmount = trade.getAmount().longValueExact(); Random random = new Random(originalTimestamp); // pseudo random generator seeded from take offer datestamp - long adjustedAmount = (long) random.nextDouble( - exactAmount * 0.95, exactAmount * 1.05); + long adjustedAmount = (long) random.nextDouble(exactAmount * (1.0 - FUZZ_AMOUNT_PCT), exactAmount * (1 + FUZZ_AMOUNT_PCT)); log.debug("trade {} fuzzed trade amount for tradeStatistics is {}", trade.getShortId(), adjustedAmount); return adjustedAmount; } @@ -111,8 +112,7 @@ public final class TradeStatistics3 implements ProcessOncePersistableNetworkPayl private static long fuzzTradeDateReproducibly(Trade trade) { // randomize completed trade info #1099 long originalTimestamp = trade.getTakeOfferDate().getTime(); Random random = new Random(originalTimestamp); // pseudo random generator seeded from take offer datestamp - long adjustedTimestamp = random.nextLong( - originalTimestamp-TimeUnit.HOURS.toMillis(24), originalTimestamp); + long adjustedTimestamp = random.nextLong(originalTimestamp - TimeUnit.HOURS.toMillis(FUZZ_DATE_HOURS), originalTimestamp); log.debug("trade {} fuzzed trade datestamp for tradeStatistics is {}", trade.getShortId(), new Date(adjustedTimestamp)); return adjustedTimestamp; } diff --git a/core/src/main/java/haveno/core/trade/statistics/TradeStatisticsManager.java b/core/src/main/java/haveno/core/trade/statistics/TradeStatisticsManager.java index 4bbd3153de..449fca5d85 100644 --- a/core/src/main/java/haveno/core/trade/statistics/TradeStatisticsManager.java +++ b/core/src/main/java/haveno/core/trade/statistics/TradeStatisticsManager.java @@ -110,35 +110,54 @@ public class TradeStatisticsManager { maybeDumpStatistics(); } - private void deduplicateEarlyTradeStatistics(Set set) { + private void deduplicateEarlyTradeStatistics(Set tradeStats) { - // collect trades before May 31, 2024 - Set tradesBeforeMay31_24 = set.stream() - .filter(e -> e.getDate().toInstant().isBefore(Instant.parse("2024-05-31T00:00:00Z"))) + // collect trades before August 7, 2024 + Set earlyTrades = tradeStats.stream() + .filter(e -> e.getDate().toInstant().isBefore(Instant.parse("2024-08-07T00:00:00Z"))) .collect(Collectors.toSet()); // collect duplicated trades - Set duplicated = new HashSet(); - Set deduplicated = new HashSet(); - for (TradeStatistics3 tradeStatistics : tradesBeforeMay31_24) { - if (hasLenientDuplicate(tradeStatistics, deduplicated)) duplicated.add(tradeStatistics); - else deduplicated.add(tradeStatistics); + Set duplicates = new HashSet(); + Set deduplicates = new HashSet(); + Set usedAsDuplicate = new HashSet(); + for (TradeStatistics3 tradeStatistic : earlyTrades) { + TradeStatistics3 fuzzyDuplicate = findFuzzyDuplicate(tradeStatistic, deduplicates, usedAsDuplicate); + if (fuzzyDuplicate == null) deduplicates.add(tradeStatistic); + else { + duplicates.add(tradeStatistic); + usedAsDuplicate.add(fuzzyDuplicate); + } } // remove duplicated trades - set.removeAll(duplicated); + tradeStats.removeAll(duplicates); } - private boolean hasLenientDuplicate(TradeStatistics3 tradeStatistics, Set set) { - return set.stream().anyMatch(e -> isLenientDuplicate(tradeStatistics, e)); + private TradeStatistics3 findFuzzyDuplicate(TradeStatistics3 tradeStatistics, Set set, Set excluded) { + return set.stream().filter(e -> !excluded.contains(e)).filter(e -> isFuzzyDuplicate(tradeStatistics, e)).findFirst().orElse(null); } - private boolean isLenientDuplicate(TradeStatistics3 tradeStatistics1, TradeStatistics3 tradeStatistics2) { - boolean isWithin2Minutes = Math.abs(tradeStatistics1.getDate().getTime() - tradeStatistics2.getDate().getTime()) < 120000; - return isWithin2Minutes && - tradeStatistics1.getCurrency().equals(tradeStatistics2.getCurrency()) && - tradeStatistics1.getAmount() == tradeStatistics2.getAmount() && - tradeStatistics1.getPrice() == tradeStatistics2.getPrice(); + private boolean isFuzzyDuplicate(TradeStatistics3 tradeStatistics1, TradeStatistics3 tradeStatistics2) { + if (!tradeStatistics1.getPaymentMethodId().equals(tradeStatistics2.getPaymentMethodId())) return false; + if (!tradeStatistics1.getCurrency().equals(tradeStatistics2.getCurrency())) return false; + if (tradeStatistics1.getPrice() != tradeStatistics2.getPrice()) return false; + return isFuzzyDuplicateV1(tradeStatistics1, tradeStatistics2) || isFuzzyDuplicateV2(tradeStatistics1, tradeStatistics2); + } + + // bug caused all peers to publish same trade with similar timestamps + private boolean isFuzzyDuplicateV1(TradeStatistics3 tradeStatistics1, TradeStatistics3 tradeStatistics2) { + boolean isWithin2Minutes = Math.abs(tradeStatistics1.getDate().getTime() - tradeStatistics2.getDate().getTime()) <= TimeUnit.MINUTES.toMillis(2); + return isWithin2Minutes; + } + + // bug caused sellers to re-publish their trades with randomized amounts + private static final double FUZZ_AMOUNT_PCT = 0.05; + private static final int FUZZ_DATE_HOURS = 24; + private boolean isFuzzyDuplicateV2(TradeStatistics3 tradeStatistics1, TradeStatistics3 tradeStatistics2) { + boolean isWithinFuzzedHours = Math.abs(tradeStatistics1.getDate().getTime() - tradeStatistics2.getDate().getTime()) <= TimeUnit.HOURS.toMillis(FUZZ_DATE_HOURS); + boolean isWithinFuzzedAmount = Math.abs(tradeStatistics1.getAmount() - tradeStatistics2.getAmount()) <= FUZZ_AMOUNT_PCT * tradeStatistics1.getAmount(); + return isWithinFuzzedHours && isWithinFuzzedAmount; } public ObservableSet getObservableTradeStatisticsSet() {