CLI: add 'analyze' subcommand for offline HIBP breach checks

This new subcommand checks all passwords in the given database against a given list of SHA-1 password hashes. Such lists are available from the "Have I Been Pwned" project at https://haveibeenpwned.com/Passwords.

Note that this support offline checking only. The HIBP project also provides a web API for checking specific hash ranges; this is not currently supported.

src/CMakeLists.txt

@@ -41,6 +41,7 @@ set(keepassx_SOURCES
         core/FileWatcher.cpp
         core/Bootstrap.cpp
         core/Group.cpp
+        core/HibpOffline.cpp
         core/InactivityTimer.cpp
         core/Merger.cpp
         core/Metadata.cpp

src/cli/Analyze.cpp

@@ -0,0 +1,81 @@
+/*
+ *  Copyright (C) 2019 KeePassXC Team <team@keepassxc.org>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 2 or (at your option)
+ *  version 3 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "Analyze.h"
+#include "cli/Utils.h"
+#include "core/HibpOffline.h"
+
+#include <QCommandLineParser>
+#include <QFile>
+#include <QString>
+
+#include "cli/TextStream.h"
+#include "core/Group.h"
+#include "core/Tools.h"
+
+const QCommandLineOption Analyze::HIBPDatabaseOption = QCommandLineOption(
+    {"H", "hibp"},
+    QObject::tr("Check if any passwords have been publicly leaked. FILENAME must be the path of a file listing "
+                "SHA-1 hashes of leaked passwords in HIBP format, as available from "
+                "https://haveibeenpwned.com/Passwords."),
+    QObject::tr("FILENAME"));
+
+Analyze::Analyze()
+{
+    name = QString("analyze");
+    description = QObject::tr("Analyze passwords for weaknesses and problems.");
+    options.append(Analyze::HIBPDatabaseOption);
+}
+
+int Analyze::executeWithDatabase(QSharedPointer<Database> database, QSharedPointer<QCommandLineParser> parser)
+{
+    TextStream inputTextStream(Utils::STDIN, QIODevice::ReadOnly);
+    TextStream outputTextStream(Utils::STDOUT, QIODevice::WriteOnly);
+    TextStream errorTextStream(Utils::STDERR, QIODevice::WriteOnly);
+
+    QString hibpDatabase = parser->value(Analyze::HIBPDatabaseOption);
+    QFile hibpFile(hibpDatabase);
+    if (!hibpFile.open(QFile::ReadOnly)) {
+        errorTextStream << QObject::tr("Failed to open HIBP file %1: %2").arg(hibpDatabase).arg(hibpFile.errorString()) << endl;
+        return EXIT_FAILURE;
+    }
+
+    outputTextStream << QObject::tr("Evaluating database entries against HIBP file, this will take a while...");
+
+    QList<QPair<const Entry*, int>> findings;
+    QString error;
+    if (!HibpOffline::report(database, hibpFile, findings, &error)) {
+        errorTextStream << error << endl;
+        return EXIT_FAILURE;
+    }
+
+    for (auto& finding : findings) {
+        printHibpFinding(finding.first, finding.second, outputTextStream);
+    }
+
+    return EXIT_SUCCESS;
+}
+
+void Analyze::printHibpFinding(const Entry* entry, int count, QTextStream& out)
+{
+    QString path = entry->title();
+    for (auto g = entry->group(); g && g != g->database()->rootGroup(); g = g->parentGroup()) {
+        path.prepend("/").prepend(g->name());
+    }
+
+    out << QObject::tr("Password for '%1' has been leaked %2 times!").arg(path).arg(count) << endl;
+}

src/cli/Analyze.h

@@ -0,0 +1,35 @@
+/*
+ *  Copyright (C) 2019 KeePassXC Team <team@keepassxc.org>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 2 or (at your option)
+ *  version 3 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef KEEPASSXC_ANALYZE_H
+#define KEEPASSXC_ANALYZE_H
+
+#include "DatabaseCommand.h"
+
+class Analyze : public DatabaseCommand
+{
+public:
+    Analyze();
+    int executeWithDatabase(QSharedPointer<Database> db, QSharedPointer<QCommandLineParser> parser) override;
+
+    static const QCommandLineOption HIBPDatabaseOption;
+
+private:
+    void printHibpFinding(const Entry* entry, int count, QTextStream& out);
+};
+
+#endif // KEEPASSXC_HIBP_H

src/cli/CMakeLists.txt

@@ -15,6 +15,7 @@
 
 set(cli_SOURCES
         Add.cpp
+        Analyze.cpp
         Clip.cpp
         Create.cpp
         Command.cpp

src/cli/Command.cpp

@@ -23,6 +23,7 @@
 #include "Command.h"
 
 #include "Add.h"
+#include "Analyze.h"
 #include "Clip.h"
 #include "Create.h"
 #include "Diceware.h"
@@ -107,6 +108,7 @@ void populateCommands()
 {
     if (commands.isEmpty()) {
         commands.insert(QString("add"), new Add());
+        commands.insert(QString("analyze"), new Analyze());
         commands.insert(QString("clip"), new Clip());
         commands.insert(QString("create"), new Create());
         commands.insert(QString("diceware"), new Diceware());

src/cli/keepassxc-cli.1

@@ -16,6 +16,9 @@ keepassxc-cli \- command line interface for the \fBKeePassXC\fP password manager
 .IP "add [options] <database> <entry>"
 Adds a new entry to a database. A password can be generated (\fI-g\fP option), or a prompt can be displayed to input the password (\fI-p\fP option).
 
+.IP "analyze [options] <database>"
+Analyze passwords in a database for weaknesses.
+
 .IP "clip [options] <database> <entry> [timeout]"
 Copies the password or the current TOTP (\fI-t\fP option) of a database entry to the clipboard. If multiple entries with the same name exist in different groups, only the password for the first one is going to be copied. For copying the password of an entry in a specific group, the group path to the entry should be specified as well, instead of just the name. Optionally, a timeout in seconds can be specified to automatically clear the clipboard.
 
@@ -120,6 +123,16 @@ Specify the title of the entry.
 Perform advanced analysis on the password.
 
 
+.SS "Analyze options"
+
+.IP "-H, --hibp <filename>"
+Check if any passwords have been publicly leaked, by comparing against the given
+list of password SHA-1 hashes, which must be in "Have I Been Pwned" format. Such
+files are available from https://haveibeenpwned.com/Passwords; note that they
+are large, and so this operation typically takes some time (minutes up to an
+hour or so).
+
+
 .SS "Clip options"
 
 .IP "-t, --totp"

src/core/Entry.cpp

@@ -341,6 +341,16 @@ bool Entry::isExpired() const
     return m_data.timeInfo.expires() && m_data.timeInfo.expiryTime() < Clock::currentDateTimeUtc();
 }
 
+bool Entry::isRecycled() const
+{
+    const Database* db = database();
+    if (!db) {
+        return false;
+    }
+
+    return m_group == db->metadata()->recycleBin() || m_group->isRecycled();
+}
+
 bool Entry::isAttributeReference(const QString& key) const
 {
     return m_attributes->isReference(key);

src/core/Entry.h

@@ -111,6 +111,7 @@ public:
 
     bool hasTotp() const;
     bool isExpired() const;
+    bool isRecycled() const;
     bool isAttributeReference(const QString& key) const;
     bool isAttributeReferenceOf(const QString& key, const QUuid& uuid) const;
     void replaceReferencesWithValues(const Entry* other);

src/core/HibpOffline.cpp

@@ -0,0 +1,109 @@
+/*
+ *  Copyright (C) 2019 KeePassXC Team <team@keepassxc.org>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 2 or (at your option)
+ *  version 3 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "HibpOffline.h"
+
+#include <QCryptographicHash>
+#include <QMultiHash>
+
+#include "core/Database.h"
+#include "core/Group.h"
+
+namespace HibpOffline
+{
+    const std::size_t SHA1_BYTES = 20;
+
+    enum class ParseResult
+    {
+        Ok,
+        Eof,
+        Error
+    };
+
+    ParseResult parseHibpLine(QIODevice& input, QByteArray& sha1, int& count)
+    {
+        QByteArray hexSha1(SHA1_BYTES * 2, '\0');
+        const qint64 rc = input.read(hexSha1.data(), hexSha1.size());
+        if (rc == 0) {
+            return ParseResult::Eof;
+        } else if (rc != hexSha1.size()) {
+            return ParseResult::Error;
+        }
+
+        sha1 = QByteArray::fromHex(hexSha1);
+
+        char c;
+        if (!input.getChar(&c) || c != ':') {
+            return ParseResult::Error;
+        }
+
+        count = 0;
+        while (true) {
+            if (!input.getChar(&c)) {
+                return ParseResult::Error;
+            }
+
+            if (c == '\n' || c == '\r') {
+                break;
+            }
+
+            if (!('0' <= c && c <= '9')) {
+                return ParseResult::Error;
+            }
+
+            count *= 10;
+            count += (c - '0');
+        }
+
+        while (1 == input.peek(&c, 1) && (c == '\n' || c == '\r')) {
+            input.getChar(&c);
+        }
+
+        return ParseResult::Ok;
+    }
+
+    bool
+    report(QSharedPointer<Database> db, QIODevice& hibpInput, QList<QPair<const Entry*, int>>& findings, QString* error)
+    {
+        QMultiHash<QByteArray, const Entry*> entriesBySha1;
+        for (const auto* entry : db->rootGroup()->entriesRecursive()) {
+            if (!entry->isRecycled()) {
+                const auto sha1 = QCryptographicHash::hash(entry->password().toUtf8(), QCryptographicHash::Sha1);
+                entriesBySha1.insert(sha1, entry);
+            }
+        }
+
+        QByteArray sha1;
+        for (quint64 lineNum = 1;; ++lineNum) {
+            int count = 0;
+
+            switch (parseHibpLine(hibpInput, sha1, count)) {
+            case ParseResult::Eof:
+                return true;
+            case ParseResult::Error:
+                *error = QObject::tr("HIBP file, line %1: parse error").arg(lineNum);
+                return false;
+            default:
+                break;
+            }
+
+            for (const auto* entry : entriesBySha1.values(sha1)) {
+                findings.append({entry, count});
+            }
+        }
+    }
+} // namespace HibpOffline

src/core/HibpOffline.h

@@ -0,0 +1,36 @@
+/*
+ *  Copyright (C) 2019 KeePassXC Team <team@keepassxc.org>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 2 or (at your option)
+ *  version 3 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef KEEPASSXC_HIBPOFFLINE_H
+#define KEEPASSXC_HIBPOFFLINE_H
+
+#include <QIODevice>
+#include <QList>
+#include <QPair>
+
+class Database;
+class Entry;
+
+namespace HibpOffline
+{
+    bool report(QSharedPointer<Database> db,
+                QIODevice& hibpInput,
+                QList<QPair<const Entry*, int>>& findings,
+                QString* error);
+}
+
+#endif // KEEPASSXC_HIBPOFFLINE_H

src/core/Tools.cpp

@@ -193,7 +193,7 @@ namespace Tools
 
     bool isHex(const QByteArray& ba)
     {
-        for (const unsigned char c : ba) {
+        for (const uchar c : ba) {
             if (!std::isxdigit(c)) {
                 return false;
             }

tests/CMakeLists.txt

@@ -193,6 +193,9 @@ add_unit_test(NAME testpasswordgenerator SOURCES TestPasswordGenerator.cpp
 add_unit_test(NAME testpassphrasegenerator SOURCES TestPassphraseGenerator.cpp
         LIBS ${TEST_LIBRARIES})
 
+add_unit_test(NAME testhibp SOURCES TestHibp.cpp
+        LIBS ${TEST_LIBRARIES})
+
 add_unit_test(NAME testtotp SOURCES TestTotp.cpp
         LIBS ${TEST_LIBRARIES})
 

tests/TestCli.cpp

@@ -32,6 +32,7 @@
 #include "format/KeePass2.h"
 
 #include "cli/Add.h"
+#include "cli/Analyze.h"
 #include "cli/Clip.h"
 #include "cli/Command.h"
 #include "cli/Create.h"
@@ -51,6 +52,7 @@
 #include <QFile>
 #include <QFuture>
 #include <QSet>
+#include <QTextStream>
 #include <QtConcurrent>
 
 #include <cstdio>
@@ -160,8 +162,9 @@ QSharedPointer<Database> TestCli::readTestDatabase() const
 
 void TestCli::testCommand()
 {
-    QCOMPARE(Command::getCommands().size(), 13);
+    QCOMPARE(Command::getCommands().size(), 14);
     QVERIFY(Command::getCommand("add"));
+    QVERIFY(Command::getCommand("analyze"));
     QVERIFY(Command::getCommand("clip"));
     QVERIFY(Command::getCommand("create"));
     QVERIFY(Command::getCommand("diceware"));
@@ -239,6 +242,22 @@ void TestCli::testAdd()
     QCOMPARE(entry->password(), QString("newpassword"));
 }
 
+void TestCli::testAnalyze()
+{
+    Analyze analyzeCmd;
+    QVERIFY(!analyzeCmd.name.isEmpty());
+    QVERIFY(analyzeCmd.getDescriptionLine().contains(analyzeCmd.name));
+
+    const QString hibpPath = QString(KEEPASSX_TEST_DATA_DIR).append("/hibp.txt");
+
+    Utils::Test::setNextPassword("a");
+    analyzeCmd.execute({"analyze", "--hibp", hibpPath, m_dbFile->fileName()});
+    m_stdoutFile->reset();
+    m_stdoutFile->readLine(); // skip password prompt
+    auto output = m_stdoutFile->readAll();
+    QVERIFY(output.contains("Sample Entry") && output.contains("123"));
+}
+
 bool isTOTP(const QString& value)
 {
     QString val = value.trimmed();

tests/TestCli.h

@@ -21,11 +21,14 @@
 #include "core/Database.h"
 #include "util/TemporaryFile.h"
 
+#include <QByteArray>
 #include <QFile>
 #include <QScopedPointer>
+#include <QSharedPointer>
 #include <QTemporaryFile>
 #include <QTest>
-#include <QTextStream>
+
+#include <stdio.h>
 
 class TestCli : public QObject
 {
@@ -42,6 +45,7 @@ private slots:
 
     void testCommand();
     void testAdd();
+    void testAnalyze();
     void testClip();
     void testCreate();
     void testDiceware();

tests/TestEntry.cpp

@@ -21,6 +21,7 @@
 #include "TestEntry.h"
 #include "TestGlobal.h"
 #include "core/Clock.h"
+#include "core/Metadata.h"
 #include "crypto/Crypto.h"
 
 QTEST_GUILESS_MAIN(TestEntry)
@@ -561,3 +562,28 @@ void TestEntry::testResolveClonedEntry()
     QCOMPARE(cclone4->resolveMultiplePlaceholders(cclone4->username()), original->username());
     QCOMPARE(cclone4->resolveMultiplePlaceholders(cclone4->password()), original->password());
 }
+
+void TestEntry::testIsRecycled()
+{
+    Entry* entry = new Entry();
+    QVERIFY(!entry->isRecycled());
+
+    Database db;
+    Group* root = db.rootGroup();
+    QVERIFY(root);
+    entry->setGroup(root);
+    QVERIFY(!entry->isRecycled());
+
+    QVERIFY(db.metadata()->recycleBinEnabled());
+    db.recycleEntry(entry);
+    QVERIFY(entry->isRecycled());
+
+    Group* group1 = new Group();
+    group1->setParent(root);
+
+    Entry* entry1 = new Entry();
+    entry1->setGroup(group1);
+    QVERIFY(!entry1->isRecycled());
+    db.recycleGroup(group1);
+    QVERIFY(entry1->isRecycled());
+}

tests/TestEntry.h

@@ -37,6 +37,7 @@ private slots:
     void testResolveReferencePlaceholders();
     void testResolveNonIdPlaceholdersToUuid();
     void testResolveClonedEntry();
+    void testIsRecycled();
 };
 
 #endif // KEEPASSX_TESTENTRY_H

tests/TestHibp.cpp

@@ -0,0 +1,125 @@
+/*
+ *  Copyright (C) 2019 KeePassXC Team <team@keepassxc.org>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 2 or (at your option)
+ *  version 3 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "TestHibp.h"
+
+#include "config-keepassx-tests.h"
+
+#include "core/Database.h"
+#include "core/Entry.h"
+#include "core/Group.h"
+#include "core/HibpOffline.h"
+#include "crypto/Crypto.h"
+
+#include <QBuffer>
+#include <QByteArray>
+#include <QFile>
+#include <QList>
+#include <QTest>
+
+QTEST_GUILESS_MAIN(TestHibp)
+
+const char* TEST_HIBP_CONTENTS = "0BEEC7B5EA3F0FDBC95D0DD47F3C5BC275DA8A33:123\n" // SHA-1 of "foo"
+                                 "62cdb7020ff920e5aa642c3d4066950dd1f01f4d:456\n"; // SHA-1 of "bar"
+
+const char* TEST_BAD_HIBP_CONTENTS = "barf:nope\n";
+
+void TestHibp::initTestCase()
+{
+    QVERIFY(Crypto::init());
+}
+
+void TestHibp::init()
+{
+    m_db.reset(new Database());
+}
+
+void TestHibp::testBadHibpFormat()
+{
+    QByteArray hibpContents(TEST_BAD_HIBP_CONTENTS);
+    QBuffer hibpBuffer(&hibpContents);
+    QVERIFY(hibpBuffer.open(QIODevice::ReadOnly));
+
+    QList<QPair<const Entry*, int>> findings;
+    QString error;
+    QVERIFY(!HibpOffline::report(m_db, hibpBuffer, findings, &error));
+    QVERIFY(!error.isEmpty());
+    QCOMPARE(findings.size(), 0);
+}
+
+void TestHibp::testEmpty()
+{
+    QByteArray hibpContents(TEST_HIBP_CONTENTS);
+    QBuffer hibpBuffer(&hibpContents);
+    QVERIFY(hibpBuffer.open(QIODevice::ReadOnly));
+
+    QList<QPair<const Entry*, int>> findings;
+    QString error;
+    QVERIFY(HibpOffline::report(m_db, hibpBuffer, findings, &error));
+    QCOMPARE(error, QString());
+    QCOMPARE(findings.size(), 0);
+}
+
+void TestHibp::testIoError()
+{
+    QBuffer hibpBuffer;
+    // hibpBuffer has not been opened, so reading will cause I/O error
+
+    QList<QPair<const Entry*, int>> findings;
+    QString error;
+    QVERIFY(!HibpOffline::report(m_db, hibpBuffer, findings, &error));
+    QVERIFY(!error.isEmpty());
+    QCOMPARE(findings.size(), 0);
+}
+
+void TestHibp::testPwned()
+{
+    QByteArray hibpContents(TEST_HIBP_CONTENTS);
+    QBuffer hibpBuffer(&hibpContents);
+    QVERIFY(hibpBuffer.open(QIODevice::ReadOnly));
+
+    Group* root = m_db->rootGroup();
+
+    Entry* entry1 = new Entry();
+    entry1->setPassword("foo");
+    entry1->setGroup(root);
+
+    Entry* entry2 = new Entry();
+    entry2->setPassword("xyz");
+    entry2->setGroup(root);
+
+    Entry* entry3 = new Entry();
+    entry3->setPassword("foo");
+    m_db->recycleEntry(entry3);
+
+    Group* group1 = new Group();
+    group1->setParent(root);
+
+    Entry* entry4 = new Entry();
+    entry4->setPassword("bar");
+    entry4->setGroup(group1);
+
+    QList<QPair<const Entry*, int>> findings;
+    QString error;
+    QVERIFY(HibpOffline::report(m_db, hibpBuffer, findings, &error));
+    QCOMPARE(error, QString());
+    QCOMPARE(findings.size(), 2);
+    QCOMPARE(findings[0].first, entry1);
+    QCOMPARE(findings[0].second, 123);
+    QCOMPARE(findings[1].first, entry4);
+    QCOMPARE(findings[1].second, 456);
+}

tests/TestHibp.h

@@ -0,0 +1,42 @@
+/*
+ *  Copyright (C) 2019 KeePassXC Team <team@keepassxc.org>
+ *
+ *  This program is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 2 or (at your option)
+ *  version 3 of the License.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef KEEPASSXC_TESTHIBP_H
+#define KEEPASSXC_TESTHIBP_H
+
+#include <QObject>
+#include <QSharedPointer>
+
+class Database;
+
+class TestHibp : public QObject
+{
+    Q_OBJECT
+
+private slots:
+    void initTestCase();
+    void init();
+    void testBadHibpFormat();
+    void testEmpty();
+    void testIoError();
+    void testPwned();
+
+private:
+    QSharedPointer<Database> m_db;
+};
+
+#endif // KEEPASSXC_TESTHIBP_H

tests/data/hibp.txt

@@ -0,0 +1,5 @@
+000000005AD76BD555C1D6D771DE417A4B87E4B4:4
+00000000A8DAE4228F821FB418F59826079BF368:2
+8BE3C943B1609FFFBFC51AAD666D0A04ADF83C9D:123
+00000000DD7F2A1C68A35673713783CA390C9E93:630
+00000001E225B908BAC31C56DB04D892E47536E0:5