split resources out

scrape.py

@@ -4,28 +4,28 @@ import os
 import requests
 import bs4
 
-os.system('mkdir -p infodump/thumbs')
+os.system("mkdir -p infodump/thumbs")
-url = 'https://moneroinfodump.neocities.org/'
+url = "https://moneroinfodump.neocities.org/"
 contents = requests.get(url, timeout=15).content
-soup = bs4.BeautifulSoup(contents, 'html.parser')
+soup = bs4.BeautifulSoup(contents, "html.parser")
-images = soup.find_all('img')
+images = soup.find_all("img")
-links = soup.find_all('a')
+links = soup.find_all("a")
 
 for image in images:
-    img = image.get('src')
+    img = image.get("src")
-    if img.startswith('http'):
+    if img.startswith("http"):
-        os.system(f'wget -q --no-clobber -O infodump/{os.path.basename(img)} {img}')
+        os.system(f"wget -q --no-clobber -O infodump/{os.path.basename(img)} {img}")
-        image['src'] = os.path.basename(img)
+        image["src"] = os.path.basename(img)
-    elif img.startswith('data:image/png'):
+    elif img.startswith("data:image/png"):
         pass
     else:
-        os.system(f'wget -q --no-clobber -O infodump/{img} {img}')
+        os.system(f"wget -q --no-clobber -O infodump/{img} {img}")
-        image['src'] = img
+        image["src"] = img
 
 for link in links:
-    href = link.get('href')
+    href = link.get("href")
-    if href and href.startswith('https://i.imgur.com'):
+    if href and href.startswith("https://i.imgur.com"):
-        link['href'] = os.path.basename(href)
+        link["href"] = os.path.basename(href)
 
-with open('infodump/index.html', 'w') as f:
+with open("infodump/index.html", "w") as f:
     f.write(str(soup))

xmrnodes/app.py

@@ -1,421 +1,23 @@
-import re
 import logging
-from random import shuffle
-from datetime import datetime, timedelta
 
-import geoip2.database
+from flask import Flask
-import arrow
-import requests
-from flask import Flask, request, redirect, jsonify
-from flask import render_template, flash, Response
-from urllib.parse import urlparse, urlencode
 
-from xmrnodes.helpers import determine_crypto, is_onion, make_request
+from xmrnodes.routes import meta, api
-from xmrnodes.helpers import retrieve_peers, rw_cache, get_highest_block
+from xmrnodes import cli, filters
-from xmrnodes.forms import SubmitNode
-from xmrnodes.models import Node, HealthCheck, Peer
-from xmrnodes import config
 
 
 logging.basicConfig(
-    level=logging.INFO,
+    level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s"
-    format="%(asctime)s - %(levelname)s - %(message)s"
 )
 
 app = Flask(__name__)
 app.config.from_envvar("FLASK_SECRETS")
 app.secret_key = app.config["SECRET_KEY"]
-HEALTHY_BLOCK_DIFF = 500 # idc to config this. hardcode is fine.
+app.register_blueprint(meta.bp)
+app.register_blueprint(api.bp)
+app.register_blueprint(cli.bp)
+app.register_blueprint(filters.bp)
 
-@app.route("/", methods=["GET", "POST"])
-def index():
-    form = SubmitNode()
-    nettype = request.args.get("network", "mainnet")
-    crypto = request.args.get("chain", "monero")
-    onion = request.args.get("onion", False)
-    show_all = "true" == request.args.get("all", "false")
-    web_compatible = request.args.get("cors", False)
-    highest_block = get_highest_block(nettype, crypto)
-    healthy_block = highest_block - HEALTHY_BLOCK_DIFF
-
-    nodes = Node.select().where(
-        Node.validated == True,
-        Node.nettype == nettype,
-        Node.crypto == crypto
-    )
-
-    if web_compatible:
-        nodes = nodes.where(Node.web_compatible == True)
-
-    nodes_all = nodes.count()
-    nodes_unhealthy = nodes.where(
-        (Node.available == False) | (Node.last_height < healthy_block)
-    ).count()
-
-    if not show_all:
-        nodes = nodes.where(
-            Node.available == True,
-            Node.last_height > healthy_block
-        )
-
-    nodes = nodes.order_by(
-        Node.datetime_entered.desc()
-    )
-    if onion:
-        nodes = nodes.where(Node.is_tor == True)
-
-    nodes = [n for n in nodes]
-    shuffle(nodes)
-
-    return render_template(
-        "index.html",
-        nodes=nodes,
-        nodes_all=nodes_all,
-        nodes_unhealthy=nodes_unhealthy,
-        nettype=nettype,
-        crypto=crypto,
-        form=form,
-        web_compatible=web_compatible
-    )
-
-@app.route("/nodes.json")
-def nodes_json():
-    nodes = Node.select().where(
-        Node.validated==True
-    ).where(
-        Node.nettype=="mainnet"
-    )
-    xmr_nodes = [n for n in nodes if n.crypto == "monero"]
-    wow_nodes = [n for n in nodes if n.crypto == "wownero"]
-    return jsonify({
-        "monero": {
-            "clear": [n.url for n in xmr_nodes if n.is_tor == False],
-            "onion": [n.url for n in xmr_nodes if n.is_tor == True],
-            "web_compatible": [n.url for n in xmr_nodes if n.web_compatible == True],
-        },
-        "wownero": {
-            "clear": [n.url for n in wow_nodes if n.is_tor == False],
-            "onion": [n.url for n in wow_nodes if n.is_tor == True]
-        }
-    })
-
-@app.route("/health.json")
-def health_json():
-    data = {}
-    nodes = Node.select().where(
-        Node.validated == True
-    )
-    for node in nodes:
-        if node.crypto not in data:
-            data[node.crypto] = {}
-        _d = {
-            "available": node.available,
-            "last_height": node.last_height,
-            "datetime_entered": node.datetime_entered,
-            "datetime_checked": node.datetime_checked,
-            "datetime_failed": node.datetime_failed,
-            "checks": [c.health for c in node.get_all_checks()]
-        }
-        nettype = "clear"
-        if node.is_tor:
-            nettype = "onion"
-        elif node.web_compatible:
-            if "web_compatible" not in data[node.crypto]:
-                data[node.crypto]["web_compatible"] = {}
-            data[node.crypto]["web_compatible"][node.url] = _d
-        if nettype not in data[node.crypto]:
-            data[node.crypto][nettype] = {}
-        data[node.crypto][nettype][node.url] = _d
-    return jsonify(data)
-
-@app.route("/haproxy.cfg")
-def haproxy():
-    crypto = request.args.get('chain') or 'monero'
-    nettype = request.args.get('network') or 'mainnet'
-    cors = request.args.get('cors') or False
-    tor = request.args.get('onion') or False
-    nodes = Node.select().where(
-        Node.validated == True,
-        Node.nettype == nettype,
-        Node.crypto == crypto,
-        Node.is_tor == tor,
-        Node.web_compatible == cors
-    )
-    tpl = render_template("haproxy.html", nodes=nodes)
-    print(tpl)
-    res = Response(tpl)
-    res.headers['Content-Disposition'] = f'attachment; filename="haproxy-{crypto}-{nettype}-cors_{cors}-tor_{tor}.cfg"'
-    return res
-
-@app.route("/wow_nodes.json")
-def wow_nodes_json():
-    nodes = Node.select().where(
-        Node.validated==True
-    ).where(
-        Node.nettype=="mainnet"
-    ).where(
-        Node.crypto=="wownero"
-    )
-    nodes = [n for n in nodes]
-    return jsonify({
-        "clear": [n.url for n in nodes if n.is_tor == False],
-        "onion": [n.url for n in nodes if n.is_tor == True]
-    })
-
-@app.route("/map")
-def map():
-    try:
-        peers = rw_cache('map_peers')
-    except:
-        flash('Couldn\'t load the map. Try again later.')
-        return redirect('/')
-    return render_template(
-        "map.html",
-        peers=peers,
-        source_node=config.NODE_HOST
-    )
-
-@app.route("/resources")
-def resources():
-    return render_template("resources.html")
-
-@app.route("/add", methods=["GET", "POST"])
-def add():
-    if request.method == "POST":
-        url = request.form.get("node_url")
-        regex = re.compile(
-            r"^(?:http)s?://" # http:// or https://
-            r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|" #domain...
-            r"localhost|" #localhost...
-            r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})" # ...or ip
-            r"(?::\d+)?" # optional port
-            r"(?:/?|[/?]\S+)$", re.IGNORECASE
-        )
-        re_match = re.match(regex, url)
-        if re_match is None:
-            flash("This doesn\"t look like a valid URL")
-        else:
-            _url = urlparse(url)
-            url = f"{_url.scheme}://{_url.netloc}".lower()
-            if Node.select().where(Node.url == url).exists():
-                flash("This node is already in the database.")
-            else:
-                flash("Seems like a valid node URL. Added to the database and will check soon.")
-                node = Node(url=url)
-                node.save()
-    return redirect("/")
-
-def cleanup_health_checks():
-    diff = datetime.utcnow() - timedelta(hours=24)
-    checks = HealthCheck.select().where(HealthCheck.datetime <= diff)
-    for check in checks:
-        print("Deleting check", check.id)
-        check.delete_instance()
-
-@app.cli.command("check")
-def check():
-    cleanup_health_checks()
-    nodes = Node.select().where(Node.validated == True)
-    for node in nodes:
-        now = datetime.utcnow()
-        hc = HealthCheck(node=node)
-        logging.info(f"Attempting to check {node.url}")
-        try:
-            r = make_request(node.url)
-            assert "status" in r.json()
-            assert "offline" in r.json()
-            assert "height" in r.json()
-            has_cors = 'Access-Control-Allow-Origin' in r.headers
-            is_ssl = node.url.startswith('https://')
-            if r.json()["status"] == "OK":
-                node.web_compatible = has_cors and is_ssl
-                node.last_height = r.json()["height"]
-                hc.health = True
-                highest_block = get_highest_block(node.nettype, node.crypto)
-                healthy_block = highest_block - HEALTHY_BLOCK_DIFF
-                if r.json()["height"] < healthy_block:
-                    node.available = False
-                    logging.info("unhealthy")
-                else:
-                    node.available = True
-                    logging.info("success")
-            else:
-                raise
-        except:
-            logging.info("fail")
-            node.datetime_failed = now
-            node.available = False
-            hc.health = False
-        finally:
-            node.datetime_checked = now
-            node.save()
-            hc.save()
-        if node.get_failed_checks().count() == node.get_all_checks().count() and node.get_all_checks().count() > 5:
-            print('this node fails all of its health checks - deleting it!')
-            for _hc in node.get_all_checks():
-                _hc.delete_instance()
-            node.delete_instance()
-
-@app.cli.command("get_peers")
-def get_peers():
-    all_peers = []
-    print('[+] Preparing to crawl Monero p2p network')
-    print(f'[.] Retrieving initial peers from {config.NODE_HOST}:{config.NODE_PORT}')
-    initial_peers = retrieve_peers(config.NODE_HOST, config.NODE_PORT)
-    with geoip2.database.Reader('./data/GeoLite2-City.mmdb') as reader:
-        for peer in initial_peers:
-            if peer not in all_peers:
-                all_peers.append(peer)
-            _url = urlparse(peer)
-            url = f"{_url.scheme}://{_url.netloc}".lower()
-            if not Peer.select().where(Peer.url == peer).exists():
-                response = reader.city(_url.hostname)
-                p = Peer(
-                    url=peer,
-                    country=response.country.name,
-                    city=response.city.name,
-                    postal=response.postal.code,
-                    lat=response.location.latitude,
-                    lon=response.location.longitude,
-                )
-                p.save()
-                print(f'{peer} - saving new peer')
-            else:
-                p = Peer.select().where(Peer.url == peer).first()
-                p.datetime = datetime.now()
-                p.save()
-
-            try:
-                print(f'[.] Retrieving crawled peers from {_url.netloc}')
-                new_peers = retrieve_peers(_url.hostname, _url.port)
-                for peer in new_peers:
-                    if peer not in all_peers:
-                        all_peers.append(peer)
-                    _url = urlparse(peer)
-                    url = f"{_url.scheme}://{_url.netloc}".lower()
-                    if not Peer.select().where(Peer.url == peer).exists():
-                        response = reader.city(_url.hostname)
-                        p = Peer(
-                            url=peer,
-                            country=response.country.name,
-                            city=response.city.name,
-                            postal=response.postal.code,
-                            lat=response.location.latitude,
-                            lon=response.location.longitude,
-                        )
-                        p.save()
-                        print(f'{peer} - saving new peer')
-                    else:
-                        p = Peer.select().where(Peer.url == peer).first()
-                        p.datetime = datetime.now()
-                        p.save()
-            except:
-                pass
-
-    print(f'[+] Found {len(all_peers)} peers from {config.NODE_HOST}:{config.NODE_PORT}')
-    print('[+] Deleting old Monero p2p peers')
-    for p in Peer.select():
-        if p.hours_elapsed() > 24:
-            print(f'[.] Deleting {p.url}')
-            p.delete_instance()
-    rw_cache('map_peers', list(Peer.select().execute()))
-
-
-@app.cli.command('init')
-def init():
-    pass
-
-@app.cli.command("validate")
-def validate():
-    nodes = Node.select().where(Node.validated == False)
-    for node in nodes:
-        now = datetime.utcnow()
-        logging.info(f"Attempting to validate {node.url}")
-        try:
-            r = make_request(node.url)
-            assert "height" in r.json()
-            assert "nettype" in r.json()
-            has_cors = 'Access-Control-Allow-Origin' in r.headers
-            is_ssl = node.url.startswith('https://')
-            nettype = r.json()["nettype"]
-            crypto = determine_crypto(node.url)
-            logging.info("success")
-            if nettype in ["mainnet", "stagenet", "testnet"]:
-                node.nettype = nettype
-                node.available = True
-                node.validated = True
-                node.web_compatible = has_cors and is_ssl
-                node.last_height = r.json()["height"]
-                node.datetime_checked = now
-                node.crypto = crypto
-                node.is_tor = is_onion(node.url)
-                node.save()
-            else:
-                logging.info("unexpected nettype")
-        except requests.exceptions.ConnectTimeout:
-            logging.info("connection timed out")
-            node.delete_instance()
-        except requests.exceptions.SSLError:
-            logging.info("invalid certificate")
-            node.delete_instance()
-        except requests.exceptions.ConnectionError:
-            logging.info("connection error")
-            node.delete_instance()
-        except requests.exceptions.HTTPError:
-            logging.info("http error, 4xx or 5xx")
-            node.delete_instance()
-        except Exception as e:
-            logging.info("failed for reasons unknown")
-            node.delete_instance()
-
-@app.cli.command("export")
-def export():
-    all_nodes = []
-    ts = int(arrow.get().timestamp())
-    export_dir = f"{config.DATA_DIR}/export.txt"
-    export_dir_stamped = f"{config.DATA_DIR}/export-{ts}.txt"
-    nodes = Node.select().where(Node.validated == True)
-    for node in nodes:
-        logging.info(f"Adding {node.url}")
-        all_nodes.append(node.url)
-    with open(export_dir, "w") as f:
-        f.write("\n".join(all_nodes))
-    with open(export_dir_stamped, "w") as f:
-        f.write("\n".join(all_nodes))
-    logging.info(f"{nodes.count()} nodes written to {export_dir} and {export_dir_stamped}")
-
-@app.cli.command("import")
-def import_():
-    all_nodes = []
-    export_dir = f"{config.DATA_DIR}/export.txt"
-    with open(export_dir, "r") as f:
-        for url in f.readlines():
-            try:
-                n = url.rstrip().lower()
-                logging.info(f"Adding {n}")
-                node = Node(url=n)
-                node.save()
-                all_nodes.append(n)
-            except:
-                pass
-    logging.info(f"{len(all_nodes)} node urls imported and ready to be validated")
-
-@app.template_filter("humanize")
-def humanize(d):
-    t = arrow.get(d, "UTC")
-    return t.humanize()
-
-@app.template_filter("hours_elapsed")
-def hours_elapsed(d):
-    now = datetime.utcnow()
-    diff = now - d
-    return diff.total_seconds() / 60 / 60
-
-@app.template_filter("pop_arg")
-def trim_arg(all_args, arg_to_trim):
-    d = all_args.to_dict()
-    d.pop(arg_to_trim)
-    return urlencode(d)
 
 if __name__ == "__main__":
     app.run()

xmrnodes/cli.py

@@ -0,0 +1,219 @@
+import logging
+from datetime import datetime, timedelta
+
+import geoip2.database
+import arrow
+import requests
+from flask import Blueprint
+from urllib.parse import urlparse
+
+from xmrnodes.helpers import determine_crypto, is_onion, make_request
+from xmrnodes.helpers import retrieve_peers, rw_cache, get_highest_block
+from xmrnodes.models import Node, HealthCheck, Peer
+from xmrnodes import config
+
+bp = Blueprint("cli", "cli", cli_group=None)
+
+
+@bp.cli.command("init")
+def init():
+    pass
+
+
+@bp.cli.command("check")
+def check():
+    diff = datetime.utcnow() - timedelta(hours=24)
+    checks = HealthCheck.select().where(HealthCheck.datetime <= diff)
+    for check in checks:
+        print("Deleting check", check.id)
+        check.delete_instance()
+    nodes = Node.select().where(Node.validated == True)
+    for node in nodes:
+        now = datetime.utcnow()
+        hc = HealthCheck(node=node)
+        logging.info(f"Attempting to check {node.url}")
+        try:
+            r = make_request(node.url)
+            assert "status" in r.json()
+            assert "offline" in r.json()
+            assert "height" in r.json()
+            has_cors = "Access-Control-Allow-Origin" in r.headers
+            is_ssl = node.url.startswith("https://")
+            if r.json()["status"] == "OK":
+                node.web_compatible = has_cors and is_ssl
+                node.last_height = r.json()["height"]
+                hc.health = True
+                highest_block = get_highest_block(node.nettype, node.crypto)
+                healthy_block = highest_block - config.HEALTHY_BLOCK_DIFF
+                if r.json()["height"] < healthy_block:
+                    node.available = False
+                    logging.info("unhealthy")
+                else:
+                    node.available = True
+                    logging.info("success")
+            else:
+                raise
+        except:
+            logging.info("fail")
+            node.datetime_failed = now
+            node.available = False
+            hc.health = False
+        finally:
+            node.datetime_checked = now
+            node.save()
+            hc.save()
+        if (
+            node.get_failed_checks().count() == node.get_all_checks().count()
+            and node.get_all_checks().count() > 5
+        ):
+            print("this node fails all of its health checks - deleting it!")
+            for _hc in node.get_all_checks():
+                _hc.delete_instance()
+            node.delete_instance()
+
+
+@bp.cli.command("get_peers")
+def get_peers():
+    all_peers = []
+    print("[+] Preparing to crawl Monero p2p network")
+    print(f"[.] Retrieving initial peers from {config.NODE_HOST}:{config.NODE_PORT}")
+    initial_peers = retrieve_peers(config.NODE_HOST, config.NODE_PORT)
+    with geoip2.database.Reader("./data/GeoLite2-City.mmdb") as reader:
+        for peer in initial_peers:
+            if peer not in all_peers:
+                all_peers.append(peer)
+            _url = urlparse(peer)
+            url = f"{_url.scheme}://{_url.netloc}".lower()
+            if not Peer.select().where(Peer.url == peer).exists():
+                response = reader.city(_url.hostname)
+                p = Peer(
+                    url=peer,
+                    country=response.country.name,
+                    city=response.city.name,
+                    postal=response.postal.code,
+                    lat=response.location.latitude,
+                    lon=response.location.longitude,
+                )
+                p.save()
+                print(f"{peer} - saving new peer")
+            else:
+                p = Peer.select().where(Peer.url == peer).first()
+                p.datetime = datetime.now()
+                p.save()
+
+            try:
+                print(f"[.] Retrieving crawled peers from {_url.netloc}")
+                new_peers = retrieve_peers(_url.hostname, _url.port)
+                for peer in new_peers:
+                    if peer not in all_peers:
+                        all_peers.append(peer)
+                    _url = urlparse(peer)
+                    url = f"{_url.scheme}://{_url.netloc}".lower()
+                    if not Peer.select().where(Peer.url == peer).exists():
+                        response = reader.city(_url.hostname)
+                        p = Peer(
+                            url=peer,
+                            country=response.country.name,
+                            city=response.city.name,
+                            postal=response.postal.code,
+                            lat=response.location.latitude,
+                            lon=response.location.longitude,
+                        )
+                        p.save()
+                        print(f"{peer} - saving new peer")
+                    else:
+                        p = Peer.select().where(Peer.url == peer).first()
+                        p.datetime = datetime.now()
+                        p.save()
+            except:
+                pass
+
+    print(
+        f"[+] Found {len(all_peers)} peers from {config.NODE_HOST}:{config.NODE_PORT}"
+    )
+    print("[+] Deleting old Monero p2p peers")
+    for p in Peer.select():
+        if p.hours_elapsed() > 24:
+            print(f"[.] Deleting {p.url}")
+            p.delete_instance()
+    rw_cache("map_peers", list(Peer.select().execute()))
+
+
+@bp.cli.command("validate")
+def validate():
+    nodes = Node.select().where(Node.validated == False)
+    for node in nodes:
+        now = datetime.utcnow()
+        logging.info(f"Attempting to validate {node.url}")
+        try:
+            r = make_request(node.url)
+            assert "height" in r.json()
+            assert "nettype" in r.json()
+            has_cors = "Access-Control-Allow-Origin" in r.headers
+            is_ssl = node.url.startswith("https://")
+            nettype = r.json()["nettype"]
+            crypto = determine_crypto(node.url)
+            logging.info("success")
+            if nettype in ["mainnet", "stagenet", "testnet"]:
+                node.nettype = nettype
+                node.available = True
+                node.validated = True
+                node.web_compatible = has_cors and is_ssl
+                node.last_height = r.json()["height"]
+                node.datetime_checked = now
+                node.crypto = crypto
+                node.is_tor = is_onion(node.url)
+                node.save()
+            else:
+                logging.info("unexpected nettype")
+        except requests.exceptions.ConnectTimeout:
+            logging.info("connection timed out")
+            node.delete_instance()
+        except requests.exceptions.SSLError:
+            logging.info("invalid certificate")
+            node.delete_instance()
+        except requests.exceptions.ConnectionError:
+            logging.info("connection error")
+            node.delete_instance()
+        except requests.exceptions.HTTPError:
+            logging.info("http error, 4xx or 5xx")
+            node.delete_instance()
+        except Exception as e:
+            logging.info("failed for reasons unknown")
+            node.delete_instance()
+
+
+@bp.cli.command("export")
+def export():
+    all_nodes = []
+    ts = int(arrow.get().timestamp())
+    export_dir = f"{config.DATA_DIR}/export.txt"
+    export_dir_stamped = f"{config.DATA_DIR}/export-{ts}.txt"
+    nodes = Node.select().where(Node.validated == True)
+    for node in nodes:
+        logging.info(f"Adding {node.url}")
+        all_nodes.append(node.url)
+    with open(export_dir, "w") as f:
+        f.write("\n".join(all_nodes))
+    with open(export_dir_stamped, "w") as f:
+        f.write("\n".join(all_nodes))
+    logging.info(
+        f"{nodes.count()} nodes written to {export_dir} and {export_dir_stamped}"
+    )
+
+
+@bp.cli.command("import")
+def import_():
+    all_nodes = []
+    export_dir = f"{config.DATA_DIR}/export.txt"
+    with open(export_dir, "r") as f:
+        for url in f.readlines():
+            try:
+                n = url.rstrip().lower()
+                logging.info(f"Adding {n}")
+                node = Node(url=n)
+                node.save()
+                all_nodes.append(n)
+            except:
+                pass
+    logging.info(f"{len(all_nodes)} node urls imported and ready to be validated")

xmrnodes/config.example.py

@@ -1,8 +0,0 @@
-import os
-
-SECRET_KEY = os.environ.get('SECRET_KEY', 'xxxx')
-DATA_DIR = os.environ.get('DATA_DIR', './data')
-TOR_HOST = os.environ.get('TOR_HOST', '127.0.0.1')
-TOR_PORT = os.environ.get('TOR_PORT', 9050)
-NODE_HOST = os.environ.get('NODE_HOST', '127.0.0.1')
-NODE_PORT = os.environ.get('NODE_PORT', 18080)

xmrnodes/config.py

@@ -1,14 +1,15 @@
-import os
+from os import environ
 from secrets import token_urlsafe
 
 from dotenv import load_dotenv
 
 load_dotenv()
 
-SECRET_KEY = os.environ.get('SECRET_KEY', token_urlsafe(14))
+SECRET_KEY = environ.get("SECRET_KEY", token_urlsafe(14))
-SERVER_NAME = os.environ.get('SERVER_NAME', '127.0.0.1:5000')
+SERVER_NAME = environ.get("SERVER_NAME", "127.0.0.1:5000")
-DATA_DIR = os.environ.get('DATA_DIR', './data')
+DATA_DIR = environ.get("DATA_DIR", "./data")
-TOR_HOST = os.environ.get('TOR_HOST', '127.0.0.1')
+TOR_HOST = environ.get("TOR_HOST", "127.0.0.1")
-TOR_PORT = os.environ.get('TOR_PORT', 9050)
+TOR_PORT = environ.get("TOR_PORT", 9050)
-NODE_HOST = os.environ.get('NODE_HOST', 'singapore.node.xmr.pm')
+NODE_HOST = environ.get("NODE_HOST", "singapore.node.xmr.pm")
-NODE_PORT = os.environ.get('NODE_PORT', 18080)
+NODE_PORT = environ.get("NODE_PORT", 18080)
+HEALTHY_BLOCK_DIFF = int(environ.get("HEALTHY_BLOCK_DIFF", 500))

xmrnodes/filters.py

@@ -0,0 +1,27 @@
+from datetime import datetime
+
+import arrow
+from flask import Blueprint
+from urllib.parse import urlencode
+
+bp = Blueprint("filters", "filters")
+
+
+@bp.app_template_filter("humanize")
+def humanize(d):
+    t = arrow.get(d, "UTC")
+    return t.humanize()
+
+
+@bp.app_template_filter("hours_elapsed")
+def hours_elapsed(d):
+    now = datetime.utcnow()
+    diff = now - d
+    return diff.total_seconds() / 60 / 60
+
+
+@bp.app_template_filter("pop_arg")
+def trim_arg(all_args, arg_to_trim):
+    d = all_args.to_dict()
+    d.pop(arg_to_trim)
+    return urlencode(d)

xmrnodes/forms.py

@@ -4,4 +4,8 @@ from wtforms.validators import DataRequired
 
 
 class SubmitNode(FlaskForm):
-    node_url = StringField('', validators=[DataRequired()], render_kw={"placeholder": "Node URL (http://xxx.tld:18081)"})
+    node_url = StringField(
+        "",
+        validators=[DataRequired()],
+        render_kw={"placeholder": "Node URL (http://xxx.tld:18081)"},
+    )

xmrnodes/helpers.py

@@ -22,22 +22,30 @@ def make_request(url: str, path="/get_info", data=None):
     else:
         proxies = None
         timeout = 10
-    r = r_get(url + path, timeout=timeout, proxies=proxies, json=data, headers=headers, verify=False)
+    r = r_get(
+        url + path,
+        timeout=timeout,
+        proxies=proxies,
+        json=data,
+        headers=headers,
+        verify=False,
+    )
     r.raise_for_status()
     return r
 
+
 def determine_crypto(url):
     data = {"method": "get_block_header_by_height", "params": {"height": 0}}
     hashes = {
         "monero": [
-            "418015bb9ae982a1975da7d79277c2705727a56894ba0fb246adaabb1f4632e3", #mainnet
+            "418015bb9ae982a1975da7d79277c2705727a56894ba0fb246adaabb1f4632e3",  # mainnet
-            "48ca7cd3c8de5b6a4d53d2861fbdaedca141553559f9be9520068053cda8430b", #testnet
+            "48ca7cd3c8de5b6a4d53d2861fbdaedca141553559f9be9520068053cda8430b",  # testnet
-            "76ee3cc98646292206cd3e86f74d88b4dcc1d937088645e9b0cbca84b7ce74eb"  #stagenet
+            "76ee3cc98646292206cd3e86f74d88b4dcc1d937088645e9b0cbca84b7ce74eb",  # stagenet
         ],
         "wownero": [
-            "a3fd635dd5cb55700317783469ba749b5259f0eeac2420ab2c27eb3ff5ffdc5c", #mainnet
+            "a3fd635dd5cb55700317783469ba749b5259f0eeac2420ab2c27eb3ff5ffdc5c",  # mainnet
-            "d81a24c7aad4628e5c9129f8f2ec85888885b28cf468597a9762c3945e9f29aa", #testnet
+            "d81a24c7aad4628e5c9129f8f2ec85888885b28cf468597a9762c3945e9f29aa",  # testnet
-        ]
+        ],
     }
     try:
         r = make_request(url, "/json_rpc", data)
@@ -52,6 +60,7 @@ def determine_crypto(url):
     except:
         return "unknown"
 
+
 def is_onion(url: str):
     _split = url.split(":")
     if len(_split) < 2:
@@ -61,21 +70,23 @@ def is_onion(url: str):
     else:
         return False
 
+
 # Use hacky filesystem cache since i dont feel like shipping redis
 def rw_cache(key_name, data=None):
-    pickle_file = path.join(config.DATA_DIR, f'{key_name}.pkl')
+    pickle_file = path.join(config.DATA_DIR, f"{key_name}.pkl")
     if data:
-        with open(pickle_file, 'wb') as f:
+        with open(pickle_file, "wb") as f:
             f.write(pickle.dumps(data))
             return data
     else:
-        with open(pickle_file, 'rb') as f:
+        with open(pickle_file, "rb") as f:
             pickled_data = pickle.load(f)
             return pickled_data
 
+
 def retrieve_peers(host, port):
     try:
-        print(f'[.] Connecting to {host}:{port}')
+        print(f"[.] Connecting to {host}:{port}")
         sock = socket.socket()
         sock.settimeout(5)
         sock.connect((host, int(port)))
@@ -109,7 +120,7 @@ def retrieve_peers(host, port):
 
             for peer in _peers:
                 try:
-                    peers.append('http://%s:%d' % (peer['ip'].ip, peer['port'].value))
+                    peers.append("http://%s:%d" % (peer["ip"].ip, peer["port"].value))
                 except:
                     pass
 
@@ -121,12 +132,15 @@ def retrieve_peers(host, port):
     else:
         return None
 
+
 def get_highest_block(nettype, crypto):
-    highest = Node.select().where(
+    highest = (
-        Node.validated == True,
+        Node.select()
-        Node.nettype == nettype,
+        .where(Node.validated == True, Node.nettype == nettype, Node.crypto == crypto)
-        Node.crypto == crypto
+        .order_by(Node.last_height.desc())
-    ).order_by(Node.last_height.desc()).limit(1).first()
+        .limit(1)
+        .first()
+    )
     if highest:
         return highest.last_height
     else:

xmrnodes/models.py

@@ -9,6 +9,7 @@ from xmrnodes import config
 
 db = SqliteQueueDatabase(f"{config.DATA_DIR}/sqlite.db")
 
+
 class Node(Model):
     id = AutoField()
     url = CharField(unique=True)
@@ -29,7 +30,9 @@ class Node(Model):
         return _url.netloc
 
     def get_failed_checks(self):
-        hcs = HealthCheck.select().where(HealthCheck.node == self, HealthCheck.health == False)
+        hcs = HealthCheck.select().where(
+            HealthCheck.node == self, HealthCheck.health == False
+        )
         return hcs
 
     def get_all_checks(self):
@@ -39,6 +42,7 @@ class Node(Model):
     class Meta:
         database = db
 
+
 class Peer(Model):
     id = AutoField()
     url = CharField(unique=True)
@@ -60,13 +64,15 @@ class Peer(Model):
     class Meta:
         database = db
 
+
 class HealthCheck(Model):
     id = AutoField()
-    node = ForeignKeyField(Node, backref='healthchecks')
+    node = ForeignKeyField(Node, backref="healthchecks")
     datetime = DateTimeField(default=datetime.utcnow)
     health = BooleanField()
 
     class Meta:
         database = db
 
+
 db.create_tables([Node, HealthCheck, Peer])

xmrnodes/routes/api.py

@@ -0,0 +1,71 @@
+from flask import jsonify, Blueprint
+
+from xmrnodes.models import Node
+
+
+bp = Blueprint('api', 'api')
+
+@bp.route("/nodes.json")
+def nodes_json():
+    nodes = Node.select().where(
+        Node.validated==True
+    ).where(
+        Node.nettype=="mainnet"
+    )
+    xmr_nodes = [n for n in nodes if n.crypto == "monero"]
+    wow_nodes = [n for n in nodes if n.crypto == "wownero"]
+    return jsonify({
+        "monero": {
+            "clear": [n.url for n in xmr_nodes if n.is_tor == False],
+            "onion": [n.url for n in xmr_nodes if n.is_tor == True],
+            "web_compatible": [n.url for n in xmr_nodes if n.web_compatible == True],
+        },
+        "wownero": {
+            "clear": [n.url for n in wow_nodes if n.is_tor == False],
+            "onion": [n.url for n in wow_nodes if n.is_tor == True]
+        }
+    })
+
+@bp.route("/health.json")
+def health_json():
+    data = {}
+    nodes = Node.select().where(
+        Node.validated == True
+    )
+    for node in nodes:
+        if node.crypto not in data:
+            data[node.crypto] = {}
+        _d = {
+            "available": node.available,
+            "last_height": node.last_height,
+            "datetime_entered": node.datetime_entered,
+            "datetime_checked": node.datetime_checked,
+            "datetime_failed": node.datetime_failed,
+            "checks": [c.health for c in node.get_all_checks()]
+        }
+        nettype = "clear"
+        if node.is_tor:
+            nettype = "onion"
+        elif node.web_compatible:
+            if "web_compatible" not in data[node.crypto]:
+                data[node.crypto]["web_compatible"] = {}
+            data[node.crypto]["web_compatible"][node.url] = _d
+        if nettype not in data[node.crypto]:
+            data[node.crypto][nettype] = {}
+        data[node.crypto][nettype][node.url] = _d
+    return jsonify(data)
+
+@bp.route("/wow_nodes.json")
+def wow_nodes_json():
+    nodes = Node.select().where(
+        Node.validated==True
+    ).where(
+        Node.nettype=="mainnet"
+    ).where(
+        Node.crypto=="wownero"
+    )
+    nodes = [n for n in nodes]
+    return jsonify({
+        "clear": [n.url for n in nodes if n.is_tor == False],
+        "onion": [n.url for n in nodes if n.is_tor == True]
+    })

xmrnodes/routes/meta.py

@@ -0,0 +1,124 @@
+import re
+from random import shuffle
+
+from flask import request, redirect, Blueprint
+from flask import render_template, flash, Response
+from urllib.parse import urlparse
+
+from xmrnodes.helpers import rw_cache, get_highest_block
+from xmrnodes.forms import SubmitNode
+from xmrnodes.models import Node
+from xmrnodes import config
+
+bp = Blueprint("meta", "meta")
+
+
+@bp.route("/", methods=["GET", "POST"])
+def index():
+    form = SubmitNode()
+    nettype = request.args.get("network", "mainnet")
+    crypto = request.args.get("chain", "monero")
+    onion = request.args.get("onion", False)
+    show_all = "true" == request.args.get("all", "false")
+    web_compatible = request.args.get("cors", False)
+    highest_block = get_highest_block(nettype, crypto)
+    healthy_block = highest_block - config.HEALTHY_BLOCK_DIFF
+
+    nodes = Node.select().where(
+        Node.validated == True, Node.nettype == nettype, Node.crypto == crypto
+    )
+
+    if web_compatible:
+        nodes = nodes.where(Node.web_compatible == True)
+
+    nodes_all = nodes.count()
+    nodes_unhealthy = nodes.where(
+        (Node.available == False) | (Node.last_height < healthy_block)
+    ).count()
+
+    if not show_all:
+        nodes = nodes.where(Node.available == True, Node.last_height > healthy_block)
+
+    nodes = nodes.order_by(Node.datetime_entered.desc())
+    if onion:
+        nodes = nodes.where(Node.is_tor == True)
+
+    nodes = [n for n in nodes]
+    shuffle(nodes)
+
+    return render_template(
+        "index.html",
+        nodes=nodes,
+        nodes_all=nodes_all,
+        nodes_unhealthy=nodes_unhealthy,
+        nettype=nettype,
+        crypto=crypto,
+        form=form,
+        web_compatible=web_compatible,
+    )
+
+
+@bp.route("/map")
+def map():
+    try:
+        peers = rw_cache("map_peers")
+    except:
+        flash("Couldn't load the map. Try again later.")
+        return redirect("/")
+    return render_template("map.html", peers=peers, source_node=config.NODE_HOST)
+
+
+@bp.route("/resources")
+def resources():
+    return render_template("resources.html")
+
+
+@bp.route("/add", methods=["GET", "POST"])
+def add():
+    if request.method == "POST":
+        url = request.form.get("node_url")
+        regex = re.compile(
+            r"^(?:http)s?://"  # http:// or https://
+            r"(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|"  # domain...
+            r"localhost|"  # localhost...
+            r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})"  # ...or ip
+            r"(?::\d+)?"  # optional port
+            r"(?:/?|[/?]\S+)$",
+            re.IGNORECASE,
+        )
+        re_match = re.match(regex, url)
+        if re_match is None:
+            flash("This doesn't look like a valid URL")
+        else:
+            _url = urlparse(url)
+            url = f"{_url.scheme}://{_url.netloc}".lower()
+            if Node.select().where(Node.url == url).exists():
+                flash("This node is already in the database.")
+            else:
+                flash(
+                    "Seems like a valid node URL. Added to the database and will check soon."
+                )
+                node = Node(url=url)
+                node.save()
+    return redirect("/")
+
+
+@bp.route("/haproxy.cfg")
+def haproxy():
+    crypto = request.args.get("chain") or "monero"
+    nettype = request.args.get("network") or "mainnet"
+    cors = request.args.get("cors") or False
+    tor = request.args.get("onion") or False
+    nodes = Node.select().where(
+        Node.validated == True,
+        Node.nettype == nettype,
+        Node.crypto == crypto,
+        Node.is_tor == tor,
+        Node.web_compatible == cors,
+    )
+    tpl = render_template("haproxy.html", nodes=nodes)
+    res = Response(tpl)
+    res.headers[
+        "Content-Disposition"
+    ] = f'attachment; filename="haproxy-{crypto}-{nettype}-cors_{cors}-tor_{tor}.cfg"'
+    return res

xmrnodes/templates/base.html

@@ -43,13 +43,15 @@
 
 <div id="" class="center">
   <br>
-  <a href="https://lzahq.tech" target="_blank">Contact me</a>
+  <a href="https://twitter.com/lza_menace" target="_blank">contact</a>
   -
-  <a href="{{ url_for('map') }}">Map</a>
+  <a href="{{ url_for('meta.map') }}">map</a>
   -
-  <a href="https://github.com/lalanza808/monero.fail" target="_blank">Source Code</a>
+  <a href="https://github.com/lalanza808/monero.fail" target="_blank">source</a>
   -
-  <a href="{{ url_for('resources') }}">Resources</a>
+  <a href="{{ url_for('meta.resources') }}">about</a>
+  -
+  <a href="https://moneroinfodump.neocities.org/" target="_blank">infodump</a>
 </div>
 
  <!-- Page Content -->

xmrnodes/templates/index.html

@@ -9,7 +9,7 @@
     <div class="title pure-u-1">
       <h2>Add A Node</h2>
     </div>
-    <form method="POST" action="{{ url_for('add') }}" class="pure-form pure-u-1">
+    <form method="POST" action="{{ url_for('meta.add') }}" class="pure-form pure-u-1">
         {{ form.csrf_token }}
         {% for f in form %}
           {% if f.name != 'csrf_token' %}
@@ -80,7 +80,7 @@
         </strong>
         <br><br>
         {% endif %}
-        <a href="{{ url_for('haproxy', chain=request.args.get('chain'), network=request.args.get('network'), cors=request.args.get('cors'), onion=request.args.get('onion')) }}">Download HAProxy config</a><br /><br />
+        <a href="{{ url_for('meta.haproxy', chain=request.args.get('chain'), network=request.args.get('network'), cors=request.args.get('cors'), onion=request.args.get('onion')) }}">Download HAProxy config</a><br /><br />
         Tracking {{ nodes_all }} {{ nettype }} {{ crypto | capitalize }} nodes in the database.
         <br>
         Of those, {{ nodes_unhealthy }} nodes failed their last check-in (unresponsive to ping or over 500 blocks away from highest reported block).
@@ -98,7 +98,7 @@
           <tr>
             <th class="js-sort-string">URL</th>
             <th class="js-sort-number">Height</th>
-            <th class="js-sort-none">Available</th>
+            <th class="js-sort-none">Up</th>
             <th class="js-sort-none">Web<br/>Compatible</th>
             <th class="js-sort-none">Network</th>
             <th class="js-sort-none">Date Added</th>

xmrnodes/templates/map.html

@@ -4,18 +4,15 @@
      <meta charset="UTF-8">
      <meta http-equiv="X-UA-Compatible" content="IE=edge">
      <meta name="viewport" content="width=device-width, initial-scale=1">
-
      <meta name="HandheldFriendly" content="True">
      <meta name="MobileOptimized" content="320">
      <link rel="shortcut icon" href="/static/favicon.ico" type="image/x-icon" />
-
      <meta property="fb:app_id" content="0" />
      <meta property="og:image" content="https://www.getmonero.org/press-kit/symbols/monero-symbol-on-white-480.png" />
      <meta property="og:description" content="xmrnodes" />
      <meta property="og:url" content="http://localhost" />
      <meta property="og:title" content="XMR Nodes" />
      <meta property="og:type" content="website" />
-
      <meta name="viewport" content="width=device-width, initial-scale=1"/>
      <meta name="theme-color" content="#ffffff">
      <meta name="apple-mobile-web-app-title" content="XMR Nodes">
@@ -26,8 +23,8 @@
      <link rel="stylesheet" href="//maxcdn.bootstrapcdn.com/bootstrap/4.5.0/css/bootstrap.min.css" type="text/css">
      <style>
        .map {
-         height: 750px;
+         height: 600px;
-         margin: 0;
+         margin: 2em;
          padding: 0;
        }
        .popover-body {
@@ -69,12 +66,6 @@
   </p>
   <br>
   <a href="/">Go home</a>
-  -
-  <a href="https://twitter.com/lza_menace" target="_blank">Contact me</a>
-  -
-  <a href="https://github.com/lalanza808/monero.fail" target="_blank">Source Code</a>
-  -
-  <a href="{{ url_for('resources') }}">Resources</a>
 </div>
 <div id="map" class="map"></div>
 <div id="popup" class="popup" title="Welcome to OpenLayers"></div>

xmrnodes/templates/resources.html

@@ -12,14 +12,12 @@
       <h3>How Do I Use Monero?</h3>
       <p>
         In order to send and receive Monero you need to have the software installed on either your phone or computer.
-        Try out these popular Monero projects:
+        Try out these solid open-source Monero projects:
       </p>
       <ul>
         <li><a href="https://www.getmonero.org/downloads/" target="_blank">Official Community Funded Monero Software (desktop)</a></li>
         <li><a href="https://featherwallet.org/" target="_blank">Feather (desktop)</a></li>
-        <li><a href="https://cakewallet.com/" target="_blank">Cake Wallet (iOS, Android)</a></li>
         <li><a href="https://www.monerujo.io/" target="_blank">Monerujo (Android)</a></li>
-        <li><a href="https://mymonero.com/" target="_blank">MyMonero (Desktop, Web)</a></li>
       </ul>
 
       <h3>How Do I Get Monero?</h3>