252 lines
8.8 KiB
Python
Raw Normal View History

2023-04-04 00:00:37 -07:00
import logging
from datetime import datetime, timedelta
2023-04-15 12:25:48 -07:00
from time import sleep
2023-04-04 00:00:37 -07:00
import geoip2.database
import arrow
import requests
from flask import Blueprint
from urllib.parse import urlparse
from xmrnodes.helpers import determine_crypto, is_onion, is_i2p, make_request
2023-06-08 19:05:52 -07:00
from xmrnodes.helpers import retrieve_peers, rw_cache, get_highest_block, get_geoip
2023-04-04 00:00:37 -07:00
from xmrnodes.models import Node, HealthCheck, Peer
from xmrnodes import config
bp = Blueprint("cli", "cli", cli_group=None)
@bp.cli.command("init")
def init():
pass
@bp.cli.command("check")
2023-05-02 10:25:01 -07:00
def check_nodes():
diff = datetime.utcnow() - timedelta(hours=72)
2023-04-04 00:00:37 -07:00
checks = HealthCheck.select().where(HealthCheck.datetime <= diff)
for check in checks:
print("Deleting check", check.id)
check.delete_instance()
nodes = Node.select().where(Node.validated == True)
for node in nodes:
try:
2023-05-02 10:25:01 -07:00
check_node(node.url)
except KeyboardInterrupt:
exit()
def check_node(_node):
if _node.startswith("http"):
node = Node.select().where(Node.url == _node).first()
else:
node = Node.select().where(Node.id == _node).first()
if not node:
print('node found')
pass
now = datetime.utcnow()
hc = HealthCheck(node=node)
logging.info(f"Attempting to check {node.url}")
try:
r = make_request(node.url)
assert "status" in r.json()
assert "offline" in r.json()
assert "height" in r.json()
if 'donation_address' in r.json():
node.donation_address = r.json()['donation_address']
has_cors = "Access-Control-Allow-Origin" in r.headers
is_ssl = node.url.startswith("https://")
if r.json()["status"] == "OK":
node.web_compatible = has_cors and is_ssl
node.last_height = r.json()["height"]
hc.health = True
highest_block = get_highest_block(node.nettype, node.crypto)
healthy_block = highest_block - config.HEALTHY_BLOCK_DIFF
if r.json()["height"] < healthy_block:
node.available = False
logging.info("unhealthy")
2023-04-04 00:00:37 -07:00
else:
2023-05-02 10:25:01 -07:00
node.available = True
logging.info("success")
else:
raise
except:
logging.info("fail")
node.datetime_failed = now
node.available = False
hc.health = False
finally:
node.datetime_checked = now
node.save()
hc.save()
if (
node.get_failed_checks().count() == node.get_all_checks().count()
and node.get_all_checks().count() > 5
):
print("this node fails all of its health checks - deleting it!")
for _hc in node.get_all_checks():
_hc.delete_instance()
node.delete_instance()
2023-04-04 00:00:37 -07:00
@bp.cli.command("get_peers")
def get_peers():
2023-04-15 12:25:48 -07:00
"""
This command requests peers from the configured upstream node and fans out
to recursively scrape all other peers on the network. This will take
several hours to run.
"""
# keep track of all peers
2023-04-04 00:00:37 -07:00
all_peers = []
print("[+] Preparing to crawl Monero p2p network")
print(f"[.] Retrieving initial peers from {config.NODE_HOST}:{config.NODE_PORT}")
2023-04-15 12:25:48 -07:00
# start initial list of peers to scan
peers_to_scan = retrieve_peers(config.NODE_HOST, config.NODE_PORT)
print(f"[+] Found {len(peers_to_scan)} initial peers to begin scraping.")
sleep(3)
# helper function to add a new peer to the db or update an existing one
def save_peer(peer):
with geoip2.database.Reader("./data/GeoLite2-City.mmdb") as reader:
2023-04-04 00:00:37 -07:00
_url = urlparse(peer)
url = f"{_url.scheme}://{_url.netloc}".lower()
2023-04-15 12:25:48 -07:00
# add new peer if not in db
2023-04-04 00:00:37 -07:00
if not Peer.select().where(Peer.url == peer).exists():
response = reader.city(_url.hostname)
p = Peer(
url=peer,
country=response.country.name,
city=response.city.name,
postal=response.postal.code,
lat=response.location.latitude,
lon=response.location.longitude,
)
p.save()
print(f"{peer} - saving new peer")
2023-04-15 12:25:48 -07:00
# or update if it does
2023-04-04 00:00:37 -07:00
else:
p = Peer.select().where(Peer.url == peer).first()
p.datetime = datetime.now()
p.save()
2023-04-15 12:25:48 -07:00
return _url
# iterate over the whole list until all peers have been scanned
# add new peers to the list
# skip the peer if we've seen it already
try:
while peers_to_scan:
_peer = peers_to_scan[0]
peers_to_scan.pop(0)
if _peer in all_peers:
print(f'already found {_peer}')
continue
all_peers.append(_peer)
2023-04-04 00:00:37 -07:00
try:
2023-04-15 12:25:48 -07:00
peer = save_peer(_peer)
peers_to_scan += retrieve_peers(peer.hostname, peer.port)
2023-04-04 00:00:37 -07:00
except:
pass
2023-04-15 12:25:48 -07:00
except KeyboardInterrupt:
print('Stopped.')
2023-04-04 00:00:37 -07:00
print(
f"[+] Found {len(all_peers)} peers from {config.NODE_HOST}:{config.NODE_PORT}"
)
print("[+] Deleting old Monero p2p peers")
for p in Peer.select():
2023-04-15 12:25:48 -07:00
if p.hours_elapsed() > config.PEER_LIFETIME:
2023-04-04 00:00:37 -07:00
print(f"[.] Deleting {p.url}")
p.delete_instance()
rw_cache("map_peers", list(Peer.select().execute()))
@bp.cli.command("validate")
def validate():
nodes = Node.select().where(Node.validated == False)
for node in nodes:
now = datetime.utcnow()
logging.info(f"Attempting to validate {node.url}")
try:
r = make_request(node.url)
assert "height" in r.json()
assert "nettype" in r.json()
has_cors = "Access-Control-Allow-Origin" in r.headers
is_ssl = node.url.startswith("https://")
nettype = r.json()["nettype"]
crypto = determine_crypto(node.url)
if nettype in ["mainnet", "stagenet", "testnet"]:
node.nettype = nettype
node.available = True
node.validated = True
node.web_compatible = has_cors and is_ssl
node.last_height = r.json()["height"]
node.datetime_checked = now
node.crypto = crypto
node.is_tor = is_onion(node.url)
node.is_i2p = is_i2p(node.url)
2023-06-08 19:05:52 -07:00
if not node.is_tor and not node.is_i2p:
geoip = get_geoip(node.url)
node.country_name = geoip.country.name
node.country_code = geoip.country.iso_code
node.city = geoip.city.name
node.postal = geoip.postal.code
node.lat = geoip.location.latitude
node.lon = geoip.location.longitude
logging.info(f"found geo data for {node.url} - {node.country_code}, {node.country_name}, {node.city}")
2023-04-04 00:00:37 -07:00
node.save()
2023-06-08 19:05:52 -07:00
logging.info("success")
2023-04-04 00:00:37 -07:00
else:
logging.info("unexpected nettype")
except requests.exceptions.ConnectTimeout:
logging.info("connection timed out")
node.delete_instance()
except requests.exceptions.SSLError:
logging.info("invalid certificate")
node.delete_instance()
except requests.exceptions.ConnectionError:
logging.info("connection error")
node.delete_instance()
except requests.exceptions.HTTPError:
logging.info("http error, 4xx or 5xx")
node.delete_instance()
except Exception as e:
2023-06-08 19:05:52 -07:00
logging.info(f"failed for reasons unknown: {e}")
2023-04-04 00:00:37 -07:00
node.delete_instance()
@bp.cli.command("export")
def export():
all_nodes = []
ts = int(arrow.get().timestamp())
export_dir = f"{config.DATA_DIR}/export.txt"
export_dir_stamped = f"{config.DATA_DIR}/export-{ts}.txt"
nodes = Node.select().where(Node.validated == True)
for node in nodes:
logging.info(f"Adding {node.url}")
all_nodes.append(node.url)
with open(export_dir, "w") as f:
f.write("\n".join(all_nodes))
with open(export_dir_stamped, "w") as f:
f.write("\n".join(all_nodes))
logging.info(
f"{nodes.count()} nodes written to {export_dir} and {export_dir_stamped}"
)
@bp.cli.command("import")
def import_():
all_nodes = []
export_dir = f"{config.DATA_DIR}/export.txt"
with open(export_dir, "r") as f:
for url in f.readlines():
try:
n = url.rstrip().lower()
logging.info(f"Adding {n}")
node = Node(url=n)
node.save()
all_nodes.append(n)
except:
pass
logging.info(f"{len(all_nodes)} node urls imported and ready to be validated")