fix forms and refactor node ingest/validation

bin/cmd

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 source .venv/bin/activate
-export FLASK_APP=nodes/app.py
+export FLASK_APP=xmrnodes/app.py
 export FLASK_SECRETS=config.py
 export FLASK_DEBUG=1
 flask $1

xmrnodes/app.py

@@ -1,19 +1,28 @@
 import json
 import requests
 import re
+import logging
 from os import makedirs
+from datetime import datetime
 from flask import Flask, request, redirect
 from flask import render_template, flash, url_for
 from urllib.parse import urlparse
+from xmrnodes.forms import SubmitNode
 from xmrnodes.models import Node
 
 
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s'
+)
+
 app = Flask(__name__)
 app.config.from_envvar("FLASK_SECRETS")
 app.secret_key = app.config["SECRET_KEY"]
 
 @app.route("/", methods=["GET", "POST"])
 def index():
+    form = SubmitNode()
     itp = 20
     page = request.args.get("page", 1)
     try:
@@ -22,61 +31,85 @@ def index():
         flash("Wow, wtf hackerman. Cool it.")
         page = 1
 
-    nodes = Node.select().where(Node.available==True).order_by(Node.datetime_entered.desc()).paginate(page, itp)
+    nodes = Node.select().where(Node.available==True).order_by(
+        Node.datetime_entered.desc()
+    ).paginate(page, itp)
     total_pages = Node.select().count() / itp
-    return render_template("index.html", nodes=nodes, page=page, total_pages=total_pages)
+    return render_template(
+        "index.html",
+        nodes=nodes,
+        page=page,
+        total_pages=total_pages,
+        form=form
+    )
 
 
 @app.route("/add", methods=["GET", "POST"])
 def add():
     if request.method == "POST":
-        url = request.form.get("url")
+        url = request.form.get("node_url")
         regex = re.compile(
-                r'^(?:http)s?://' # http:// or https://
-                r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
-                r'localhost|' #localhost...
-                r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
-                r'(?::\d+)?' # optional port
-                r'(?:/?|[/?]\S+)$', re.IGNORECASE)
-
-        re_match = re.match(regex, url)
-
-        if re_match is not None:
-            _url = urlparse(url)
-            try:
-                endpoint = f"{_url.scheme}://{_url.netloc}"
-                r = requests.get(endpoint + "/get_info", timeout=3)
-                r.raise_for_status()
-                # print(r.json())
-                return {"status": "success"}
-            except requests.exceptions.ConnectTimeout:
-                flash("connection timed out. double-check the port")
-                return {"status": "fail", "reason": "timeout"}
-            except requests.exceptions.SSLError:
-                flash("invalid certificate")
-                return {"status": "fail", "reason": "invalid cert"}
-            except Exception as e:
-                flash("failed to send req", str(e))
-                print(e)
-                return {"status": "fail"}
-        else:
-            flash("invalid url provided")
-            return {"status": "fail"}
-
-        return "ok"
-        node = Node(
-            scheme=proto,
-            address=addr,
-            port=port,
-            version=r.json()["version"],
-            tor=addr.endswith(".onion"),
-            available=r.json()["status"] == "OK",
-            mainnet=r.json()["mainnet"],
+            r'^(?:http)s?://' # http:// or https://
+            r'(?:(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?)|' #domain...
+            r'localhost|' #localhost...
+            r'\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})' # ...or ip
+            r'(?::\d+)?' # optional port
+            r'(?:/?|[/?]\S+)$', re.IGNORECASE
         )
-        node.save()
-        return {"status": "success"}
+        re_match = re.match(regex, url)
+        if re_match is None:
+            flash("This doesn't look like a valid URL")
+        else:
+            _url = urlparse(url)
+            url = f"{_url.scheme}://{_url.netloc}"
+            if Node.select().where(Node.url == url).exists():
+                flash("This node is already in the database.")
+            else:
+                flash("Seems like a valid node. Added to the database and will check soon.")
+                node = Node(url=url)
+                node.save()
     return redirect("/")
 
+@app.cli.command("validate")
+def validate():
+    nodes = Node.select().where(Node.validated == False)
+    for node in nodes:
+        now = datetime.now()
+        is_onion = node.url.split(":")[1].endswith(".onion")
+        logging.info(f"Attempting to validate {node.url}")
+        if is_onion:
+            logging.info("onion address found")
+            node.tor = True
+        try:
+            r = requests.get(node.url + "/get_info", timeout=3)
+            r.raise_for_status()
+            assert "height" in r.json()
+            assert "nettype" in r.json()
+            nettype = r.json()["nettype"]
+            logging.info("success")
+            if nettype in ["mainnet", "stagenet", "testnet"]:
+                node.nettype = nettype
+                node.available = True
+                node.validated = True
+                node.datetime_checked = now
+                node.save()
+            else:
+                logging.info("unexpected nettype")
+        except requests.exceptions.ConnectTimeout:
+            logging.info("connection timed out")
+            node.delete_instance()
+        except requests.exceptions.SSLError:
+            logging.info("invalid certificate")
+            node.delete_instance()
+        except requests.exceptions.ConnectionError:
+            logging.info("connection error")
+            node.delete_instance()
+        except requests.exceptions.HTTPError:
+            logging.info("http error, 4xx or 5xx")
+            node.delete_instance()
+        except Exception as e:
+            logging.info("failed for reasons unknown")
+            node.delete_instance()
 
 @app.route("/about")
 def about():

xmrnodes/forms.py

@@ -0,0 +1,7 @@
+from flask_wtf import FlaskForm
+from wtforms import StringField
+from wtforms.validators import DataRequired
+
+
+class SubmitNode(FlaskForm):
+    node_url = StringField('Node URL:', validators=[DataRequired()])

xmrnodes/models.py

@@ -4,19 +4,17 @@ from xmrnodes import config
 
 
 data_dir = getattr(config, 'DATA_FOLDER', './data')
-db = SqliteDatabase(f"{data_dir}/db/sqlite.db")
+db = SqliteDatabase(f"{data_dir}/sqlite.db")
 
 class Node(Model):
     id = AutoField()
-    scheme = CharField()
-    address = CharField()
-    port = IntegerField()
-    version = CharField(null=True)
+    url = CharField()
     tor = BooleanField(default=False)
     available = BooleanField(default=False)
-    mainnet = BooleanField(default=False)
+    validated = BooleanField(default=False)
+    nettype = CharField(null=True)
     datetime_entered = DateTimeField(default=datetime.now)
-    datetime_checked = DateTimeField(default=datetime.now)
+    datetime_checked = DateTimeField(default=None, null=True)
     datetime_failed = DateTimeField(default=None, null=True)
 
     class Meta:

xmrnodes/templates/index.html

@@ -9,13 +9,25 @@
   </div>
 
   {% for node in nodes %}
-  {{ node }}<br>
+  {{ node.url }}<br>
   {% endfor %}
 
-  <form id="addnode" method="POST">
-    <label>Node URL:</label>
-    <input type="text" name="url"/>
-    <input type="submit" value="Submit" name="submit" class="submit" id="submit" />
+  <form method="POST" action="{{ url_for('add') }}">
+      {{ form.csrf_token }}
+      {% for f in form %}
+        {% if f.name != 'csrf_token' %}
+          <div class="form-group">
+            {{ f.label }}
+            {{ f }}
+          </div>
+        {% endif %}
+      {% endfor %}
+      <ul>
+      {% for field, errors in form.errors.items() %}
+          <li>{{ form[field].label }}: {{ ', '.join(errors) }}</li>
+      {% endfor %}
+    </ul>
+      <input type="submit" value="Send" class="btn btn-link btn-outline btn-xl">
   </form>