Allow Synapse to send registration emails + choose Synapse or an external server to handle 3pid validation (#5987)

This is a combination of a few different PRs, finally all being merged into `develop`:

* #5875 
* #5876 
* #5868 (This one added the `/versions` flag but the flag itself was actually [backed out](891afb57cb (diff-e591d42d30690ffb79f63bb726200891)) in #5969. What's left is just giving /versions access to the config file, which could be useful in the future)
* #5835 
* #5969 
* #5940

Clients should not actually use the new registration functionality until https://github.com/matrix-org/synapse/pull/5972 is merged.

UPGRADE.rst, changelog entries and config file changes should all be reviewed closely before this PR is merged.

synapse/push/mailer.py

@@ -131,14 +131,11 @@ class Mailer(object):
             email_address (str): Email address we're sending the password
                 reset to
             token (str): Unique token generated by the server to verify
-                password reset email was received
+                the email was received
             client_secret (str): Unique token generated by the client to
                 group together multiple email sending attempts
             sid (str): The generated session ID
         """
-        if email.utils.parseaddr(email_address)[1] == "":
-            raise RuntimeError("Invalid 'to' email address")
-
         link = (
             self.hs.config.public_baseurl
             + "_matrix/client/unstable/password_reset/email/submit_token"
@@ -149,7 +146,34 @@ class Mailer(object):
 
         yield self.send_email(
             email_address,
-            "[%s] Password Reset Email" % self.hs.config.server_name,
+            "[%s] Password Reset" % self.hs.config.server_name,
+            template_vars,
+        )
+
+    @defer.inlineCallbacks
+    def send_registration_mail(self, email_address, token, client_secret, sid):
+        """Send an email with a registration confirmation link to a user
+
+        Args:
+            email_address (str): Email address we're sending the registration
+                link to
+            token (str): Unique token generated by the server to verify
+                the email was received
+            client_secret (str): Unique token generated by the client to
+                group together multiple email sending attempts
+            sid (str): The generated session ID
+        """
+        link = (
+            self.hs.config.public_baseurl
+            + "_matrix/client/unstable/registration/email/submit_token"
+            "?token=%s&client_secret=%s&sid=%s" % (token, client_secret, sid)
+        )
+
+        template_vars = {"link": link}
+
+        yield self.send_email(
+            email_address,
+            "[%s] Register your Email Address" % self.hs.config.server_name,
             template_vars,
         )
 
@@ -605,25 +629,50 @@ def format_ts_filter(value, format):
     return time.strftime(format, time.localtime(value / 1000))
 
 
-def load_jinja2_templates(config, template_html_name, template_text_name):
-    """Load the jinja2 email templates from disk
+def load_jinja2_templates(
+    template_dir,
+    template_filenames,
+    apply_format_ts_filter=False,
+    apply_mxc_to_http_filter=False,
+    public_baseurl=None,
+):
+    """Loads and returns one or more jinja2 templates and applies optional filters
+
+    Args:
+        template_dir (str): The directory where templates are stored
+        template_filenames (list[str]): A list of template filenames
+        apply_format_ts_filter (bool): Whether to apply a template filter that formats
+            timestamps
+        apply_mxc_to_http_filter (bool): Whether to apply a template filter that converts
+            mxc urls to http urls
+        public_baseurl (str|None): The public baseurl of the server. Required for
+            apply_mxc_to_http_filter to be enabled
 
     Returns:
-        (template_html, template_text)
+        A list of jinja2 templates corresponding to the given list of filenames,
+        with order preserved
     """
-    logger.info("loading email templates from '%s'", config.email_template_dir)
-    loader = jinja2.FileSystemLoader(config.email_template_dir)
+    logger.info(
+        "loading email templates %s from '%s'", template_filenames, template_dir
+    )
+    loader = jinja2.FileSystemLoader(template_dir)
     env = jinja2.Environment(loader=loader)
-    env.filters["format_ts"] = format_ts_filter
-    env.filters["mxc_to_http"] = _create_mxc_to_http_filter(config)
 
-    template_html = env.get_template(template_html_name)
-    template_text = env.get_template(template_text_name)
+    if apply_format_ts_filter:
+        env.filters["format_ts"] = format_ts_filter
 
-    return template_html, template_text
+    if apply_mxc_to_http_filter and public_baseurl:
+        env.filters["mxc_to_http"] = _create_mxc_to_http_filter(public_baseurl)
+
+    templates = []
+    for template_filename in template_filenames:
+        template = env.get_template(template_filename)
+        templates.append(template)
+
+    return templates
 
 
-def _create_mxc_to_http_filter(config):
+def _create_mxc_to_http_filter(public_baseurl):
     def mxc_to_http_filter(value, width, height, resize_method="crop"):
         if value[0:6] != "mxc://":
             return ""
@@ -636,7 +685,7 @@ def _create_mxc_to_http_filter(config):
 
         params = {"width": width, "height": height, "method": resize_method}
         return "%s_matrix/media/v1/thumbnail/%s?%s%s" % (
-            config.public_baseurl,
+            public_baseurl,
             serverAndMediaId,
             urllib.parse.urlencode(params),
             fragment or "",