diff --git a/changelog.d/4992.misc b/changelog.d/4992.misc
index 8a9eaea4c..3ee4228c0 100644
--- a/changelog.d/4992.misc
+++ b/changelog.d/4992.misc
@@ -1 +1 @@
-Remove legacy tables detailed in #1830.
+Remove a number of unused tables from the database schema.
diff --git a/changelog.d/5028.misc b/changelog.d/5028.misc
new file mode 100644
index 000000000..3ee4228c0
--- /dev/null
+++ b/changelog.d/5028.misc
@@ -0,0 +1 @@
+Remove a number of unused tables from the database schema.
diff --git a/synapse/replication/slave/storage/keys.py b/synapse/replication/slave/storage/keys.py
index 8032f53fe..de00660c0 100644
--- a/synapse/replication/slave/storage/keys.py
+++ b/synapse/replication/slave/storage/keys.py
@@ -27,8 +27,5 @@ class SlavedKeyStore(BaseSlavedStore):
     get_server_verify_keys = __func__(DataStore.get_server_verify_keys)
     store_server_verify_key = __func__(DataStore.store_server_verify_key)
 
-    get_server_certificate = __func__(DataStore.get_server_certificate)
-    store_server_certificate = __func__(DataStore.store_server_certificate)
-
     get_server_keys_json = __func__(DataStore.get_server_keys_json)
     store_server_keys_json = __func__(DataStore.store_server_keys_json)
diff --git a/synapse/storage/keys.py b/synapse/storage/keys.py
index f24ab3eed..47a9aa784 100644
--- a/synapse/storage/keys.py
+++ b/synapse/storage/keys.py
@@ -13,14 +13,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import hashlib
 import logging
 
 import six
 
 from signedjson.key import decode_verify_key_bytes
 
-import OpenSSL
 from twisted.internet import defer
 
 from synapse.util.caches.descriptors import cachedInlineCallbacks
@@ -38,53 +36,8 @@ else:
 
 
 class KeyStore(SQLBaseStore):
-    """Persistence for signature verification keys and tls X.509 certificates
+    """Persistence for signature verification keys
     """
-
-    @defer.inlineCallbacks
-    def get_server_certificate(self, server_name):
-        """Retrieve the TLS X.509 certificate for the given server
-        Args:
-            server_name (bytes): The name of the server.
-        Returns:
-            (OpenSSL.crypto.X509): The tls certificate.
-        """
-        tls_certificate_bytes, = yield self._simple_select_one(
-            table="server_tls_certificates",
-            keyvalues={"server_name": server_name},
-            retcols=("tls_certificate",),
-            desc="get_server_certificate",
-        )
-        tls_certificate = OpenSSL.crypto.load_certificate(
-            OpenSSL.crypto.FILETYPE_ASN1, tls_certificate_bytes
-        )
-        defer.returnValue(tls_certificate)
-
-    def store_server_certificate(
-        self, server_name, from_server, time_now_ms, tls_certificate
-    ):
-        """Stores the TLS X.509 certificate for the given server
-        Args:
-            server_name (str): The name of the server.
-            from_server (str): Where the certificate was looked up
-            time_now_ms (int): The time now in milliseconds
-            tls_certificate (OpenSSL.crypto.X509): The X.509 certificate.
-        """
-        tls_certificate_bytes = OpenSSL.crypto.dump_certificate(
-            OpenSSL.crypto.FILETYPE_ASN1, tls_certificate
-        )
-        fingerprint = hashlib.sha256(tls_certificate_bytes).hexdigest()
-        return self._simple_upsert(
-            table="server_tls_certificates",
-            keyvalues={"server_name": server_name, "fingerprint": fingerprint},
-            values={
-                "from_server": from_server,
-                "ts_added_ms": time_now_ms,
-                "tls_certificate": db_binary_type(tls_certificate_bytes),
-            },
-            desc="store_server_certificate",
-        )
-
     @cachedInlineCallbacks()
     def _get_server_verify_key(self, server_name, key_id):
         verify_key_bytes = yield self._simple_select_one_onecol(
diff --git a/synapse/storage/schema/delta/54/drop_legacy_tables.sql b/synapse/storage/schema/delta/54/drop_legacy_tables.sql
index 77b39dc2d..ecca005d9 100644
--- a/synapse/storage/schema/delta/54/drop_legacy_tables.sql
+++ b/synapse/storage/schema/delta/54/drop_legacy_tables.sql
@@ -24,7 +24,5 @@ DROP TABLE IF EXISTS event_edge_hashes;
 DROP TABLE IF EXISTS event_signatures;
 DROP TABLE IF EXISTS feedback;
 DROP TABLE IF EXISTS room_hosts;
+DROP TABLE IF EXISTS server_tls_certificates;
 DROP TABLE IF EXISTS state_forward_extremities;
-
-
-
diff --git a/synapse/storage/schema/full_schemas/16/keys.sql b/synapse/storage/schema/full_schemas/16/keys.sql
index ca0ca1b69..11cdffdbb 100644
--- a/synapse/storage/schema/full_schemas/16/keys.sql
+++ b/synapse/storage/schema/full_schemas/16/keys.sql
@@ -12,14 +12,9 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-CREATE TABLE IF NOT EXISTS server_tls_certificates(
-  server_name TEXT, -- Server name.
-  fingerprint TEXT, -- Certificate fingerprint.
-  from_server TEXT, -- Which key server the certificate was fetched from.
-  ts_added_ms BIGINT, -- When the certifcate was added.
-  tls_certificate bytea, -- DER encoded x509 certificate.
-  UNIQUE (server_name, fingerprint)
-);
+
+-- we used to create a table called server_tls_certificates, but this is no
+-- longer used, and is removed in delta 54.
 
 CREATE TABLE IF NOT EXISTS server_signature_keys(
   server_name TEXT, -- Server name.