diff --git a/changelog.d/3911.misc b/changelog.d/3911.misc
new file mode 100644
index 000000000..e31311d52
--- /dev/null
+++ b/changelog.d/3911.misc
@@ -0,0 +1 @@
+Fix the docker image building on python 3
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 20d3fe3bd..1d00defc2 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -1,4 +1,5 @@
-FROM docker.io/python:2-alpine3.8
+ARG PYTHON_VERSION=2
+FROM docker.io/python:${PYTHON_VERSION}-alpine3.8
 
 COPY . /synapse
 
diff --git a/docker/start.py b/docker/start.py
index 90e8b9c51..346df8c87 100755
--- a/docker/start.py
+++ b/docker/start.py
@@ -5,6 +5,7 @@ import os
 import sys
 import subprocess
 import glob
+import codecs
 
 # Utility functions
 convert = lambda src, dst, environ: open(dst, "w").write(jinja2.Template(open(src).read()).render(**environ))
@@ -23,7 +24,7 @@ def generate_secrets(environ, secrets):
                 with open(filename) as handle: value = handle.read()
             else:
                 print("Generating a random secret for {}".format(name))
-                value = os.urandom(32).encode("hex")
+                value = codecs.encode(os.urandom(32), "hex").decode()
                 with open(filename, "w") as handle: handle.write(value)
             environ[secret] = value