diff --git a/changelog.d/4562.misc b/changelog.d/4562.misc
new file mode 100644
index 000000000..f7185fa76
--- /dev/null
+++ b/changelog.d/4562.misc
@@ -0,0 +1 @@
+Docker: only copy what we need to the build image
diff --git a/docker/Dockerfile b/docker/Dockerfile
index 4b739e7d0..d21233484 100644
--- a/docker/Dockerfile
+++ b/docker/Dockerfile
@@ -31,7 +31,10 @@ RUN pip install --prefix="/install" --no-warn-script-location \
 
 # now install synapse and all of the python deps to /install.
 
-COPY . /synapse
+COPY synapse /synapse/synapse/
+COPY scripts /synapse/scripts/
+COPY MANIFEST.in README.rst setup.py synctl /synapse/
+
 RUN pip install --prefix="/install" --no-warn-script-location \
         /synapse[all]