diff --git a/synapse/config/_base.py b/synapse/config/_base.py
index 1a6784a71..8a75c4873 100644
--- a/synapse/config/_base.py
+++ b/synapse/config/_base.py
@@ -182,7 +182,7 @@ class Config(object):
                             ) % (entry_path, )
                             continue
 
-                        files.add(config_path)
+                        files.append(entry_path)
 
                     config_files.extend(sorted(files))
                 else: