Get uploads working with new media repo

synapse/api/urls.py

@@ -20,3 +20,4 @@ FEDERATION_PREFIX = "/_matrix/federation/v1"
 WEB_CLIENT_PREFIX = "/_matrix/client"
 CONTENT_REPO_PREFIX = "/_matrix/content"
 SERVER_KEY_PREFIX = "/_matrix/key/v1"
+MEDIA_PREFIX = "/_matrix/media/v1"

synapse/app/homeserver.py

@@ -24,12 +24,13 @@ from twisted.web.resource import Resource
 from twisted.web.static import File
 from twisted.web.server import Site
 from synapse.http.server import JsonResource, RootRedirect
-from synapse.http.content_repository import ContentRepoResource
+from synapse.media.v0.content_repository import ContentRepoResource
+from synapse.media.v1.media_repository import MediaRepositoryResource
 from synapse.http.server_key_resource import LocalKey
 from synapse.http.matrixfederationclient import MatrixFederationHttpClient
 from synapse.api.urls import (
     CLIENT_PREFIX, FEDERATION_PREFIX, WEB_CLIENT_PREFIX, CONTENT_REPO_PREFIX,
-    SERVER_KEY_PREFIX,
+    SERVER_KEY_PREFIX, MEDIA_PREFIX
 )
 from synapse.config.homeserver import HomeServerConfig
 from synapse.crypto import context_factory
@@ -69,6 +70,9 @@ class SynapseHomeServer(HomeServer):
             self, self.upload_dir, self.auth, self.content_addr
         )
 
+    def build_resource_for_media_repository(self):
+        return MediaRepositoryResource(self)
+
     def build_resource_for_server_key(self):
         return LocalKey(self)
 
@@ -99,6 +103,7 @@ class SynapseHomeServer(HomeServer):
             (FEDERATION_PREFIX, self.get_resource_for_federation()),
             (CONTENT_REPO_PREFIX, self.get_resource_for_content_repo()),
             (SERVER_KEY_PREFIX, self.get_resource_for_server_key()),
+            (MEDIA_PREFIX, self.get_resource_for_media_repository()),
         ]
         if web_client:
             logger.info("Adding the web client.")

synapse/config/_base.py

@@ -50,12 +50,26 @@ class Config(object):
             )
         return cls.abspath(file_path)
 
+    @staticmethod
+    def ensure_directory(dir_path):
+        if not os.path.exists(dir_path):
+            os.makedirs(dir_path)
+        if not os.path.isdir(dir_path):
+            raise ConfigError(
+                "%s is not a directory" % (dir_path,)
+            )
+        return dir_path
+
     @classmethod
     def read_file(cls, file_path, config_name):
         cls.check_file(file_path, config_name)
         with open(file_path) as file_stream:
             return file_stream.read()
 
+    @staticmethod
+    def default_path(name):
+        return os.path.abspath(os.path.join(os.path.curdir, name))
+
     @staticmethod
     def read_config_file(file_path):
         with open(file_path) as file_stream:

synapse/config/repository.py

@@ -20,6 +20,7 @@ class ContentRepositoryConfig(Config):
     def __init__(self, args):
         super(ContentRepositoryConfig, self).__init__(args)
         self.max_upload_size = self.parse_size(args.max_upload_size)
+        self.media_store_path = self.ensure_directory(args.media_store_path)
 
     def parse_size(self, string):
         sizes = {"K": 1024, "M": 1024 * 1024}
@@ -37,3 +38,6 @@ class ContentRepositoryConfig(Config):
         db_group.add_argument(
             "--max-upload-size", default="1M"
         )
+        db_group.add_argument(
+            "--media-store-path", default=cls.default_path("media_store")
+        )

synapse/http/server.py

@@ -201,9 +201,9 @@ class RootRedirect(resource.Resource):
 def respond_with_json(request, code, json_object, send_cors=False,
                       response_code_message=None, pretty_print=False):
     if not pretty_print:
-        json_bytes = encode_pretty_printed_json(response_json_object)
+        json_bytes = encode_pretty_printed_json(json_object)
     else:
-        json_bytes = encode_canonical_json(response_json_object)
+        json_bytes = encode_canonical_json(json_object)
 
     return respond_with_json_bytes(request, code, json_bytes, send_cors,
                                    response_code_message=response_code_message)

synapse/media/v0/content_repository.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from .server import respond_with_json_bytes
+from synapse.http.server import respond_with_json_bytes
 
 from synapse.util.stringutils import random_string
 from synapse.api.errors import (

synapse/media/v1/media_repository.py

@@ -13,27 +13,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from synapse.http.server import respond_with_json_bytes
+from .upload_resource import UploadResource
+from .filepath import MediaFilePaths
 
-from synapse.util.stringutils import random_string
+from twisted.web.resource import Resource
-from synapse.api.errors import (
-    cs_exception, SynapseError, CodeMessageException, Codes, cs_error
-)
 
-from twisted.protocols.basic import FileSender
-from twisted.web import server, resource
-from twisted.internet import defer
-
-import base64
-import json
 import logging
-import os
-import re
 
 logger = logging.getLogger(__name__)
 
 
-class MediaRepository():
+class MediaRepositoryResource(Resource):
     """Profiles file uploading and downloading.
 
     Uploads are POSTed to a resource which returns a token which is used to GET
@@ -68,5 +58,6 @@ class MediaRepository():
     """
 
     def __init__(self, hs):
-        filepaths = MediaFilePaths
+        Resource.__init__(self)
-
+        filepaths = MediaFilePaths(hs.config.media_store_path)
+        self.putChild("upload", UploadResource(hs, filepaths))

synapse/media/v1/upload_resource.py

@@ -23,6 +23,8 @@ from synapse.api.errors import (
 from twisted.web import server, resource
 from twisted.internet import defer
 
+import os
+
 import logging
 
 logger = logging.getLogger(__name__)
@@ -31,8 +33,9 @@ class UploadResource(resource.Resource):
 
     def __init__(self, hs, filepaths):
         self.auth = hs.get_auth()
+        self.clock = hs.get_clock()
         self.store = hs.get_datastore()
-        self.max_upload_size = hs.config.max_upload_size()
+        self.max_upload_size = hs.config.max_upload_size
         self.filepaths = filepaths
 
     def render_POST(self, request):
@@ -45,10 +48,8 @@ class UploadResource(resource.Resource):
 
     @defer.inlineCallbacks
     def _async_render_POST(self, request):
-
-        auth_user = yield self.auth.get_user_by_req(request)
-
         try:
+            auth_user = yield self.auth.get_user_by_req(request)
             # TODO: The checks here are a bit late. The content will have
             # already been uploaded to a tmp file at this point
             content_length = request.getHeader("Content-Length")
@@ -62,7 +63,7 @@ class UploadResource(resource.Resource):
                     code=413,
                 )
 
-            headers = request.requestHeaders()
+            headers = request.requestHeaders
 
             if headers.hasHeader("Content-Type"):
                 media_type = headers.getRawHeaders("Content-Type")[0]
@@ -78,7 +79,8 @@ class UploadResource(resource.Resource):
 
             media_id = random_string(24)
 
-            fname = self.filepaths.local_media_file_path(media_id)
+            fname = self.filepaths.local_media_filepath(media_id)
+            os.makedirs(os.path.dirname(fname))
 
             # This shouldn't block for very long because the content will have
             # already been uploaded at this point.

synapse/server.py

@@ -78,6 +78,7 @@ class BaseHomeServer(object):
         'resource_for_web_client',
         'resource_for_content_repo',
         'resource_for_server_key',
+        'resource_for_media_repository',
         'event_sources',
         'ratelimiter',
         'keyring',

synapse/storage/__init__.py

@@ -33,6 +33,7 @@ from .stream import StreamStore
 from .transactions import TransactionStore
 from .keys import KeyStore
 from .event_federation import EventFederationStore
+from .media_repository import MediaRepositoryStore
 
 from .state import StateStore
 from .signatures import SignatureStore
@@ -62,6 +63,7 @@ SCHEMAS = [
     "state",
     "event_edges",
     "event_signatures",
+    "media_repository",
 ]
 
 
@@ -81,7 +83,9 @@ class DataStore(RoomMemberStore, RoomStore,
                 RegistrationStore, StreamStore, ProfileStore, FeedbackStore,
                 PresenceStore, TransactionStore,
                 DirectoryStore, KeyStore, StateStore, SignatureStore,
-                EventFederationStore, ):
+                EventFederationStore,
+                MediaRepositoryStore,
+                ):
 
     def __init__(self, hs):
         super(DataStore, self).__init__(hs)

synapse/storage/media_repository.py

@@ -20,10 +20,15 @@ class MediaRepositoryStore(SQLBaseStore):
     """Persistence for attachments and avatars"""
 
     def get_local_media(self, media_id):
+        """Get the metadata for a local piece of media
+        Returns:
+            None if the media_id doesn't exist.
+        """
         return self._simple_select_one(
             "local_media_repository",
             {"media_id": media_id},
             ("media_type", "media_length", "upload_name", "created_ts"),
+            True,
         )
 
     def store_local_media(self, media_id, media_type, time_now_ms, upload_name,
@@ -36,7 +41,7 @@ class MediaRepositoryStore(SQLBaseStore):
                 "created_ts": time_now_ms,
                 "upload_name": upload_name,
                 "media_length": media_length,
-                "user_id": user_id,
+                "user_id": user_id.to_string(),
             }
         )