refactor calc_og; spider image URLs; fix xpath; add a (broken) expiringcache; loads of other fixes

synapse/rest/media/v1/preview_url_resource.py

@@ -20,6 +20,7 @@ from twisted.internet import defer
 from lxml import html
 from urlparse import urlparse, urlunparse
 from synapse.util.stringutils import random_string
+from synapse.util.caches.expiringcache import ExpiringCache
 from synapse.http.client import SpiderHttpClient
 from synapse.http.server import request_handler, respond_with_json, respond_with_json_bytes
 
@@ -36,6 +37,12 @@ class PreviewUrlResource(BaseMediaResource):
     def __init__(self, hs, filepaths):
         BaseMediaResource.__init__(self, hs, filepaths)
         self.client = SpiderHttpClient(hs)
+        self.cache = ExpiringCache(
+            cache_name = "url_previews",
+            clock = self.clock,
+            expiry_ms = 60*60*1000, # don't spider URLs more often than once an hour
+        )
+        self.cache.start()
 
     def render_GET(self, request):
         self._async_render_GET(request)
@@ -50,6 +57,11 @@ class PreviewUrlResource(BaseMediaResource):
             requester = yield self.auth.get_user_by_req(request)
             url = request.args.get("url")[0]
 
+            if self.cache:
+                og = self.cache.get(url)
+                respond_with_json_bytes(request, 200, json.dumps(og), send_cors=True)
+                return
+
             # TODO: keep track of whether there's an ongoing request for this preview
             # and block and return their details if there is one.
 
@@ -74,9 +86,46 @@ class PreviewUrlResource(BaseMediaResource):
             elif self._is_html(media_info['media_type']):
                 # TODO: somehow stop a big HTML tree from exploding synapse's RAM
 
-                def _calc_og():
+                try:
-                    # suck it up into lxml and define our OG response.
+                    tree = html.parse(media_info['filename'])
-                    # if we see any URLs in the OG response, then spider them
+                    og = yield self._calc_og(tree, media_info, requester)
+                except UnicodeDecodeError:
+                    # XXX: evil evil bodge
+                    file = open(media_info['filename'])
+                    body = file.read()
+                    file.close()
+                    tree = html.fromstring(body.decode('utf-8','ignore'))
+                    og = yield self._calc_og(tree, media_info, requester)
+
+            else:
+                logger.warn("Failed to find any OG data in %s", url)
+                og = {}
+
+            if self.cache:
+                self.cache[url] = og
+
+            logger.warn(og);
+
+            respond_with_json_bytes(request, 200, json.dumps(og), send_cors=True)
+        except:
+            # XXX: if we don't explicitly respond here, the request never returns.
+            # isn't this what server.py's wrapper is meant to be doing for us?
+            respond_with_json(
+                request,
+                500,
+                {
+                    "error": "Internal server error",
+                    "errcode": Codes.UNKNOWN,
+                },
+                send_cors=True
+            )
+            raise
+
+    @defer.inlineCallbacks
+    def _calc_og(self, tree, media_info, requester):
+        # suck our tree into lxml and define our OG response.
+
+        # if we see any image URLs in the OG response, then spider them
         # (although the client could choose to do this by asking for previews of those URLs to avoid DoSing the server)
 
         # "og:type"        : "article"
@@ -104,35 +153,59 @@ class PreviewUrlResource(BaseMediaResource):
         for tag in tree.xpath("//*/meta[starts-with(@property, 'og:')]"):
             og[tag.attrib['property']] = tag.attrib['content']
 
+        # TODO: grab article: meta tags too, e.g.:
+
+        # <meta property="article:publisher" content="https://www.facebook.com/thethudonline" />
+        # <meta property="article:author" content="https://www.facebook.com/thethudonline" />
+        # <meta property="article:tag" content="baby" />
+        # <meta property="article:section" content="Breaking News" />
+        # <meta property="article:published_time" content="2016-03-31T19:58:24+00:00" />
+        # <meta property="article:modified_time" content="2016-04-01T18:31:53+00:00" />
+
         if 'og:title' not in og:
             # do some basic spidering of the HTML
             title = tree.xpath("(//title)[1] | (//h1)[1] | (//h2)[1] | (//h3)[1]")
-                        og['og:title'] = title[0].text if title else None
+            og['og:title'] = title[0].text.strip() if title else None
 
 
         if 'og:image' not in og:
+            # TODO: extract a favicon failing all else
             meta_image = tree.xpath("//*/meta[@itemprop='image']/@content");
             if meta_image:
                 og['og:image'] = self._rebase_url(meta_image[0], media_info['uri'])
             else:
-                            images = [ i for i in tree.xpath("//img") if 'src' in i.attrib ]
+                images = tree.xpath("//img[@src][number(@width)>10][number(@height)>10]")
-                            big_images = [ i for i in images if (
+                images = sorted(images, key=lambda i: (-1 * int(i.attrib['width']) * int(i.attrib['height'])))
-                                'width' in i.attrib and 'height' in i.attrib and
+                if not images:
-                                i.attrib['width'] > 64 and i.attrib['height'] > 64
+                    images = tree.xpath("//img[@src]")
-                            )]
-                            big_images = big_images.sort(key=lambda i: (-1 * int(i.attrib['width']) * int(i.attrib['height'])))
-                            images = big_images if big_images else images
-
                 if images:
                     og['og:image'] = self._rebase_url(images[0].attrib['src'], media_info['uri'])
 
+        # pre-cache the image for posterity
+        if 'og:image' in og and og['og:image']:
+            image_info = yield self._download_url(og['og:image'], requester.user)
+
+            if self._is_media(image_info['media_type']):
+                # TODO: make sure we don't choke on white-on-transparent images
+                dims = yield self._generate_local_thumbnails(
+                        image_info['filesystem_id'], image_info
+                      )
+                og["og:image"] = "mxc://%s/%s" % (self.server_name, image_info['filesystem_id'])
+                og["og:image:type"] = image_info['media_type']
+                og["og:image:width"] = dims['width']
+                og["og:image:height"] = dims['height']
+            else:
+                del og["og:image"]
+
         if 'og:description' not in og:
             meta_description = tree.xpath("//*/meta[@name='description']/@content");
             if meta_description:
                 og['og:description'] = meta_description[0]
             else:
-                            text_nodes = tree.xpath("//h1/text() | //h2/text() | //h3/text() | //p/text() | //div/text() | //span/text() | //a/text()")
+                # text_nodes = tree.xpath("//h1/text() | //h2/text() | //h3/text() | //p/text() | //div/text() | //span/text() | //a/text()")                
-                            # text_nodes = tree.xpath("//h1/text() | //h2/text() | //h3/text() | //p/text() | //div/text()")
+                text_nodes = tree.xpath("//text()[not(ancestor::header | ancestor::nav | ancestor::aside | " +
+                                        "ancestor::footer | ancestor::script | ancestor::style)]" +
+                                        "[ancestor::body]")
                 text = ''
                 for text_node in text_nodes:
                     if len(text) < 500:
@@ -144,49 +217,16 @@ class PreviewUrlResource(BaseMediaResource):
                 text = text.strip()[:500]
                 og['og:description'] = text if text else None
 
-                    # TODO: extract a favicon?
+        # TODO: persist a cache mapping { url, etag } -> { og, mxc of url (if we bother keeping it around), age }
-                    # TODO: turn any OG media URLs into mxc URLs to capture and thumbnail them too
+        # TODO: delete the url downloads to stop diskfilling, as we only ever cared about its OG
-                    # TODO: store our OG details in a cache (and expire them when stale)
+        defer.returnValue(og);
-                    # TODO: delete the content to stop diskfilling, as we only ever cared about its OG
-                    return og
-
-                try:
-                    tree = html.parse(media_info['filename'])
-                    og = _calc_og()
-                except UnicodeDecodeError:
-                    # XXX: evil evil bodge
-                    file = open(media_info['filename'])
-                    body = file.read()
-                    file.close()
-                    tree = html.fromstring(body.decode('utf-8','ignore'))
-                    og = _calc_og()
-
-            else:
-                logger.warn("Failed to find any OG data in %s", url)
-                og = {}
-
-            logger.warn(og)
-
-            respond_with_json_bytes(request, 200, json.dumps(og), send_cors=True)
-        except:
-            # XXX: if we don't explicitly respond here, the request never returns.
-            # isn't this what server.py's wrapper is meant to be doing for us?
-            respond_with_json(
-                request,
-                500,
-                {
-                    "error": "Internal server error",
-                    "errcode": Codes.UNKNOWN,
-                },
-                send_cors=True
-            )
-            raise
 
     def _rebase_url(self, url, base):
         base = list(urlparse(base))
         url = list(urlparse(url))
-        if not url[0] and not url[1]:
+        if not url[0]:
-            url[0] = base[0]
+            url[0] = base[0] or "http"
+        if not url[1]:
             url[1] = base[1]
             if not url[2].startswith('/'):
                 url[2] = re.sub(r'/[^/]+$', '/', base[2]) + url[2]