mirror of
https://git.anonymousland.org/anonymousland/synapse.git
synced 2025-05-12 17:42:11 -04:00
Attempt different character encodings when previewing a URL. (#11077)
This follows similar logic to BeautifulSoup where we attempt different character encodings until we find one which works.
This commit is contained in:
Patrick Cloke
GitHub
parent
1609ccf8fe
commit
e2f0b49b3f
3 changed files with 80 additions and 67 deletions
|
|
@ -15,7 +15,7 @@
|
|||
from synapse.rest.media.v1.preview_url_resource import (
|
||||
_calc_og,
|
||||
decode_body,
|
||||
get_html_media_encoding,
|
||||
get_html_media_encodings,
|
||||
summarize_paragraphs,
|
||||
)
|
||||
|
||||
|
|
@ -159,7 +159,7 @@ class CalcOgTestCase(unittest.TestCase):
|
|||
</html>
|
||||
"""
|
||||
|
||||
tree = decode_body(html)
|
||||
tree = decode_body(html, "http://example.com/test.html")
|
||||
og = _calc_og(tree, "http://example.com/test.html")
|
||||
|
||||
self.assertEqual(og, {"og:title": "Foo", "og:description": "Some text."})
|
||||
|
|
@ -175,7 +175,7 @@ class CalcOgTestCase(unittest.TestCase):
|
|||
</html>
|
||||
"""
|
||||
|
||||
tree = decode_body(html)
|
||||
tree = decode_body(html, "http://example.com/test.html")
|
||||
og = _calc_og(tree, "http://example.com/test.html")
|
||||
|
||||
self.assertEqual(og, {"og:title": "Foo", "og:description": "Some text."})
|
||||
|
|
@ -194,7 +194,7 @@ class CalcOgTestCase(unittest.TestCase):
|
|||
</html>
|
||||
"""
|
||||
|
||||
tree = decode_body(html)
|
||||
tree = decode_body(html, "http://example.com/test.html")
|
||||
og = _calc_og(tree, "http://example.com/test.html")
|
||||
|
||||
self.assertEqual(
|
||||
|
|
@ -216,7 +216,7 @@ class CalcOgTestCase(unittest.TestCase):
|
|||
</html>
|
||||
"""
|
||||
|
||||
tree = decode_body(html)
|
||||
tree = decode_body(html, "http://example.com/test.html")
|
||||
og = _calc_og(tree, "http://example.com/test.html")
|
||||
|
||||
self.assertEqual(og, {"og:title": "Foo", "og:description": "Some text."})
|
||||
|
|
@ -230,7 +230,7 @@ class CalcOgTestCase(unittest.TestCase):
|
|||
</html>
|
||||
"""
|
||||
|
||||
tree = decode_body(html)
|
||||
tree = decode_body(html, "http://example.com/test.html")
|
||||
og = _calc_og(tree, "http://example.com/test.html")
|
||||
|
||||
self.assertEqual(og, {"og:title": None, "og:description": "Some text."})
|
||||
|
|
@ -245,7 +245,7 @@ class CalcOgTestCase(unittest.TestCase):
|
|||
</html>
|
||||
"""
|
||||
|
||||
tree = decode_body(html)
|
||||
tree = decode_body(html, "http://example.com/test.html")
|
||||
og = _calc_og(tree, "http://example.com/test.html")
|
||||
|
||||
self.assertEqual(og, {"og:title": "Title", "og:description": "Some text."})
|
||||
|
|
@ -260,7 +260,7 @@ class CalcOgTestCase(unittest.TestCase):
|
|||
</html>
|
||||
"""
|
||||
|
||||
tree = decode_body(html)
|
||||
tree = decode_body(html, "http://example.com/test.html")
|
||||
og = _calc_og(tree, "http://example.com/test.html")
|
||||
|
||||
self.assertEqual(og, {"og:title": None, "og:description": "Some text."})
|
||||
|
|
@ -268,13 +268,13 @@ class CalcOgTestCase(unittest.TestCase):
|
|||
def test_empty(self):
|
||||
"""Test a body with no data in it."""
|
||||
html = b""
|
||||
tree = decode_body(html)
|
||||
tree = decode_body(html, "http://example.com/test.html")
|
||||
self.assertIsNone(tree)
|
||||
|
||||
def test_no_tree(self):
|
||||
"""A valid body with no tree in it."""
|
||||
html = b"\x00"
|
||||
tree = decode_body(html)
|
||||
tree = decode_body(html, "http://example.com/test.html")
|
||||
self.assertIsNone(tree)
|
||||
|
||||
def test_invalid_encoding(self):
|
||||
|
|
@ -287,7 +287,7 @@ class CalcOgTestCase(unittest.TestCase):
|
|||
</body>
|
||||
</html>
|
||||
"""
|
||||
tree = decode_body(html, "invalid-encoding")
|
||||
tree = decode_body(html, "http://example.com/test.html", "invalid-encoding")
|
||||
og = _calc_og(tree, "http://example.com/test.html")
|
||||
self.assertEqual(og, {"og:title": "Foo", "og:description": "Some text."})
|
||||
|
||||
|
|
@ -302,15 +302,29 @@ class CalcOgTestCase(unittest.TestCase):
|
|||
</body>
|
||||
</html>
|
||||
"""
|
||||
tree = decode_body(html)
|
||||
tree = decode_body(html, "http://example.com/test.html")
|
||||
og = _calc_og(tree, "http://example.com/test.html")
|
||||
self.assertEqual(og, {"og:title": "ÿÿ Foo", "og:description": "Some text."})
|
||||
|
||||
def test_windows_1252(self):
|
||||
"""A body which uses windows-1252, but doesn't declare that."""
|
||||
html = b"""
|
||||
<html>
|
||||
<head><title>\xf3</title></head>
|
||||
<body>
|
||||
Some text.
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
tree = decode_body(html, "http://example.com/test.html")
|
||||
og = _calc_og(tree, "http://example.com/test.html")
|
||||
self.assertEqual(og, {"og:title": "ó", "og:description": "Some text."})
|
||||
|
||||
|
||||
class MediaEncodingTestCase(unittest.TestCase):
|
||||
def test_meta_charset(self):
|
||||
"""A character encoding is found via the meta tag."""
|
||||
encoding = get_html_media_encoding(
|
||||
encodings = get_html_media_encodings(
|
||||
b"""
|
||||
<html>
|
||||
<head><meta charset="ascii">
|
||||
|
|
@ -319,10 +333,10 @@ class MediaEncodingTestCase(unittest.TestCase):
|
|||
""",
|
||||
"text/html",
|
||||
)
|
||||
self.assertEqual(encoding, "ascii")
|
||||
self.assertEqual(list(encodings), ["ascii", "utf-8", "windows-1252"])
|
||||
|
||||
# A less well-formed version.
|
||||
encoding = get_html_media_encoding(
|
||||
encodings = get_html_media_encodings(
|
||||
b"""
|
||||
<html>
|
||||
<head>< meta charset = ascii>
|
||||
|
|
@ -331,11 +345,11 @@ class MediaEncodingTestCase(unittest.TestCase):
|
|||
""",
|
||||
"text/html",
|
||||
)
|
||||
self.assertEqual(encoding, "ascii")
|
||||
self.assertEqual(list(encodings), ["ascii", "utf-8", "windows-1252"])
|
||||
|
||||
def test_meta_charset_underscores(self):
|
||||
"""A character encoding contains underscore."""
|
||||
encoding = get_html_media_encoding(
|
||||
encodings = get_html_media_encodings(
|
||||
b"""
|
||||
<html>
|
||||
<head><meta charset="Shift_JIS">
|
||||
|
|
@ -344,11 +358,11 @@ class MediaEncodingTestCase(unittest.TestCase):
|
|||
""",
|
||||
"text/html",
|
||||
)
|
||||
self.assertEqual(encoding, "Shift_JIS")
|
||||
self.assertEqual(list(encodings), ["Shift_JIS", "utf-8", "windows-1252"])
|
||||
|
||||
def test_xml_encoding(self):
|
||||
"""A character encoding is found via the meta tag."""
|
||||
encoding = get_html_media_encoding(
|
||||
encodings = get_html_media_encodings(
|
||||
b"""
|
||||
<?xml version="1.0" encoding="ascii"?>
|
||||
<html>
|
||||
|
|
@ -356,11 +370,11 @@ class MediaEncodingTestCase(unittest.TestCase):
|
|||
""",
|
||||
"text/html",
|
||||
)
|
||||
self.assertEqual(encoding, "ascii")
|
||||
self.assertEqual(list(encodings), ["ascii", "utf-8", "windows-1252"])
|
||||
|
||||
def test_meta_xml_encoding(self):
|
||||
"""Meta tags take precedence over XML encoding."""
|
||||
encoding = get_html_media_encoding(
|
||||
encodings = get_html_media_encodings(
|
||||
b"""
|
||||
<?xml version="1.0" encoding="ascii"?>
|
||||
<html>
|
||||
|
|
@ -370,7 +384,7 @@ class MediaEncodingTestCase(unittest.TestCase):
|
|||
""",
|
||||
"text/html",
|
||||
)
|
||||
self.assertEqual(encoding, "UTF-16")
|
||||
self.assertEqual(list(encodings), ["UTF-16", "ascii", "utf-8", "windows-1252"])
|
||||
|
||||
def test_content_type(self):
|
||||
"""A character encoding is found via the Content-Type header."""
|
||||
|
|
@ -384,10 +398,10 @@ class MediaEncodingTestCase(unittest.TestCase):
|
|||
'text/html; charset=ascii";',
|
||||
)
|
||||
for header in headers:
|
||||
encoding = get_html_media_encoding(b"", header)
|
||||
self.assertEqual(encoding, "ascii")
|
||||
encodings = get_html_media_encodings(b"", header)
|
||||
self.assertEqual(list(encodings), ["ascii", "utf-8", "windows-1252"])
|
||||
|
||||
def test_fallback(self):
|
||||
"""A character encoding cannot be found in the body or header."""
|
||||
encoding = get_html_media_encoding(b"", "text/html")
|
||||
self.assertEqual(encoding, "utf-8")
|
||||
encodings = get_html_media_encodings(b"", "text/html")
|
||||
self.assertEqual(list(encodings), ["utf-8", "windows-1252"])
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue