diff --git a/setup.py b/setup.py index db7a56c..fee81ff 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ setuptools.setup( 'youtube-dl', 'reppy==0.3.4', 'requests', - 'websocket-client!=0.39.0', + 'websocket-client!=0.39.0,!=0.49.0', 'pillow>=5.2.0', 'urlcanon>=0.1.dev23', 'doublethink>=0.2.0.dev88', diff --git a/tests/htdocs/site10/index.html b/tests/htdocs/site10/index.html new file mode 100644 index 0000000..0794c84 --- /dev/null +++ b/tests/htdocs/site10/index.html @@ -0,0 +1,34 @@ + + + segmented (hls) video test + + + + + + + + + + diff --git a/tests/htdocs/site10/small.m3u8 b/tests/htdocs/site10/small.m3u8 new file mode 100644 index 0000000..d86749c --- /dev/null +++ b/tests/htdocs/site10/small.m3u8 @@ -0,0 +1,15 @@ +#EXTM3U +#EXT-X-VERSION:3 +#EXT-X-TARGETDURATION:1 +#EXT-X-MEDIA-SEQUENCE:1 +#EXTINF:1.000000, +small1.ts +#EXTINF:1.000000, +small2.ts +#EXTINF:1.000000, +small3.ts +#EXTINF:1.000000, +small4.ts +#EXTINF:0.533333, +small5.ts +#EXT-X-ENDLIST diff --git a/tests/htdocs/site10/small0.ts b/tests/htdocs/site10/small0.ts new file mode 100644 index 0000000..53d6c87 Binary files /dev/null and b/tests/htdocs/site10/small0.ts differ diff --git a/tests/htdocs/site10/small1.ts b/tests/htdocs/site10/small1.ts new file mode 100644 index 0000000..d4d6df6 Binary files /dev/null and b/tests/htdocs/site10/small1.ts differ diff --git a/tests/htdocs/site10/small2.ts b/tests/htdocs/site10/small2.ts new file mode 100644 index 0000000..d594a09 Binary files /dev/null and b/tests/htdocs/site10/small2.ts differ diff --git a/tests/htdocs/site10/small3.ts b/tests/htdocs/site10/small3.ts new file mode 100644 index 0000000..e035203 Binary files /dev/null and b/tests/htdocs/site10/small3.ts differ diff --git a/tests/htdocs/site10/small4.ts b/tests/htdocs/site10/small4.ts new file mode 100644 index 0000000..38e5059 Binary files /dev/null and b/tests/htdocs/site10/small4.ts differ diff --git a/tests/htdocs/site10/small5.ts b/tests/htdocs/site10/small5.ts new file mode 100644 index 0000000..b923b07 Binary files /dev/null and b/tests/htdocs/site10/small5.ts differ diff --git a/tests/test_cluster.py b/tests/test_cluster.py index ae4f7a4..878cd35 100644 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -3,7 +3,7 @@ test_cluster.py - integration tests for a brozzler cluster, expects brozzler, warcprox, pywb, rethinkdb and other dependencies to be running already -Copyright (C) 2016-2017 Internet Archive +Copyright (C) 2016-2018 Internet Archive Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -796,3 +796,31 @@ def test_time_limit(httpd): job.refresh() assert job.status == 'FINISHED' +def test_ydl_stitching(httpd): + test_id = 'test_ydl_stitching-%s' % datetime.datetime.utcnow().isoformat() + rr = doublethink.Rethinker('localhost', db='brozzler') + frontier = brozzler.RethinkDbFrontier(rr) + site = brozzler.Site(rr, { + 'seed': 'http://localhost:%s/site10/' % httpd.server_port}) + brozzler.new_site(frontier, site) + + # the site should be brozzled fairly quickly + start = time.time() + while site.status != 'FINISHED' and time.time() - start < 300: + time.sleep(0.5) + site.refresh() + assert site.status == 'FINISHED' + + # check page.videos + pages = list(frontier.site_pages(site.id)) + assert len(pages) == 1 + page = pages[0] + assert len(page.videos) == 6 + assert { + 'blame': 'youtube-dl', + 'content-length': 267900, + 'content-type': 'video/mp4', + 'response_code': 204, + 'url': 'youtube-dl:00001:http://localhost:%s/site10/' % httpd.server_port, + } in page.videos +