Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 00:29:53 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

ssurts are strings now

Browse Source 
because they're friendlier that way in rethinkdb
This commit is contained in:
Noah Levitt 2018-05-16 16:43:10 -07:00
parent 399c097c7c
commit 5bb392ec7c
2 changed files with 18 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/test_cluster.py
View File

@ -454,7 +454,7 @@ def test_seed_redirect(httpd):
site = brozzler.Site(rr, { site = brozzler.Site(rr, {
'seed': 'http://localhost:%s/site5/redirect/' % httpd.server_port, 'seed': 'http://localhost:%s/site5/redirect/' % httpd.server_port,
'warcprox_meta': {'captures-table-extra-fields':{'test_id':test_id}}}) 'warcprox_meta': {'captures-table-extra-fields':{'test_id':test_id}}})
assert site.scope == {'accepts': [{'ssurt': ('localhost,//%s:http:/site5/redirect/' % httpd.server_port).encode('ascii')}]} assert site.scope == {'accepts': [{'ssurt': 'localhost,//%s:http:/site5/redirect/' % httpd.server_port}]}
frontier = brozzler.RethinkDbFrontier(rr) frontier = brozzler.RethinkDbFrontier(rr)
brozzler.new_site(frontier, site) brozzler.new_site(frontier, site)

34
tests/test_frontier.py
View File

@ -73,7 +73,7 @@ def test_basics():
'job_id': job.id, 'job_id': job.id,
'last_claimed': brozzler.EPOCH_UTC, 'last_claimed': brozzler.EPOCH_UTC,
'last_disclaimed': brozzler.EPOCH_UTC, 'last_disclaimed': brozzler.EPOCH_UTC,
'scope': {'accepts': [{'ssurt': b'com,example,//http:/'}]}, 'scope': {'accepts': [{'ssurt': 'com,example,//http:/'}]},
'seed': 'http://example.com', 'seed': 'http://example.com',
'starts_and_stops': [ 'starts_and_stops': [
{ {
@ -89,7 +89,7 @@ def test_basics():
'job_id': job.id, 'job_id': job.id,
'last_claimed': brozzler.EPOCH_UTC, 'last_claimed': brozzler.EPOCH_UTC,
'last_disclaimed': brozzler.EPOCH_UTC, 'last_disclaimed': brozzler.EPOCH_UTC,
'scope': {'accepts': [{'ssurt': b'org,example,//https:/'}]}, 'scope': {'accepts': [{'ssurt': 'org,example,//https:/'}]},
'seed': 'https://example.org/', 'seed': 'https://example.org/',
'starts_and_stops': [ 'starts_and_stops': [
{ {
@ -439,7 +439,7 @@ def test_field_defaults():
brozzler.Site.table_ensure(rr) brozzler.Site.table_ensure(rr)
site = brozzler.Site(rr, {'seed': 'http://example.com/'}) site = brozzler.Site(rr, {'seed': 'http://example.com/'})
assert site.id is None assert site.id is None
assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/'}]} assert site.scope == {'accepts': [{'ssurt': 'com,example,//http:/'}]}
site.save() site.save()
assert site.id assert site.id
assert site.scope assert site.scope
@ -633,15 +633,15 @@ def test_completed_page():
'hops_from_seed': 0, 'hops_from_seed': 0,
'redirect_url':'http://example.com/b/', }) 'redirect_url':'http://example.com/b/', })
page.save() page.save()
assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]} assert site.scope == {'accepts': [{'ssurt': 'com,example,//http:/a/'}]}
frontier.completed_page(site, page) frontier.completed_page(site, page)
assert site.scope == {'accepts': [ assert site.scope == {'accepts': [
{'ssurt': b'com,example,//http:/a/'}, {'ssurt': 'com,example,//http:/a/'},
{'ssurt': b'com,example,//http:/b/'}]} {'ssurt': 'com,example,//http:/b/'}]}
site.refresh() site.refresh()
assert site.scope == {'accepts': [ assert site.scope == {'accepts': [
{'ssurt': b'com,example,//http:/a/'}, {'ssurt': 'com,example,//http:/a/'},
{'ssurt': b'com,example,//http:/b/'}]} {'ssurt': 'com,example,//http:/b/'}]}
assert page.brozzle_count == 1 assert page.brozzle_count == 1
assert page.claimed == False assert page.claimed == False
page.refresh() page.refresh()
@ -660,11 +660,11 @@ def test_completed_page():
'hops_from_seed': 0, 'hops_from_seed': 0,
'redirect_url':'http://example.com/a/x/', }) 'redirect_url':'http://example.com/a/x/', })
page.save() page.save()
assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]} assert site.scope == {'accepts': [{'ssurt': 'com,example,//http:/a/'}]}
frontier.completed_page(site, page) frontier.completed_page(site, page)
assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]} assert site.scope == {'accepts': [{'ssurt': 'com,example,//http:/a/'}]}
site.refresh() site.refresh()
assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]} assert site.scope == {'accepts': [{'ssurt': 'com,example,//http:/a/'}]}
assert page.brozzle_count == 1 assert page.brozzle_count == 1
assert page.claimed == False assert page.claimed == False
page.refresh() page.refresh()
@ -682,11 +682,11 @@ def test_completed_page():
'hops_from_seed': 1, 'hops_from_seed': 1,
'redirect_url':'http://example.com/d/', }) 'redirect_url':'http://example.com/d/', })
page.save() page.save()
assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]} assert site.scope == {'accepts': [{'ssurt': 'com,example,//http:/a/'}]}
frontier.completed_page(site, page) frontier.completed_page(site, page)
assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]} assert site.scope == {'accepts': [{'ssurt': 'com,example,//http:/a/'}]}
site.refresh() site.refresh()
assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]} assert site.scope == {'accepts': [{'ssurt': 'com,example,//http:/a/'}]}
assert page.brozzle_count == 1 assert page.brozzle_count == 1
assert page.claimed == False assert page.claimed == False
page.refresh() page.refresh()
@ -726,7 +726,7 @@ def test_hashtag_seed():
site = brozzler.Site(rr, {'seed': 'http://example.org/'}) site = brozzler.Site(rr, {'seed': 'http://example.org/'})
brozzler.new_site(frontier, site) brozzler.new_site(frontier, site)
assert site.scope == {'accepts': [{'ssurt': b'org,example,//http:/'}]} assert site.scope == {'accepts': [{'ssurt': 'org,example,//http:/'}]}
pages = list(frontier.site_pages(site.id)) pages = list(frontier.site_pages(site.id))
assert len(pages) == 1 assert len(pages) == 1
@ -737,7 +737,7 @@ def test_hashtag_seed():
site = brozzler.Site(rr, {'seed': 'http://example.org/#hash'}) site = brozzler.Site(rr, {'seed': 'http://example.org/#hash'})
brozzler.new_site(frontier, site) brozzler.new_site(frontier, site)
assert site.scope == {'accepts': [{'ssurt': b'org,example,//http:/'}]} assert site.scope == {'accepts': [{'ssurt': 'org,example,//http:/'}]}
pages = list(frontier.site_pages(site.id)) pages = list(frontier.site_pages(site.id))
assert len(pages) == 1 assert len(pages) == 1
@ -985,7 +985,7 @@ def test_max_hops_off():
'seed': 'http://example.com/', 'seed': 'http://example.com/',
'scope': { 'scope': {
'max_hops_off_surt': 1, 'max_hops_off_surt': 1,
'blocks': [{'ssurt': b'domain,bad,'}]}}) 'blocks': [{'ssurt': 'domain,bad,'}]}})
brozzler.new_site(frontier, site) brozzler.new_site(frontier, site)
site.refresh() # get it back from the db site.refresh() # get it back from the db