Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

tests for new approach without of scope['surt']

Browse Source 
replaced by an accept rule (two rules in some cases of seed redirects)
This commit is contained in:
Noah Levitt 2018-03-22 13:33:34 -07:00
parent f26712ce93
commit 245e27a21a
1 changed files with 18 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
tests/test_frontier.py
View File

@ -73,9 +73,7 @@ def test_basics():
'job_id': job.id, 'job_id': job.id,
'last_claimed': brozzler.EPOCH_UTC, 'last_claimed': brozzler.EPOCH_UTC,
'last_disclaimed': brozzler.EPOCH_UTC, 'last_disclaimed': brozzler.EPOCH_UTC,
'scope': { 'scope': {'accepts': [{'ssurt': b'com,example,//http:/'}]},
'surt': 'http://(com,example,)/'
},
'seed': 'http://example.com', 'seed': 'http://example.com',
'starts_and_stops': [ 'starts_and_stops': [
{ {
@ -91,9 +89,7 @@ def test_basics():
'job_id': job.id, 'job_id': job.id,
'last_claimed': brozzler.EPOCH_UTC, 'last_claimed': brozzler.EPOCH_UTC,
'last_disclaimed': brozzler.EPOCH_UTC, 'last_disclaimed': brozzler.EPOCH_UTC,
'scope': { 'scope': {'accepts': [{'ssurt': b'org,example,//https:/'}]},
'surt': 'https://(org,example,)/',
},
'seed': 'https://example.org/', 'seed': 'https://example.org/',
'starts_and_stops': [ 'starts_and_stops': [
{ {
@ -443,8 +439,7 @@ def test_field_defaults():
brozzler.Site.table_ensure(rr) brozzler.Site.table_ensure(rr)
site = brozzler.Site(rr, {'seed': 'http://example.com/'}) site = brozzler.Site(rr, {'seed': 'http://example.com/'})
assert site.id is None assert site.id is None
assert site.scope assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/'}]}
assert site.scope['surt'] == 'http://(com,example,)/'
site.save() site.save()
assert site.id assert site.id
assert site.scope assert site.scope
@ -638,11 +633,15 @@ def test_completed_page():
'hops_from_seed': 0, 'hops_from_seed': 0,
'redirect_url':'http://example.com/b/', }) 'redirect_url':'http://example.com/b/', })
page.save() page.save()
assert site.scope == {'surt': 'http://(com,example,)/a/'} assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]}
frontier.completed_page(site, page) frontier.completed_page(site, page)
assert site.scope == {'surt': 'http://(com,example,)/b/'} assert site.scope == {'accepts': [
{'ssurt': b'com,example,//http:/a/'},
{'ssurt': b'com,example,//http:/b/'}]}
site.refresh() site.refresh()
assert site.scope == {'surt': 'http://(com,example,)/b/'} assert site.scope == {'accepts': [
{'ssurt': b'com,example,//http:/a/'},
{'ssurt': b'com,example,//http:/b/'}]}
assert page.brozzle_count == 1 assert page.brozzle_count == 1
assert page.claimed == False assert page.claimed == False
page.refresh() page.refresh()
@ -661,11 +660,11 @@ def test_completed_page():
'hops_from_seed': 0, 'hops_from_seed': 0,
'redirect_url':'http://example.com/a/x/', }) 'redirect_url':'http://example.com/a/x/', })
page.save() page.save()
assert site.scope == {'surt': 'http://(com,example,)/a/'} assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]}
frontier.completed_page(site, page) frontier.completed_page(site, page)
assert site.scope == {'surt': 'http://(com,example,)/a/'} assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]}
site.refresh() site.refresh()
assert site.scope == {'surt': 'http://(com,example,)/a/'} assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]}
assert page.brozzle_count == 1 assert page.brozzle_count == 1
assert page.claimed == False assert page.claimed == False
page.refresh() page.refresh()
@ -683,11 +682,11 @@ def test_completed_page():
'hops_from_seed': 1, 'hops_from_seed': 1,
'redirect_url':'http://example.com/d/', }) 'redirect_url':'http://example.com/d/', })
page.save() page.save()
assert site.scope == {'surt': 'http://(com,example,)/a/'} assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]}
frontier.completed_page(site, page) frontier.completed_page(site, page)
assert site.scope == {'surt': 'http://(com,example,)/a/'} assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]}
site.refresh() site.refresh()
assert site.scope == {'surt': 'http://(com,example,)/a/'} assert site.scope == {'accepts': [{'ssurt': b'com,example,//http:/a/'}]}
assert page.brozzle_count == 1 assert page.brozzle_count == 1
assert page.claimed == False assert page.claimed == False
page.refresh() page.refresh()
@ -727,7 +726,7 @@ def test_hashtag_seed():
site = brozzler.Site(rr, {'seed': 'http://example.org/'}) site = brozzler.Site(rr, {'seed': 'http://example.org/'})
brozzler.new_site(frontier, site) brozzler.new_site(frontier, site)
assert site.scope['surt'] == 'http://(org,example,)/' assert site.scope == {'accepts': [{'ssurt': b'org,example,//http:/'}]}
pages = list(frontier.site_pages(site.id)) pages = list(frontier.site_pages(site.id))
assert len(pages) == 1 assert len(pages) == 1
@ -738,7 +737,7 @@ def test_hashtag_seed():
site = brozzler.Site(rr, {'seed': 'http://example.org/#hash'}) site = brozzler.Site(rr, {'seed': 'http://example.org/#hash'})
brozzler.new_site(frontier, site) brozzler.new_site(frontier, site)
assert site.scope['surt'] == 'http://(org,example,)/' assert site.scope == {'accepts': [{'ssurt': b'org,example,//http:/'}]}
pages = list(frontier.site_pages(site.id)) pages = list(frontier.site_pages(site.id))
assert len(pages) == 1 assert len(pages) == 1