Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-07-20 13:38:49 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix bug in test, add another one

Browse source
This commit is contained in:
Noah Levitt 2018-06-22 16:10:23 -05:00
parent aeb7c3f825
commit c52c16c260
1 changed files with 10 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

14
tests/test_units.py
View file

@ -74,9 +74,9 @@ def test_robots_http_statuses():
500, 501, 502, 503, 504, 505): 500, 501, 502, 503, 504, 505):
class Handler(http.server.BaseHTTPRequestHandler): class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self): def do_GET(self):
response = (b'HTTP/1.1 %s Meaningless message\r\n' response = (('HTTP/1.1 %s Meaningless message\r\n'
+ b'Content-length: 0\r\n' + 'Content-length: 0\r\n'
+ b'\r\n') % status + '\r\n') % status).encode('utf-8')
self.connection.sendall(response) self.connection.sendall(response)
# self.send_response(status) # self.send_response(status)
# self.end_headers() # self.end_headers()
@ -93,7 +93,7 @@ def test_robots_http_statuses():
httpd.server_close() httpd.server_close()
httpd_thread.join() httpd_thread.join()
def test_robots_empty_respone(): def test_robots_empty_response():
class Handler(http.server.BaseHTTPRequestHandler): class Handler(http.server.BaseHTTPRequestHandler):
def do_GET(self): def do_GET(self):
self.connection.shutdown(socket.SHUT_RDWR) self.connection.shutdown(socket.SHUT_RDWR)
@ -143,6 +143,12 @@ def test_robots_dns_failure():
site = brozzler.Site(None, {'seed': url}) site = brozzler.Site(None, {'seed': url})
assert brozzler.is_permitted_by_robots(site, url) assert brozzler.is_permitted_by_robots(site, url)
def test_robots_connection_failure():
# .invalid. is guaranteed nonexistent per rfc 6761
url = 'http://localhost:4/' # nobody listens on port 4
site = brozzler.Site(None, {'seed': url})
assert brozzler.is_permitted_by_robots(site, url)
def test_scoping(): def test_scoping():
test_scope = yaml.load(''' test_scope = yaml.load('''
max_hops: 100 max_hops: 100