Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-21 08:06:27 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Merge branch 'master' into qa

Browse Source 
* master:
  add tests of backwards compatibility handling of start/stop times and fix a bug or two
This commit is contained in:
Noah Levitt 2017-03-02 16:53:31 -08:00
commit d55dd0b26f
3 changed files with 39 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
brozzler/job.py
View File

@ -136,6 +136,8 @@ class Job(doublethink.Document):
"start": self.get("started"),
"stop": self.get("finished")}]
del self["started"]
if "finished" in self:
del self["finished"]
else:
self.starts_and_stops = [
{"start":doublethink.utcnow(),"stop":None}]

10
brozzler/site.py
View File

@ -102,12 +102,14 @@ class Site(doublethink.Document):
self.last_claimed = self.get('last_claimed', _EPOCH_UTC)
if not self.get('starts_and_stops'):
if self.get('start_time'): # backward compatibility
self.starts_and_stops = [{"start":start_time,"stop":None}]
if self.get('status') != "ACTIVE":
self.starts_and_stops[0]["stop"] = self.last_disclaimed
self.starts_and_stops = [{
'start':self.get('start_time'),'stop':None}]
if self.get('status') != 'ACTIVE':
self.starts_and_stops[0]['stop'] = self.last_disclaimed
del self['start_time']
else:
self.starts_and_stops = [
{"start":doublethink.utcnow(),"stop":None}]
{'start':doublethink.utcnow(),'stop':None}]
if not self.scope:
self.scope = {}
if not 'surt' in self.scope:

31
tests/test_units.py
View File

@ -26,6 +26,7 @@ import brozzler.chrome
import socket
import logging
import yaml
import datetime
@pytest.fixture(scope='module')
def httpd(request):
@ -107,3 +108,33 @@ blocks:
assert site.is_in_scope(
'https://www.youtube.com/watch?v=dUIn5OAPS5s', yt_user_page)
def test_start_stop_backwards_compat():
site = brozzler.Site(None, {'seed': 'http://example.com/'})
assert len(site.starts_and_stops) == 1
assert site.starts_and_stops[0]['start']
assert site.starts_and_stops[0]['stop'] is None
assert not 'start_time' in site
site = brozzler.Site(None, {
'seed': 'http://example.com/',
'start_time': datetime.datetime(2017,1,1)})
assert len(site.starts_and_stops) == 1
assert site.starts_and_stops[0]['start'] == datetime.datetime(2017, 1, 1)
assert site.starts_and_stops[0]['stop'] is None
assert not 'start_time' in site
job = brozzler.Job(None, {'seeds': [{'url':'https://example.com/'}]})
assert job.starts_and_stops[0]['start']
assert job.starts_and_stops[0]['stop'] is None
assert not 'started' in job
assert not 'finished' in job
job = brozzler.Job(None, {
'seeds': [{'url':'https://example.com/'}],
'started': datetime.datetime(2017, 1, 1),
'finished': datetime.datetime(2017, 1, 2)})
assert job.starts_and_stops[0]['start'] == datetime.datetime(2017, 1, 1)
assert job.starts_and_stops[0]['stop'] == datetime.datetime(2017, 1, 2)
assert not 'started' in job
assert not 'finished' in job