Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 00:29:53 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

set in_progress=0 for completed url

Browse Source
This commit is contained in:
Noah Levitt 2015-07-11 13:24:38 -07:00
parent 610f9c8cf4
commit 384120928c
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bin/brozzler-hq
View File

@ -74,6 +74,11 @@ class BrozzlerHQDb:
else:
return None
def completed(self, crawl_url):
cursor = self._conn.cursor()
cursor.execute("update brozzler_urls set in_progress=0 where id=?", (crawl_url.id,))
self._conn.commit()
def new_site(self, site):
cursor = self._conn.cursor()
cursor.execute("insert into brozzler_sites (site_json) values (?)", (site.to_json(),))
@ -172,6 +177,7 @@ class BrozzlerHQ:
msg = q.get(block=False)
completed_url = umbra.CrawlUrl(**msg.payload)
msg.ack()
self._db.completed(completed_url)
counts = {"added":0,"updated":0,"rejected":0}
if completed_url.outlinks: