Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 16:49:56 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Fix naming conventions.

Browse Source
This commit is contained in:
Adam Miller 2016-07-26 00:39:50 +00:00
parent 2029964a74
commit 77dabd4057
3 changed files with 22 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
brozzler/browser.py
View File

@ -129,26 +129,22 @@ class Browser:
def __exit__(self, *args): def __exit__(self, *args):
self.stop() self.stop()
def start(self, proxy=None, cookieDb=None): def start(self, proxy=None, cookie_db=None):
if not self._chrome_instance: if not self._chrome_instance:
# these can raise exceptions # these can raise exceptions
self.chrome_port = self._find_available_port() self.chrome_port = self._find_available_port()
self._work_dir = tempfile.TemporaryDirectory() self._work_dir = tempfile.TemporaryDirectory()
if cookieDb is not None: if cookie_db is not None:
cookieDir = os.sep.join([self._work_dir.name, "chrome-user-data","Default"]) cookie_dir = os.sep.join([self._work_dir.name, "chrome-user-data","Default"])
cookieLocation = os.sep.join([cookieDir,"Cookies"]) cookie_location = os.sep.join([cookie_dir,"Cookies"])
self.logger.debug("Cookie DB provided. Writing to: %s", cookieLocation) self.logger.debug("Cookie DB provided. Writing to: %s", cookie_location)
try: os.makedirs(cookie_dir, exist_ok=True)
os.makedirs(cookieDir)
except OSError as ex:
if ex.errno!=errno.EEXIST:
self.logger.error("Error creating cookie directory: %s", cookieDir, exc_info=True)
try: try:
with open(cookieLocation,'wb') as cookieFile: with open(cookie_location,'wb') as cookie_file:
cookieFile.write(cookieDb) cookie_file.write(cookie_db)
except EnvironmentError: except EnvironmentError:
self.logger.error("exception writing cookie file at: %s", cookieLocation, exc_info=True) self.logger.error("exception writing cookie file at: %s", cookie_location, exc_info=True)
self._chrome_instance = Chrome( self._chrome_instance = Chrome(
port=self.chrome_port, executable=self.chrome_exe, port=self.chrome_port, executable=self.chrome_exe,
@ -178,23 +174,23 @@ class Browser:
except: except:
self.logger.error("problem stopping", exc_info=True) self.logger.error("problem stopping", exc_info=True)
def read_cookie_db(self): def persist_and_read_cookie_db(self):
cookieLocation = os.sep.join([self._work_dir.name, "chrome-user-data","Default","Cookies"]) cookie_location = os.sep.join([self._work_dir.name, "chrome-user-data","Default","Cookies"])
self.logger.debug("Saving Cookie DB from: %s", cookieLocation) self.logger.debug("Saving Cookie DB from: %s", cookie_location)
try: try:
with sqlite3.connect(cookieLocation) as conn: with sqlite3.connect(cookie_location) as conn:
cur = conn.cursor() cur = conn.cursor()
cur.execute("UPDATE cookies SET persistent = 1") cur.execute("UPDATE cookies SET persistent = 1")
except sqlite3.Error: except sqlite3.Error:
self.logger.error("exception updating cookie DB", exc_info=True) self.logger.error("exception updating cookie DB", exc_info=True)
cookieDb=None cookie_db=None
try: try:
with open(cookieLocation, "rb") as cookieFile: with open(cookie_location, "rb") as cookie_file:
cookieDb=bytearray(cookieFile.read()) cookie_db=cookie_file.read()
except EnvironmentError: except EnvironmentError:
self.logger.error("exception reading from cookie DB file at: %s", cookieLocation, exc_info=True) self.logger.error("exception reading from cookie DB file at: %s", cookie_location, exc_info=True)
return cookieDb return cookie_db
def _find_available_port(self): def _find_available_port(self):
port_available = False port_available = False

4
brozzler/site.py
View File

@ -91,7 +91,7 @@ class Site(brozzler.BaseDictable):
enable_warcprox_features=False, reached_limit=None, enable_warcprox_features=False, reached_limit=None,
status="ACTIVE", claimed=False, start_time=None, status="ACTIVE", claimed=False, start_time=None,
last_disclaimed=_EPOCH_UTC, last_claimed_by=None, last_disclaimed=_EPOCH_UTC, last_claimed_by=None,
last_claimed=_EPOCH_UTC, metadata={}, remember_outlinks=None, cookieDb=None): last_claimed=_EPOCH_UTC, metadata={}, remember_outlinks=None, cookie_db=None):
self.seed = seed self.seed = seed
self.id = id self.id = id
@ -110,7 +110,7 @@ class Site(brozzler.BaseDictable):
self.last_claimed = last_claimed self.last_claimed = last_claimed
self.metadata = metadata self.metadata = metadata
self.remember_outlinks = remember_outlinks self.remember_outlinks = remember_outlinks
self.cookieDb = bytearray(cookieDb) if cookieDb is not None else None self.cookie_db = cookie_db
self.scope = scope or {} self.scope = scope or {}
if not "surt" in self.scope: if not "surt" in self.scope:

4
brozzler/worker.py
View File

@ -257,7 +257,7 @@ class BrozzlerWorker:
if self._needs_browsing(page, ydl_spy): if self._needs_browsing(page, ydl_spy):
self.logger.info('needs browsing: %s', page) self.logger.info('needs browsing: %s', page)
if not browser.is_running(): if not browser.is_running():
browser.start(proxy=self._proxy(site), cookieDb=site.cookieDb) browser.start(proxy=self._proxy(site), cookie_db=site.cookie_db)
outlinks = browser.browse_page( outlinks = browser.browse_page(
page.url, extra_headers=site.extra_headers(), page.url, extra_headers=site.extra_headers(),
on_screenshot=_on_screenshot, on_screenshot=_on_screenshot,
@ -312,7 +312,7 @@ class BrozzlerWorker:
page = self._frontier.claim_page(site, "%s:%s" % ( page = self._frontier.claim_page(site, "%s:%s" % (
socket.gethostname(), browser.chrome_port)) socket.gethostname(), browser.chrome_port))
outlinks = self.brozzle_page(browser, site, page) outlinks = self.brozzle_page(browser, site, page)
site.cookieDb=browser.read_cookie_db() site.cookie_db=browser.persist_and_read_cookie_db()
self._frontier.completed_page(site, page) self._frontier.completed_page(site, page)
self._frontier.scope_and_schedule_outlinks( self._frontier.scope_and_schedule_outlinks(
site, page, outlinks) site, page, outlinks)