Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-09-20 12:44:40 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'ARI-5460' into qa

Browse source
This commit is contained in:
Barbara Miller 2017-08-09 16:24:13 -07:00
commit 44e2b8579a
2 changed files with 16 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

13
brozzler/js-templates/instagram.js
View file

@ -26,14 +26,23 @@ var umbraInstagramBehavior = {
previousBigImage: null, previousBigImage: null,
intervalFunc: function() { intervalFunc: function() {
if (this.state === "loading-thumbs") { if (this.state === "loading-thumbs") {
var signUpButton = document.querySelectorAll("span._lilm5");
if (signUpButton.length > 0) {
console.log("clicking sign up button");
signUpButton[0].click();
this.idleSince = null;
return;
}
if (window.scrollY + window.innerHeight < document.documentElement.scrollHeight) { if (window.scrollY + window.innerHeight < document.documentElement.scrollHeight) {
window.scrollBy(0, 200); window.scrollBy(0, 200);
this.idleSince = null; this.idleSince = null;
return; return;
} }
var moreButtons = document.querySelectorAll("a._oidfu"); var moreButtons = document.querySelectorAll("a._1cr2e._epyes");
if (moreButtons.length > 0) { if (moreButtons.length > 0) {
console.log("clicking load more button"); console.log("clicking load more button");
moreButtons[0].click(); moreButtons[0].click();
@ -56,7 +65,7 @@ var umbraInstagramBehavior = {
} }
if (this.state === "clicking-first-thumb") { if (this.state === "clicking-first-thumb") {
var images = document.querySelectorAll("div._ovg3g"); var images = document.querySelectorAll("div._si7dy");
if (images && images !== "undefined") { if (images && images !== "undefined") {
this.imageCount = images.length; this.imageCount = images.length;
if (images.length > 0) { if (images.length > 0) {

9
brozzler/worker.py
View file

@ -317,7 +317,7 @@ class BrozzlerWorker:
return full_jpeg, thumb_jpeg return full_jpeg, thumb_jpeg
def brozzle_page(self, browser, site, page, on_screenshot=None, def brozzle_page(self, browser, site, page, on_screenshot=None,
enable_youtube_dl=True): on_request=None, enable_youtube_dl=True):
self.logger.info("brozzling {}".format(page)) self.logger.info("brozzling {}".format(page))
if enable_youtube_dl: if enable_youtube_dl:
try: try:
@ -347,7 +347,8 @@ class BrozzlerWorker:
if self._needs_browsing(page, ydl_spy): if self._needs_browsing(page, ydl_spy):
self.logger.info('needs browsing: %s', page) self.logger.info('needs browsing: %s', page)
outlinks = self._browse_page(browser, site, page, on_screenshot) outlinks = self._browse_page(browser, site, page, on_screenshot,
on_request)
return outlinks return outlinks
else: else:
if not self._already_fetched(page, ydl_spy): if not self._already_fetched(page, ydl_spy):
@ -357,7 +358,7 @@ class BrozzlerWorker:
self.logger.info('already fetched: %s', page) self.logger.info('already fetched: %s', page)
return [] return []
def _browse_page(self, browser, site, page, on_screenshot=None): def _browse_page(self, browser, site, page, on_screenshot=None, on_request=None):
def _on_screenshot(screenshot_png): def _on_screenshot(screenshot_png):
if on_screenshot: if on_screenshot:
on_screenshot(screenshot_png) on_screenshot(screenshot_png)
@ -413,7 +414,7 @@ class BrozzlerWorker:
username=site.get('username'), password=site.get('password'), username=site.get('username'), password=site.get('password'),
user_agent=site.get('user_agent'), user_agent=site.get('user_agent'),
on_screenshot=_on_screenshot, on_response=_on_response, on_screenshot=_on_screenshot, on_response=_on_response,
hashtags=page.hashtags, on_request=on_request, hashtags=page.hashtags,
skip_extract_outlinks=self._skip_extract_outlinks, skip_extract_outlinks=self._skip_extract_outlinks,
skip_visit_hashtags=self._skip_visit_hashtags) skip_visit_hashtags=self._skip_visit_hashtags)
if final_page_url != page.url: if final_page_url != page.url: