From 64cde55ba97359075b3b2d00c188abc294feed8b Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 30 Sep 2016 17:18:53 -0700 Subject: [PATCH 1/3] Noah's ExtraHTTPHeaders encoding fix --- brozzler/browser.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/brozzler/browser.py b/brozzler/browser.py index 92e3479..f4be840 100644 --- a/brozzler/browser.py +++ b/brozzler/browser.py @@ -457,8 +457,11 @@ __brzl_compileOutlinks(window).join(' '); self.send_to_chrome(method="Debugger.enable") self.send_to_chrome(method="Runtime.enable") - if self.extra_headers: - self.send_to_chrome(method="Network.setExtraHTTPHeaders", params={"headers":self.extra_headers}) + headers = self.extra_headers or {} + headers['Accept-Encoding'] = 'gzip, deflate' + self.send_to_chrome( + method="Network.setExtraHTTPHeaders", + params={"headers":headers}) # disable google analytics, see _handle_message() where breakpoint is caught "Debugger.paused" self.send_to_chrome(method="Debugger.setBreakpointByUrl", params={"lineNumber": 1, "urlRegex":"https?://www.google-analytics.com/analytics.js"}) From b9668e4205ced2d2c79f05af07ebf246f52f3fb2 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Fri, 30 Sep 2016 17:18:53 -0700 Subject: [PATCH 2/3] Noah's ExtraHTTPHeaders encoding fix --- brozzler/browser.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/brozzler/browser.py b/brozzler/browser.py index 1b3301b..4b6a890 100644 --- a/brozzler/browser.py +++ b/brozzler/browser.py @@ -459,8 +459,11 @@ __brzl_compileOutlinks(window).join(' '); self.send_to_chrome(method="Debugger.enable") self.send_to_chrome(method="Runtime.enable") - if self.extra_headers: - self.send_to_chrome(method="Network.setExtraHTTPHeaders", params={"headers":self.extra_headers}) + headers = self.extra_headers or {} + headers['Accept-Encoding'] = 'gzip, deflate' + self.send_to_chrome( + method="Network.setExtraHTTPHeaders", + params={"headers":headers}) if self.user_agent: self.send_to_chrome(method="Network.setUserAgentOverride", params={"userAgent": self.user_agent}) From 105deb3bd447f4018d7bc1d86d8471514a09f64b Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Mon, 17 Oct 2016 12:21:14 -0700 Subject: [PATCH 3/3] Accept-Encoding identity --- brozzler/browser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brozzler/browser.py b/brozzler/browser.py index 4b6a890..5eaf4c6 100644 --- a/brozzler/browser.py +++ b/brozzler/browser.py @@ -460,7 +460,7 @@ __brzl_compileOutlinks(window).join(' '); self.send_to_chrome(method="Runtime.enable") headers = self.extra_headers or {} - headers['Accept-Encoding'] = 'gzip, deflate' + headers['Accept-Encoding'] = 'identity' self.send_to_chrome( method="Network.setExtraHTTPHeaders", params={"headers":headers})