Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 08:39:59 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

fix exception happening now that we have binary data in rethinkdb (the cookie db) "TypeError: <binary, 7168 bytes, '53 51 4c 69 74 65...'> is not JSON serializable"

Browse Source
This commit is contained in:
Noah Levitt 2016-08-05 17:12:22 -07:00
parent a211cc0514
commit ba6b342e28
2 changed files with 12 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
brozzler/webconsole/__init__.py
View File

@ -34,6 +34,7 @@ import os
import importlib import importlib
import rethinkdb import rethinkdb
import yaml import yaml
import base64
# flask does its own logging config # flask does its own logging config
# logging.basicConfig( # logging.basicConfig(
@ -120,8 +121,10 @@ def page_yaml(page_id):
@app.route("/api/sites/<site_id>") @app.route("/api/sites/<site_id>")
@app.route("/api/site/<site_id>") @app.route("/api/site/<site_id>")
def site(site_id): def site(site_id):
site_ = r.table("sites").get(site_id).run() s = r.table("sites").get(site_id).run()
return flask.jsonify(site_) if "cookie_db" in s:
s["cookie_db"] = base64.b64encode(s["cookie_db"]).decode("ascii")
return flask.jsonify(s)
@app.route("/api/sites/<site_id>/yaml") @app.route("/api/sites/<site_id>/yaml")
@app.route("/api/site/<site_id>/yaml") @app.route("/api/site/<site_id>/yaml")
@ -139,8 +142,12 @@ def stats(bucket):
@app.route("/api/jobs/<int:job_id>/sites") @app.route("/api/jobs/<int:job_id>/sites")
@app.route("/api/job/<int:job_id>/sites") @app.route("/api/job/<int:job_id>/sites")
def sites(job_id): def sites(job_id):
sites_ = r.table("sites").get_all(job_id, index="job_id").run() sites_ = list(r.table("sites").get_all(job_id, index="job_id").run())
return flask.jsonify(sites=list(sites_)) # TypeError: <binary, 7168 bytes, '53 51 4c 69 74 65...'> is not JSON serializable
for s in sites_:
if "cookie_db" in s:
s["cookie_db"] = base64.b64encode(s["cookie_db"]).decode("ascii")
return flask.jsonify(sites=sites_)
@app.route("/api/jobs/<int:job_id>") @app.route("/api/jobs/<int:job_id>")
@app.route("/api/job/<int:job_id>") @app.route("/api/job/<int:job_id>")

2
setup.py
View File

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.1b6.dev68', version='1.1b6.dev69',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',