add license headers

bin/brozzle-page

@@ -1,5 +1,22 @@
 #!/usr/bin/env python
-# vim: set sw=4 et:
+#
+# brozzle-page - command line utility for brozzling a single page, i.e. opening
+# it in a browser, running some javascript behaviors, and printing outlinks
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 
 import argparse
 import os

bin/brozzler-new-job

@@ -1,4 +1,23 @@
 #!/usr/bin/env python
+#
+# brozzler-new-job - takes a yaml brozzler job configuration file, creates
+# job, sites, and pages objects in rethinkdb, which brozzler-workers will look
+# at and start crawling
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 
 import argparse
 import os

bin/brozzler-new-site

@@ -1,5 +1,22 @@
 #!/usr/bin/env python
-# vim: set sw=4 et:
+#
+# brozzler-new-site - takes a seed url and creates a site and page object in
+# rethinkdb, which brozzler-workers will look at and start crawling
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 
 import argparse
 import os

bin/brozzler-worker

@@ -1,5 +1,23 @@
 #!/usr/bin/env python
-# vim: set sw=4 et:
+#
+# brozzler-worker - main entrypoint for brozzler, gets sites and pages to
+# brozzle from rethinkdb, brozzles them
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 
 import argparse
 import os

brozzler/__init__.py

@@ -1,3 +1,22 @@
+#
+# brozzler/__init__.py - __init__.py for brozzler package, contains some common
+# code
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import json as _json
 import logging as _logging
 from pkg_resources import get_distribution as _get_distribution

brozzler/behaviors.d/default.js

@@ -1,8 +1,21 @@
-// vim:set sw=8 et:
-//
-// Scrolls to the bottom of the page, and clicks on embedded soundcloud
-// elements.
-//
+/*
+ * brozzler/behaviors.d/default.js - default behavior, scrolls to the bottom of
+ * the page and clicks on embedded soundcloud elements
+ *
+ * Copyright (C) 2014-2016 Internet Archive
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 var umbraAboveBelowOrOnScreen = function(e) {
         var eTop = e.getBoundingClientRect().top;

brozzler/behaviors.d/facebook.js

@@ -1,4 +1,21 @@
-// vim:set sw=8 et:
+/*
+ * brozzler/behaviors.d/facebook.js - facebook behavior, scrolls to the bottom
+ * of the page, clicks to expand images, a few other things
+ *
+ * Copyright (C) 2014-2016 Internet Archive
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 var umbraAboveBelowOrOnScreen = function(e) {
         var eTop = e.getBoundingClientRect().top;

brozzler/behaviors.d/flickr.js

@@ -1,4 +1,20 @@
-// vim:set sw=8 et:
+/*
+ * brozzler/behaviors.d/flickr.js - behavior for flickr.com
+ *
+ * Copyright (C) 2014-2016 Internet Archive
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 setInterval(function() { window.scrollBy(0,50); }, 100);
 

brozzler/behaviors.d/instagram.js

@@ -1,5 +1,20 @@
-// vim:set sw=8 et:
-//
+/*
+ * brozzler/behaviors.d/flickr.js - behavior for instagram
+ *
+ * Copyright (C) 2014-2016 Internet Archive
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 var umbraInstagramBehavior = {
         IDLE_TIMEOUT_SEC: 20,

brozzler/behaviors.d/marquette_edu.js

@@ -1,4 +1,21 @@
-// vim:set sw=8 et:
+/*
+ * brozzler/behaviors.d/flickr.js - behavior for marquette.edu, clicks to
+ * play/crawl embedded videos
+ *
+ * Copyright (C) 2014-2016 Internet Archive
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 var umbraState = {'idleSince':null};
 var umbraIntervalID = setInterval(umbraScrollInterval,50);

brozzler/behaviors.d/psu24.js

@@ -1,3 +1,21 @@
+/*
+ * brozzler/behaviors.d/flickr.js - behavior for marquette.edu, clicks to
+ * play/crawl embedded videos
+ *
+ * Copyright (C) 2014-2016 Internet Archive
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 var umbraBehavior = {
 	IDLE_TIMEOUT_SEC : 10,

brozzler/behaviors.d/simpleclicks.js.in

@@ -1,3 +1,22 @@
+/*
+ * brozzler/behaviors.d/simpleclicks.js.in - simpleclicks behavior template,
+ * clicks on elements matching templatized css selector
+ *
+ * Copyright (C) 2014-2016 Internet Archive
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 var umbraBehavior = {
 	IDLE_TIMEOUT_SEC : 10,
 	idleSince : null,

brozzler/behaviors.d/vimeo.js

@@ -1,4 +1,21 @@
-// vim:set sw=8 et:
+/*
+ * brozzler/behaviors.d/vimeo.js - behavior for vimeo.com, clicks to play/crawl
+ * videos
+ *
+ * Copyright (C) 2014-2016 Internet Archive
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
 
 var umbraState = {'idleSince':null};
 var umbraVideoElements = document.getElementsByTagName('video');

brozzler/behaviors.py

@@ -1,4 +1,21 @@
-# vim: set sw=4 et:
+#
+# brozzler/behaviors.py - manages behaviors, which are javascript scripts that
+# run in brozzled web pages
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 
 import json
 import itertools

brozzler/behaviors.yaml

@@ -1,3 +1,21 @@
+#
+# brozzler/behaviors.yaml - behavior configuration
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 # first matched behavior is used, so order matters here
 behaviors:
  -

brozzler/browser.py

@@ -1,5 +1,21 @@
-#!/usr/bin/env python
-# vim: set sw=4 et:
+#
+# brozzler/browser.py - classes responsible for running web browsers
+# (chromium/chromium) and browsing web pages in them
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 
 import logging
 import json

brozzler/frontier.py

@@ -1,3 +1,21 @@
+#
+# brozzler/frontier.py - RethinkDbFrontier manages crawl jobs, sites and pages
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import logging
 import brozzler
 import random

brozzler/job.py

@@ -1,3 +1,22 @@
+#
+# brozzler/job.py - Job class representing a brozzler crawl job, and functions
+# for setting up a job with supplied configuration
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import logging
 import brozzler
 import yaml

brozzler/robots.py

@@ -1,4 +1,20 @@
-# vim: set sw=4 et:
+#
+# brozzler/robots.py - robots.txt support
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
 
 import json
 import logging

brozzler/site.py

@@ -1,3 +1,21 @@
+#
+# brozzler/site.py - classes representing sites and pages
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import surt
 import json
 import logging

brozzler/worker.py

@@ -1,3 +1,23 @@
+#
+# brozzler/worker.py - BrozzlerWorker brozzles pages from the frontier, meaning
+# it runs youtube-dl on them, browses them and runs behaviors if appropriate,
+# scopes and adds outlinks to the frontier
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import os
 import logging
 import brozzler

setup.py

@@ -1,3 +1,21 @@
+#
+# setup.py - brozzler setup script
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import setuptools
 import glob
 

webconsole/brozzler-webconsole/__init__.py

@@ -1,3 +1,22 @@
+#
+# brozzler-webconsole/__init__.py - flask app for brozzler web console, defines
+# api endspoints etc
+#
+# Copyright (C) 2014-2016 Internet Archive
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import flask
 import rethinkstuff
 import json

webconsole/brozzler-webconsole/static/js/app.js

@@ -1,3 +1,21 @@
+/*
+ * brozzler-webconsole/static/js/app.js - brozzler console angularjs code
+ *
+ * Copyright (C) 2014-2016 Internet Archive
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
 "use strict";
 
 var brozzlerConsoleApp = angular.module("brozzlerConsoleApp", [