Add wait time option and add handling for reddit_pool

This commit is contained in:
Omar Roth 2018-03-03 15:26:47 -06:00
parent 84c8fb9b99
commit 323d27a5f6
2 changed files with 20 additions and 4 deletions

View File

@ -252,8 +252,8 @@ end
def make_client(url, context) def make_client(url, context)
client = HTTP::Client.new(url, context) client = HTTP::Client.new(url, context)
client.read_timeout = 10.seconds client.read_timeout = 30.seconds
client.connect_timeout = 10.seconds client.connect_timeout = 30.seconds
return client return client
end end

View File

@ -21,7 +21,7 @@ require "xml"
require "./helpers" require "./helpers"
threads = 10 threads = 10
sleep_time = 1.0
Kemal.config.extra_options do |parser| Kemal.config.extra_options do |parser|
parser.banner = "Usage: invidious [arguments]" parser.banner = "Usage: invidious [arguments]"
parser.on("-t THREADS", "--threads=THREADS", "Number of threads for crawling (default: 10)") do |number| parser.on("-t THREADS", "--threads=THREADS", "Number of threads for crawling (default: 10)") do |number|
@ -32,6 +32,14 @@ Kemal.config.extra_options do |parser|
exit exit
end end
end end
parser.on("-w TIME", "--wait=TIME", "Time to wait between server requests in seconds (default: 1 second)") do |number|
begin
sleep_time = number.to_f64
rescue ex
puts "TIME must be integer or float"
exit
end
end
end end
Kemal::CLI.new Kemal::CLI.new
@ -108,6 +116,8 @@ threads.times do
end end
end end
sleep sleep_time.seconds
youtube_pool << client youtube_pool << client
end end
end end
@ -118,8 +128,14 @@ threads.times do
loop do loop do
client = get_client(reddit_pool) client = get_client(reddit_pool)
begin
client.get("/") client.get("/")
sleep 10.seconds rescue ex
reddit_pool << make_client(URI.parse("https://api.reddit.com"), CONTEXT)
next
end
sleep sleep_time.seconds
reddit_pool << client reddit_pool << client
end end