iv-org/instances-api
1
0
Fork 0
mirror of https://github.com/iv-org/instances-api.git synced 2025-03-13 09:46:34 -04:00
Code Issues Packages Projects Releases Wiki Activity

Rewrite parser for new instance list fmt

Browse Source 
iv-org/documentation#76
This commit is contained in:
syeopite 2021-06-19 07:09:48 -07:00
parent c4da7f353f
commit fc43f45e30
No known key found for this signature in database
GPG Key ID: 6FA616E5A5294A82
4 changed files with 181 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

125
src/fetch.cr Normal file
View File

@ -0,0 +1,125 @@
# Fetch a country's emoji flag from their country code (ISO 3166 alpha-2).
#
# A flag is made out of two regional indicator symbols.
# So in order to convert from an ISO 3166 alpha-2 code into unicode we'll have to
# add a specific offset to each character to make them into the required regional
# indicator symbols. This offset is exactly 0x1f1a5.
#
# Reference implementation https://schinckel.net/2015/10/29/unicode-flags-in-python/
private def fetch_flag(country_code)
return country_code.codepoints.map { |codepoint| (codepoint + 0x1f1a5).chr }.join("")
end
# Extracts the nested modified information containing source url and changes.
private def extract_modified_information(modified_hash)
if modified = modified_hash.as_h?
return Modified.new(
source: modified["source"].as_s,
changes: modified["changes"].as_s
)
end
end
# Extracts information common to all instance types.
private def extract_prerequisites(instance_data)
uri = URI.parse(instance_data["url"].to_s)
host = uri.host
# Fetch country data
region = instance_data["country"].to_s
flag = fetch_flag(region)
privacy_policy = instance_data["privacy_policy"].as_s?
owner = {name: instance_data["owner"].to_s.split("/")[-1].to_s, url: instance_data["owner"].as_s}
modified = extract_modified_information(instance_data["modified"])
notes = instance_data["notes"].as_a?
mirrors = [] of Mirrors
instance_data["mirrors"].as_a?.try &.each do |m|
mirrors << Mirrors.new(
url: m["url"].as_s,
region: m["country"].as_s,
flag: fetch_flag(m["country"].as_s)
)
end
return uri, host, region, flag, privacy_policy, owner, modified, notes, mirrors
end
def prepare_http_instance(instance_data, instances_storage, monitors)
uri, host, region, flag, privacy_policy, owner, modified, notes, mirrors = extract_prerequisites(instance_data)
# Fetch status information
if status = instance_data["status"].as_h?
status_url = status["url"].as_s
else
status_url = nil
end
ddos_mitm_protection = instance_data["ddos_mitm_protection"].as_s?
client = HTTP::Client.new(uri)
client.connect_timeout = 5.seconds
client.read_timeout = 5.seconds
begin
stats = JSON.parse(client.get("/api/v1/stats").body)
rescue ex
stats = nil
end
monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]?
return {
region: region,
flag: flag,
stats: stats,
type: "https",
uri: uri.to_s,
status_url: status_url,
privacy_policy: privacy_policy,
ddos_mitm_protection: ddos_mitm_protection,
owner: owner,
modified: modified,
mirrors: mirrors,
notes: notes,
monitor: monitor || instances_storage[host]?.try &.[:monitor]?,
}
end
def prepare_onion_instance(instance_data, instances_storage)
uri, host, region, flag, privacy_policy, owner, modified, notes, mirrors = extract_prerequisites(instance_data)
associated_clearnet_instance = instance_data["associated_clearnet_instance"].as_s?
if CONFIG["fetch_onion_instance_stats"]?
begin
args = Process.parse_arguments("--socks5-hostname '#{CONFIG["tor_sock_proxy_address"]}:#{CONFIG["tor_sock_proxy_port"]}' 'http://#{uri.host}/api/v1/stats'")
response = nil
Process.run("curl", args: args) do |result|
data = result.output.read_line
response = JSON.parse(data)
end
stats = response
rescue ex
stats = nil
end
else
stats = nil
end
return {
region: region,
flag: flag,
stats: stats,
type: "onion",
uri: uri.to_s,
associated_clearnet_instance: associated_clearnet_instance,
privacy_policy: privacy_policy,
owner: owner,
modified: modified,
mirrors: mirrors,
notes: notes,
monitor: nil,
}
end

7
src/helpers/helpers.cr
View File

@ -1,6 +1,9 @@
require "yaml"
def load_config
config = YAML.parse(File.read("config.yml"))
return config
return YAML.parse(File.read("config.yml"))
end
def load_instance_yaml(contents)
return YAML.parse(contents)
end

103
src/instances.cr
View File

@ -18,6 +18,7 @@ require "http/client"
require "kemal"
require "uri"
require "./fetch.cr"
require "./helpers/*"
CONFIG = load_config()
@ -28,9 +29,41 @@ macro rendered(filename)
render "src/instances/views/#{{{filename}}}.ecr"
end
alias Instance = NamedTuple(flag: String?, region: String?, stats: JSON::Any?, type: String, uri: String, monitor: JSON::Any?)
# Nested data within instances
alias Owner = NamedTuple(name: String, url: String)
alias Modified = NamedTuple(source: String, changes: String)
alias Mirrors = NamedTuple(url: String, region: String, flag: String)
INSTANCES = {} of String => Instance
alias ClearNetInstance = NamedTuple(
flag: String,
region: String,
stats: JSON::Any?,
type: String,
uri: String,
status_url: String?,
privacy_policy: String?,
ddos_mitm_protection: String?,
owner: Owner,
modified: Modified?,
mirrors: Array(Mirrors)?,
notes: Array(YAML::Any)?,
monitor: JSON::Any?)
alias OnionInstance = NamedTuple(
flag: String,
region: String,
stats: JSON::Any?,
type: String,
uri: String,
associated_clearnet_instance: String?,
privacy_policy: String?,
owner: Owner,
modified: Modified?,
mirrors: Array(Mirrors)?,
notes: Array(YAML::Any)?,
monitor: JSON::Any?)
INSTANCES = {} of String => ClearNetInstance | OnionInstance
spawn do
loop do
@ -54,59 +87,23 @@ spawn do
break
end
end
begin
body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/iv-org/documentation/master/Invidious-Instances.md")).body
# Needs to be replaced once merged!
body = HTTP::Client.get(URI.parse("https://raw.githubusercontent.com/syeopite/documentation/alt-instance-list/instances.yaml")).body
rescue ex
body = ""
end
instances = {} of String => Instance
instance_yaml = load_instance_yaml(body)
body = body.split("### Blocked:")[0]
body.scan(/\[(?<host>[^ \]]+)\]\((?<uri>[^\)]+)\)( .(?<region>[\x{1f100}-\x{1f1ff}]{2}))?/mx).each do |md|
region = md["region"]?.try { |region| region.codepoints.map { |codepoint| (codepoint - 0x1f1a5).chr }.join("") }
flag = md["region"]?
instance_storage = {} of String => ClearNetInstance | OnionInstance
uri = URI.parse(md["uri"])
host = md["host"]
case type = host.split(".")[-1]
when "onion"
type = "onion"
if CONFIG["fetch_onion_instance_stats"]?
begin
args = Process.parse_arguments("--socks5-hostname '#{CONFIG["tor_sock_proxy_address"]}:#{CONFIG["tor_sock_proxy_port"]}' 'http://#{uri.host}/api/v1/stats'")
response = nil
Process.run("curl", args: args) do |result|
data = result.output.read_line
response = JSON.parse(data)
end
stats = response
rescue ex
stats = nil
end
end
when "i2p"
else
type = uri.scheme.not_nil!
client = HTTP::Client.new(uri)
client.connect_timeout = 5.seconds
client.read_timeout = 5.seconds
begin
stats = JSON.parse(client.get("/api/v1/stats").body)
rescue ex
stats = nil
end
end
monitor = monitors.try &.select { |monitor| monitor["name"].try &.as_s == host }[0]?
instances[host] = {flag: flag, region: region, stats: stats, type: type, uri: uri.to_s, monitor: monitor || instances[host]?.try &.[:monitor]?}
end
instance_yaml["instances"]["https"].as_a.each { |i| instance_storage[URI.parse(i["url"].to_s).host.not_nil!] = prepare_http_instance(i, instance_storage, monitors) }
instance_yaml["instances"]["onion"].as_a.each { |i| instance_storage[URI.parse(i["url"].to_s).host.not_nil!] = prepare_onion_instance(i, instance_storage) }
INSTANCES.clear
INSTANCES.merge! instances
INSTANCES.merge! instance_storage
sleep CONFIG["minutes_between_refresh"].as_i.minutes
end
@ -154,13 +151,13 @@ static_headers do |response, filepath, filestat|
end
SORT_PROCS = {
"health" => ->(name : String, instance : Instance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) },
"location" => ->(name : String, instance : Instance) { instance[:region]? || "ZZ" },
"name" => ->(name : String, instance : Instance) { name },
"signup" => ->(name : String, instance : Instance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 },
"type" => ->(name : String, instance : Instance) { instance[:type] },
"users" => ->(name : String, instance : Instance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) },
"version" => ->(name : String, instance : Instance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] },
"health" => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:monitor]?.try &.["30dRatio"]["ratio"].as_s.to_f || 0.0) },
"location" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:region]? || "ZZ" },
"name" => ->(name : String, instance : ClearNetInstance | OnionInstance) { name },
"signup" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? 0 : 1 } || 2 },
"type" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:type] },
"users" => ->(name : String, instance : ClearNetInstance | OnionInstance) { -(instance[:stats]?.try &.["usage"]?.try &.["users"]["total"].as_i || 0) },
"version" => ->(name : String, instance : ClearNetInstance | OnionInstance) { instance[:stats]?.try &.["software"]?.try &.["version"].as_s.try &.split("-", 2)[0].split(".").map { |a| -a.to_i } || [0, 0, 0] },
}
def sort_instances(instances, sort_by)

2
src/instances/views/index.ecr
View File

@ -118,7 +118,7 @@
<td><%= instance[:type] %></td>
<td><%= instance[:stats]?.try &.["usage"]?.try &.["users"]["total"] || "-" %></td>
<td><%= instance[:stats]?.try &.["openRegistrations"]?.try { |bool| bool.as_bool ? "✔" : "❌" } || "-" %></td>
<td><%= instance[:flag]? ? "#{instance[:flag]} #{instance[:region]}" : "-" %></td>
<td><%= instance[:region]? ? "#{instance[:flag]} #{instance[:region]}" : "-" %></td>
<td><%= instance[:monitor]?.try &.["30dRatio"]["ratio"] || "-" %></td>
</tr>
<% end %>