add error handling

This commit is contained in:
mithereal 2022-09-17 03:38:06 -07:00
parent 0e17984d6f
commit 03e46f81f8
12 changed files with 57 additions and 36 deletions

View File

@ -15,4 +15,4 @@ config :farside,
"weather",
"time"
],
recv_timeout: System.get_env("FARSIDE_TIMEOUT") || "8000"
recv_timeout: System.get_env("FARSIDE_TIMEOUT") || "8000"

View File

@ -3,4 +3,4 @@ import Config
config :farside,
port: System.get_env("FARSIDE_PORT", nil),
services_json: System.get_env("FARSIDE_SERVICES_JSON", "services.json"),
services_json_data: System.get_env("FARSIDE_SERVICES_JSON_DATA") || ""
services_json_data: System.get_env("FARSIDE_SERVICES_JSON_DATA") || ""

View File

@ -110,11 +110,12 @@ defmodule Farside do
end
|> Enum.reject(fn x -> x == nil end)
service = Map.put(
service,
:instances,
instances
)
service =
Map.put(
service,
:instances,
instances
)
case Enum.count(service.instances) > 0 do
true -> Enum.random(service.instances)

View File

@ -23,7 +23,7 @@ defmodule Farside.Application do
maybe_loaded_children =
case is_nil(System.get_env("FARSIDE_TEST")) do
true ->
[{HealthyCheck, []},{UnHealthyCheck, []},{DeadCheck, []}]
[{HealthyCheck, []}, {UnHealthyCheck, []}, {DeadCheck, []}]
false ->
Logger.info("Skipping sync job setup...")
@ -45,7 +45,7 @@ defmodule Farside.Application do
{DynamicSupervisor, strategy: :one_for_one, name: :service_supervisor},
{Registry, keys: :unique, name: :instance},
{Registry, keys: :unique, name: :service},
{Registry, keys: :duplicate, name: :status, partitions: System.schedulers_online()},
{Registry, keys: :duplicate, name: :status, partitions: System.schedulers_online()}
] ++ maybe_loaded_children
opts = [strategy: :one_for_one, name: Farside.Supervisor]
@ -81,8 +81,6 @@ defmodule Farside.Application do
|> HealthyCheck.load()
end
LastUpdated.value(DateTime.utc_now())
response
end
end

View File

@ -3,6 +3,7 @@ defmodule Farside.Server.DeadCheck do
Module to check/validate the instance list only for servers with empty instance list every 90 secs, if a sync/check process isnt already running
"""
use Task
alias Farside.LastUpdated
def child_spec(args) do
%{
@ -26,6 +27,8 @@ defmodule Farside.Server.DeadCheck do
end
def run() do
LastUpdated.value(DateTime.utc_now())
Registry.dispatch(:status, "dead", fn entries ->
for {pid, _} <- entries, do: GenServer.cast(pid, :check)
end)

View File

@ -3,6 +3,7 @@ defmodule Farside.Server.HealthyCheck do
Module to validate healthy servers
"""
use Task
alias Farside.LastUpdated
def child_spec(args) do
%{
@ -31,10 +32,13 @@ defmodule Farside.Server.HealthyCheck do
GenServer.cast(pid, :check)
end
end)
params
end
def run() do
LastUpdated.value(DateTime.utc_now())
Registry.dispatch(:status, "healthy", fn entries ->
for {pid, url} <- entries do
GenServer.cast(pid, :check)

View File

@ -27,7 +27,6 @@ defmodule Farside.Http do
end
def request(url, type) do
cond do
System.get_env("FARSIDE_TEST") ->
:good
@ -41,6 +40,10 @@ defmodule Farside.Http do
Logger.info("Type: #{type}, Response: [#{n}], Url: #{url}")
:good
nil ->
Logger.error("Type: #{type}, Response: [408], Url: #{url}")
:bad
n ->
Logger.error("Type: #{type}, Response: [#{n}], Url: #{url}")
:bad
@ -82,7 +85,8 @@ defmodule Farside.Http do
end
def test_service(service) do
url = service.url <> service.test_url
url = service.url <> service.test_url
test_url =
EEx.eval_string(
url,
@ -107,7 +111,7 @@ url = service.url <> service.test_url
unless is_nil(data) do
{_test_url, value, _service} = data
value
else
else
:bad
end
end

View File

@ -55,7 +55,6 @@ defmodule Farside.Instance do
{:stop, :normal, state}
end
@doc false
def via_tuple(id, registry \\ @registry_name) do
{:via, Registry, {registry, id}}
@ -67,12 +66,12 @@ defmodule Farside.Instance do
{:noreply, {names, refs}}
end
@impl true
def handle_info(:load, state) do
service = :ets.lookup(String.to_atom(state.type), :default)
{_, service} = List.first(service)
service.instances
|> Enum.each(fn url ->
initial_state = %{url: url, type: service.type, test_url: service.test_url}

View File

@ -31,11 +31,11 @@ defmodule Farside.Instance.Supervisor do
def start(opts \\ %{}) do
child_spec = {SERVER, opts}
{:ok,pid} = DynamicSupervisor.start_child(@name, child_spec)
{:ok, pid} = DynamicSupervisor.start_child(@name, child_spec)
send(pid, :load)
{:ok,pid}
{:ok, pid}
end
def stop(id) do

View File

@ -91,11 +91,18 @@ defmodule Farside.Service do
@impl true
def handle_cast(:check, state) do
reply = Http.test_service(state)
status = state.status ++ [reply]
max_queue = Application.get_env(:farside, :max_fail_rate, 50) + 5
status =
case Enum.count(status) < max_queue do
true -> status
false -> []
end
state = %{state | status: status}
state = %{state | last_update: DateTime.utc_now()}
@ -112,24 +119,28 @@ defmodule Farside.Service do
Registry.unregister_match(:status, unhealthy, state.url)
Registry.unregister_match(:status, dead, state.url)
if reply != :good do
filtered = Enum.reject(status, fn x -> x == :good end)
state =
if reply != :good do
filtered = Enum.reject(status, fn x -> x == :good end)
fails_before_death = Application.get_env(:farside, :max_fail_rate, 50)
fails_before_death = Application.get_env(:farside, :max_fail_rate, 50)
case Enum.count(filtered) < fails_before_death do
true ->
Registry.register(:status, "unhealthy", state.url)
Registry.register(:status, unhealthy, state.url)
case Enum.count(filtered) < fails_before_death do
true ->
Registry.register(:status, "unhealthy", state.url)
Registry.register(:status, unhealthy, state.url)
state
false ->
Registry.register(:status, "dead", state.url)
Registry.register(:status, dead, state.url)
false ->
Registry.register(:status, "dead", state.url)
Registry.register(:status, dead, state.url)
%{state | status: [:bad]}
end
else
Registry.register(:status, "healthy", state.url)
Registry.register(:status, healthy, state.url)
state
end
else
Registry.register(:status, "healthy", state.url)
Registry.register(:status, healthy, state.url)
end
{:noreply, state}
end

View File

@ -3,6 +3,7 @@ defmodule Farside.Server.UnHealthyCheck do
Module to check/validate the instance list only for servers with empty instance list every 90 secs, if a sync/check process isnt already running
"""
use Task
alias Farside.LastUpdated
def child_spec(args) do
%{
@ -26,6 +27,8 @@ defmodule Farside.Server.UnHealthyCheck do
end
def run() do
LastUpdated.value(DateTime.utc_now())
Registry.dispatch(:status, "unhealthy", fn entries ->
for {pid, _} <- entries, do: GenServer.cast(pid, :check)
end)

View File

@ -31,8 +31,7 @@ defmodule Farside.MixProject do
end
defp aliases do
[
]
[]
end
# Run "mix help deps" to learn about dependencies.
@ -46,7 +45,6 @@ defmodule Farside.MixProject do
]
end
defp description() do
"A redirecting service for FOSS alternative frontends."
end