mirror of
https://github.com/benbusby/farside.git
synced 2025-04-19 23:05:56 -04:00
add error handling
This commit is contained in:
parent
0e17984d6f
commit
03e46f81f8
@ -15,4 +15,4 @@ config :farside,
|
||||
"weather",
|
||||
"time"
|
||||
],
|
||||
recv_timeout: System.get_env("FARSIDE_TIMEOUT") || "8000"
|
||||
recv_timeout: System.get_env("FARSIDE_TIMEOUT") || "8000"
|
||||
|
@ -3,4 +3,4 @@ import Config
|
||||
config :farside,
|
||||
port: System.get_env("FARSIDE_PORT", nil),
|
||||
services_json: System.get_env("FARSIDE_SERVICES_JSON", "services.json"),
|
||||
services_json_data: System.get_env("FARSIDE_SERVICES_JSON_DATA") || ""
|
||||
services_json_data: System.get_env("FARSIDE_SERVICES_JSON_DATA") || ""
|
||||
|
@ -110,11 +110,12 @@ defmodule Farside do
|
||||
end
|
||||
|> Enum.reject(fn x -> x == nil end)
|
||||
|
||||
service = Map.put(
|
||||
service,
|
||||
:instances,
|
||||
instances
|
||||
)
|
||||
service =
|
||||
Map.put(
|
||||
service,
|
||||
:instances,
|
||||
instances
|
||||
)
|
||||
|
||||
case Enum.count(service.instances) > 0 do
|
||||
true -> Enum.random(service.instances)
|
||||
|
@ -23,7 +23,7 @@ defmodule Farside.Application do
|
||||
maybe_loaded_children =
|
||||
case is_nil(System.get_env("FARSIDE_TEST")) do
|
||||
true ->
|
||||
[{HealthyCheck, []},{UnHealthyCheck, []},{DeadCheck, []}]
|
||||
[{HealthyCheck, []}, {UnHealthyCheck, []}, {DeadCheck, []}]
|
||||
|
||||
false ->
|
||||
Logger.info("Skipping sync job setup...")
|
||||
@ -45,7 +45,7 @@ defmodule Farside.Application do
|
||||
{DynamicSupervisor, strategy: :one_for_one, name: :service_supervisor},
|
||||
{Registry, keys: :unique, name: :instance},
|
||||
{Registry, keys: :unique, name: :service},
|
||||
{Registry, keys: :duplicate, name: :status, partitions: System.schedulers_online()},
|
||||
{Registry, keys: :duplicate, name: :status, partitions: System.schedulers_online()}
|
||||
] ++ maybe_loaded_children
|
||||
|
||||
opts = [strategy: :one_for_one, name: Farside.Supervisor]
|
||||
@ -81,8 +81,6 @@ defmodule Farside.Application do
|
||||
|> HealthyCheck.load()
|
||||
end
|
||||
|
||||
LastUpdated.value(DateTime.utc_now())
|
||||
|
||||
response
|
||||
end
|
||||
end
|
||||
|
@ -3,6 +3,7 @@ defmodule Farside.Server.DeadCheck do
|
||||
Module to check/validate the instance list only for servers with empty instance list every 90 secs, if a sync/check process isnt already running
|
||||
"""
|
||||
use Task
|
||||
alias Farside.LastUpdated
|
||||
|
||||
def child_spec(args) do
|
||||
%{
|
||||
@ -26,6 +27,8 @@ defmodule Farside.Server.DeadCheck do
|
||||
end
|
||||
|
||||
def run() do
|
||||
LastUpdated.value(DateTime.utc_now())
|
||||
|
||||
Registry.dispatch(:status, "dead", fn entries ->
|
||||
for {pid, _} <- entries, do: GenServer.cast(pid, :check)
|
||||
end)
|
||||
|
@ -3,6 +3,7 @@ defmodule Farside.Server.HealthyCheck do
|
||||
Module to validate healthy servers
|
||||
"""
|
||||
use Task
|
||||
alias Farside.LastUpdated
|
||||
|
||||
def child_spec(args) do
|
||||
%{
|
||||
@ -31,10 +32,13 @@ defmodule Farside.Server.HealthyCheck do
|
||||
GenServer.cast(pid, :check)
|
||||
end
|
||||
end)
|
||||
|
||||
params
|
||||
end
|
||||
|
||||
def run() do
|
||||
LastUpdated.value(DateTime.utc_now())
|
||||
|
||||
Registry.dispatch(:status, "healthy", fn entries ->
|
||||
for {pid, url} <- entries do
|
||||
GenServer.cast(pid, :check)
|
||||
|
@ -27,7 +27,6 @@ defmodule Farside.Http do
|
||||
end
|
||||
|
||||
def request(url, type) do
|
||||
|
||||
cond do
|
||||
System.get_env("FARSIDE_TEST") ->
|
||||
:good
|
||||
@ -41,6 +40,10 @@ defmodule Farside.Http do
|
||||
Logger.info("Type: #{type}, Response: [#{n}], Url: #{url}")
|
||||
:good
|
||||
|
||||
nil ->
|
||||
Logger.error("Type: #{type}, Response: [408], Url: #{url}")
|
||||
:bad
|
||||
|
||||
n ->
|
||||
Logger.error("Type: #{type}, Response: [#{n}], Url: #{url}")
|
||||
:bad
|
||||
@ -82,7 +85,8 @@ defmodule Farside.Http do
|
||||
end
|
||||
|
||||
def test_service(service) do
|
||||
url = service.url <> service.test_url
|
||||
url = service.url <> service.test_url
|
||||
|
||||
test_url =
|
||||
EEx.eval_string(
|
||||
url,
|
||||
@ -107,7 +111,7 @@ url = service.url <> service.test_url
|
||||
unless is_nil(data) do
|
||||
{_test_url, value, _service} = data
|
||||
value
|
||||
else
|
||||
else
|
||||
:bad
|
||||
end
|
||||
end
|
||||
|
@ -55,7 +55,6 @@ defmodule Farside.Instance do
|
||||
{:stop, :normal, state}
|
||||
end
|
||||
|
||||
|
||||
@doc false
|
||||
def via_tuple(id, registry \\ @registry_name) do
|
||||
{:via, Registry, {registry, id}}
|
||||
@ -67,12 +66,12 @@ defmodule Farside.Instance do
|
||||
{:noreply, {names, refs}}
|
||||
end
|
||||
|
||||
|
||||
@impl true
|
||||
def handle_info(:load, state) do
|
||||
service = :ets.lookup(String.to_atom(state.type), :default)
|
||||
|
||||
{_, service} = List.first(service)
|
||||
|
||||
service.instances
|
||||
|> Enum.each(fn url ->
|
||||
initial_state = %{url: url, type: service.type, test_url: service.test_url}
|
||||
|
@ -31,11 +31,11 @@ defmodule Farside.Instance.Supervisor do
|
||||
def start(opts \\ %{}) do
|
||||
child_spec = {SERVER, opts}
|
||||
|
||||
{:ok,pid} = DynamicSupervisor.start_child(@name, child_spec)
|
||||
{:ok, pid} = DynamicSupervisor.start_child(@name, child_spec)
|
||||
|
||||
send(pid, :load)
|
||||
|
||||
{:ok,pid}
|
||||
{:ok, pid}
|
||||
end
|
||||
|
||||
def stop(id) do
|
||||
|
@ -91,11 +91,18 @@ defmodule Farside.Service do
|
||||
|
||||
@impl true
|
||||
def handle_cast(:check, state) do
|
||||
|
||||
reply = Http.test_service(state)
|
||||
|
||||
status = state.status ++ [reply]
|
||||
|
||||
max_queue = Application.get_env(:farside, :max_fail_rate, 50) + 5
|
||||
|
||||
status =
|
||||
case Enum.count(status) < max_queue do
|
||||
true -> status
|
||||
false -> []
|
||||
end
|
||||
|
||||
state = %{state | status: status}
|
||||
|
||||
state = %{state | last_update: DateTime.utc_now()}
|
||||
@ -112,24 +119,28 @@ defmodule Farside.Service do
|
||||
Registry.unregister_match(:status, unhealthy, state.url)
|
||||
Registry.unregister_match(:status, dead, state.url)
|
||||
|
||||
if reply != :good do
|
||||
filtered = Enum.reject(status, fn x -> x == :good end)
|
||||
state =
|
||||
if reply != :good do
|
||||
filtered = Enum.reject(status, fn x -> x == :good end)
|
||||
|
||||
fails_before_death = Application.get_env(:farside, :max_fail_rate, 50)
|
||||
fails_before_death = Application.get_env(:farside, :max_fail_rate, 50)
|
||||
|
||||
case Enum.count(filtered) < fails_before_death do
|
||||
true ->
|
||||
Registry.register(:status, "unhealthy", state.url)
|
||||
Registry.register(:status, unhealthy, state.url)
|
||||
case Enum.count(filtered) < fails_before_death do
|
||||
true ->
|
||||
Registry.register(:status, "unhealthy", state.url)
|
||||
Registry.register(:status, unhealthy, state.url)
|
||||
state
|
||||
|
||||
false ->
|
||||
Registry.register(:status, "dead", state.url)
|
||||
Registry.register(:status, dead, state.url)
|
||||
false ->
|
||||
Registry.register(:status, "dead", state.url)
|
||||
Registry.register(:status, dead, state.url)
|
||||
%{state | status: [:bad]}
|
||||
end
|
||||
else
|
||||
Registry.register(:status, "healthy", state.url)
|
||||
Registry.register(:status, healthy, state.url)
|
||||
state
|
||||
end
|
||||
else
|
||||
Registry.register(:status, "healthy", state.url)
|
||||
Registry.register(:status, healthy, state.url)
|
||||
end
|
||||
|
||||
{:noreply, state}
|
||||
end
|
||||
|
@ -3,6 +3,7 @@ defmodule Farside.Server.UnHealthyCheck do
|
||||
Module to check/validate the instance list only for servers with empty instance list every 90 secs, if a sync/check process isnt already running
|
||||
"""
|
||||
use Task
|
||||
alias Farside.LastUpdated
|
||||
|
||||
def child_spec(args) do
|
||||
%{
|
||||
@ -26,6 +27,8 @@ defmodule Farside.Server.UnHealthyCheck do
|
||||
end
|
||||
|
||||
def run() do
|
||||
LastUpdated.value(DateTime.utc_now())
|
||||
|
||||
Registry.dispatch(:status, "unhealthy", fn entries ->
|
||||
for {pid, _} <- entries, do: GenServer.cast(pid, :check)
|
||||
end)
|
||||
|
Loading…
x
Reference in New Issue
Block a user