From 03e46f81f8d9bccd2d68a8c6b2a8d3f4e912633c Mon Sep 17 00:00:00 2001 From: mithereal Date: Sat, 17 Sep 2022 03:38:06 -0700 Subject: [PATCH] add error handling --- config/config.exs | 2 +- config/runtime.exs | 2 +- lib/farside.ex | 11 ++++---- lib/farside/application.ex | 6 ++--- lib/farside/deadcheck.ex | 3 +++ lib/farside/healthycheck.ex | 4 +++ lib/farside/http.ex | 10 +++++--- lib/farside/instance.ex | 3 +-- lib/farside/instance.supervisor.ex | 4 +-- lib/farside/service.ex | 41 +++++++++++++++++++----------- lib/farside/unhealthycheck.ex | 3 +++ mix.exs | 4 +-- 12 files changed, 57 insertions(+), 36 deletions(-) diff --git a/config/config.exs b/config/config.exs index 2cb4b05..970aae1 100644 --- a/config/config.exs +++ b/config/config.exs @@ -15,4 +15,4 @@ config :farside, "weather", "time" ], - recv_timeout: System.get_env("FARSIDE_TIMEOUT") || "8000" \ No newline at end of file + recv_timeout: System.get_env("FARSIDE_TIMEOUT") || "8000" diff --git a/config/runtime.exs b/config/runtime.exs index afb3b40..632a3cc 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -3,4 +3,4 @@ import Config config :farside, port: System.get_env("FARSIDE_PORT", nil), services_json: System.get_env("FARSIDE_SERVICES_JSON", "services.json"), - services_json_data: System.get_env("FARSIDE_SERVICES_JSON_DATA") || "" \ No newline at end of file + services_json_data: System.get_env("FARSIDE_SERVICES_JSON_DATA") || "" diff --git a/lib/farside.ex b/lib/farside.ex index 2b96851..7d7b8ea 100644 --- a/lib/farside.ex +++ b/lib/farside.ex @@ -110,11 +110,12 @@ defmodule Farside do end |> Enum.reject(fn x -> x == nil end) - service = Map.put( - service, - :instances, - instances - ) + service = + Map.put( + service, + :instances, + instances + ) case Enum.count(service.instances) > 0 do true -> Enum.random(service.instances) diff --git a/lib/farside/application.ex b/lib/farside/application.ex index e6218d6..b57d253 100644 --- a/lib/farside/application.ex +++ b/lib/farside/application.ex @@ -23,7 +23,7 @@ defmodule Farside.Application do maybe_loaded_children = case is_nil(System.get_env("FARSIDE_TEST")) do true -> - [{HealthyCheck, []},{UnHealthyCheck, []},{DeadCheck, []}] + [{HealthyCheck, []}, {UnHealthyCheck, []}, {DeadCheck, []}] false -> Logger.info("Skipping sync job setup...") @@ -45,7 +45,7 @@ defmodule Farside.Application do {DynamicSupervisor, strategy: :one_for_one, name: :service_supervisor}, {Registry, keys: :unique, name: :instance}, {Registry, keys: :unique, name: :service}, - {Registry, keys: :duplicate, name: :status, partitions: System.schedulers_online()}, + {Registry, keys: :duplicate, name: :status, partitions: System.schedulers_online()} ] ++ maybe_loaded_children opts = [strategy: :one_for_one, name: Farside.Supervisor] @@ -81,8 +81,6 @@ defmodule Farside.Application do |> HealthyCheck.load() end - LastUpdated.value(DateTime.utc_now()) - response end end diff --git a/lib/farside/deadcheck.ex b/lib/farside/deadcheck.ex index c36a4f9..1494acc 100644 --- a/lib/farside/deadcheck.ex +++ b/lib/farside/deadcheck.ex @@ -3,6 +3,7 @@ defmodule Farside.Server.DeadCheck do Module to check/validate the instance list only for servers with empty instance list every 90 secs, if a sync/check process isnt already running """ use Task + alias Farside.LastUpdated def child_spec(args) do %{ @@ -26,6 +27,8 @@ defmodule Farside.Server.DeadCheck do end def run() do + LastUpdated.value(DateTime.utc_now()) + Registry.dispatch(:status, "dead", fn entries -> for {pid, _} <- entries, do: GenServer.cast(pid, :check) end) diff --git a/lib/farside/healthycheck.ex b/lib/farside/healthycheck.ex index d49672b..50fae45 100644 --- a/lib/farside/healthycheck.ex +++ b/lib/farside/healthycheck.ex @@ -3,6 +3,7 @@ defmodule Farside.Server.HealthyCheck do Module to validate healthy servers """ use Task + alias Farside.LastUpdated def child_spec(args) do %{ @@ -31,10 +32,13 @@ defmodule Farside.Server.HealthyCheck do GenServer.cast(pid, :check) end end) + params end def run() do + LastUpdated.value(DateTime.utc_now()) + Registry.dispatch(:status, "healthy", fn entries -> for {pid, url} <- entries do GenServer.cast(pid, :check) diff --git a/lib/farside/http.ex b/lib/farside/http.ex index d3fc30f..e30d1c1 100644 --- a/lib/farside/http.ex +++ b/lib/farside/http.ex @@ -27,7 +27,6 @@ defmodule Farside.Http do end def request(url, type) do - cond do System.get_env("FARSIDE_TEST") -> :good @@ -41,6 +40,10 @@ defmodule Farside.Http do Logger.info("Type: #{type}, Response: [#{n}], Url: #{url}") :good + nil -> + Logger.error("Type: #{type}, Response: [408], Url: #{url}") + :bad + n -> Logger.error("Type: #{type}, Response: [#{n}], Url: #{url}") :bad @@ -82,7 +85,8 @@ defmodule Farside.Http do end def test_service(service) do -url = service.url <> service.test_url + url = service.url <> service.test_url + test_url = EEx.eval_string( url, @@ -107,7 +111,7 @@ url = service.url <> service.test_url unless is_nil(data) do {_test_url, value, _service} = data value - else + else :bad end end diff --git a/lib/farside/instance.ex b/lib/farside/instance.ex index 42f584e..d0a382f 100644 --- a/lib/farside/instance.ex +++ b/lib/farside/instance.ex @@ -55,7 +55,6 @@ defmodule Farside.Instance do {:stop, :normal, state} end - @doc false def via_tuple(id, registry \\ @registry_name) do {:via, Registry, {registry, id}} @@ -67,12 +66,12 @@ defmodule Farside.Instance do {:noreply, {names, refs}} end - @impl true def handle_info(:load, state) do service = :ets.lookup(String.to_atom(state.type), :default) {_, service} = List.first(service) + service.instances |> Enum.each(fn url -> initial_state = %{url: url, type: service.type, test_url: service.test_url} diff --git a/lib/farside/instance.supervisor.ex b/lib/farside/instance.supervisor.ex index 5147ef6..c053f5a 100644 --- a/lib/farside/instance.supervisor.ex +++ b/lib/farside/instance.supervisor.ex @@ -31,11 +31,11 @@ defmodule Farside.Instance.Supervisor do def start(opts \\ %{}) do child_spec = {SERVER, opts} - {:ok,pid} = DynamicSupervisor.start_child(@name, child_spec) + {:ok, pid} = DynamicSupervisor.start_child(@name, child_spec) send(pid, :load) - {:ok,pid} + {:ok, pid} end def stop(id) do diff --git a/lib/farside/service.ex b/lib/farside/service.ex index 7c8897b..eaee103 100644 --- a/lib/farside/service.ex +++ b/lib/farside/service.ex @@ -91,11 +91,18 @@ defmodule Farside.Service do @impl true def handle_cast(:check, state) do - reply = Http.test_service(state) status = state.status ++ [reply] + max_queue = Application.get_env(:farside, :max_fail_rate, 50) + 5 + + status = + case Enum.count(status) < max_queue do + true -> status + false -> [] + end + state = %{state | status: status} state = %{state | last_update: DateTime.utc_now()} @@ -112,24 +119,28 @@ defmodule Farside.Service do Registry.unregister_match(:status, unhealthy, state.url) Registry.unregister_match(:status, dead, state.url) - if reply != :good do - filtered = Enum.reject(status, fn x -> x == :good end) + state = + if reply != :good do + filtered = Enum.reject(status, fn x -> x == :good end) - fails_before_death = Application.get_env(:farside, :max_fail_rate, 50) + fails_before_death = Application.get_env(:farside, :max_fail_rate, 50) - case Enum.count(filtered) < fails_before_death do - true -> - Registry.register(:status, "unhealthy", state.url) - Registry.register(:status, unhealthy, state.url) + case Enum.count(filtered) < fails_before_death do + true -> + Registry.register(:status, "unhealthy", state.url) + Registry.register(:status, unhealthy, state.url) + state - false -> - Registry.register(:status, "dead", state.url) - Registry.register(:status, dead, state.url) + false -> + Registry.register(:status, "dead", state.url) + Registry.register(:status, dead, state.url) + %{state | status: [:bad]} + end + else + Registry.register(:status, "healthy", state.url) + Registry.register(:status, healthy, state.url) + state end - else - Registry.register(:status, "healthy", state.url) - Registry.register(:status, healthy, state.url) - end {:noreply, state} end diff --git a/lib/farside/unhealthycheck.ex b/lib/farside/unhealthycheck.ex index e3e7b77..05a8f1f 100644 --- a/lib/farside/unhealthycheck.ex +++ b/lib/farside/unhealthycheck.ex @@ -3,6 +3,7 @@ defmodule Farside.Server.UnHealthyCheck do Module to check/validate the instance list only for servers with empty instance list every 90 secs, if a sync/check process isnt already running """ use Task + alias Farside.LastUpdated def child_spec(args) do %{ @@ -26,6 +27,8 @@ defmodule Farside.Server.UnHealthyCheck do end def run() do + LastUpdated.value(DateTime.utc_now()) + Registry.dispatch(:status, "unhealthy", fn entries -> for {pid, _} <- entries, do: GenServer.cast(pid, :check) end) diff --git a/mix.exs b/mix.exs index 012f636..fab2e2d 100644 --- a/mix.exs +++ b/mix.exs @@ -31,8 +31,7 @@ defmodule Farside.MixProject do end defp aliases do - [ - ] + [] end # Run "mix help deps" to learn about dependencies. @@ -46,7 +45,6 @@ defmodule Farside.MixProject do ] end - defp description() do "A redirecting service for FOSS alternative frontends." end