[CI] Auto update instances

Cron init blocks the server from starting until it finishes, which is
not ideal.

.formatter.exs

@@ -1,4 +0,0 @@
-# Used by "mix format"
-[
-  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
-]

.github/workflows/elixir.yml

@@ -1,41 +0,0 @@
-name: Elixir CI
-
-on:
-  push:
-    branches: [ main ]
-  pull_request:
-    branches: [ main ]
-
-jobs:
-  build:
-
-    name: Build and test
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v2
-
-    - name: Set up Elixir
-      uses: erlef/setup-elixir@885971a72ed1f9240973bd92ab57af8c1aa68f24
-      with:
-        elixir-version: '1.12.3'
-        otp-version: '24'
-
-    - name: Restore dependencies cache
-      uses: actions/cache@v2
-      with:
-        path: deps
-        key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }}
-        restore-keys: ${{ runner.os }}-mix-
-
-    - name: Start Redis
-      uses: supercharge/redis-github-action@1.2.0
-
-    - name: Install dependencies
-      run: mix deps.get
-
-    - name: Initialize services
-      run: FARSIDE_TEST=1 mix run -e Farside.Instances.sync
-
-    - name: Run tests
-      run: FARSIDE_TEST=1 mix test --trace

.github/workflows/tests.yml

@@ -0,0 +1,19 @@
+on: [push, pull_request]
+name: Tests
+jobs:
+  test:
+    strategy:
+      matrix:
+        go-version: [1.21.x, 1.22.x, 1.23.x]
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go-version }}
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Test
+      run: go test -v ./...
+

.github/workflows/update-instances.yml

@@ -8,9 +8,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Install dependencies
-      run: sudo apt-get install -y jq
+      run: |
+        sudo apt-get install -y jq dnsutils
+        sudo curl -L https://github.com/mikefarah/yq/releases/latest/download/yq_linux_amd64 -o /usr/local/bin/yq
+        sudo chmod a+rx /usr/local/bin/yq
 
     - uses: webfactory/ssh-agent@v0.5.3
       with:
@@ -20,11 +23,11 @@ jobs:
     - name: Fetch instances
       run: |
         function apply_update() {
-          mv services-tmp.json services.json
+          mv services-tmp.json services-full.json
           rm -f *-tmp.json
 
           # Ensure no trailing slashes for any instance
-          sed -i 's/\/"/"/g' services.json
+          sed -i 's/\/"/"/g' services-full.json
         }
 
         # ==============================================================
@@ -36,42 +39,91 @@ jobs:
         git checkout main
 
         # ==============================================================
-        # Searx update
+        # Bibliogram update
+        # NOTE: Bibliogram has been discontinued and the official instance
+        # at bibliogram.art is no longer around. Unless a new "official"
+        # instance pops up, Bibliogram will likely be removed from farside
+        # ==============================================================
+        #curl -s https://bibliogram.art/api/instances | \
+          #jq '[
+            #.data |
+            #.[] |
+            #select(.onion_site | not) |
+            #.address
+          #] | sort' > bibliogram-tmp.json
+        #jq --slurpfile bibliogram bibliogram-tmp.json \
+          #'( .[] | select(.type == "bibliogram") )
+          #.instances |= $bibliogram[0]' services-full.json > services-tmp.json
+
+        #apply_update
+
+        # ==============================================================
+        # searx update
+        # ==============================================================
+        curl -s https://raw.githubusercontent.com/searx/searx-instances/master/searxinstances/instances.yml | \
+          yq -o=json 'keys | sort' > searx-tmp.json
+        jq --slurpfile searx searx-tmp.json \
+          '( .[] | select(.type == "searx") )
+          .instances |= $searx[0]' services-full.json > services-tmp.json
+
+        apply_update
+
+        # ==============================================================
+        # SearXNG update
         # ==============================================================
-        echo "Fetching searx instances..."
         curl -s https://searx.space/data/instances.json | \
           jq '[
             .instances |
             to_entries[] |
             select(.value.network_type == "normal") |
+            select(.value.generator | . != null) |
+            select(.value.generator | contains("searxng")) |
             select(.value.version | . != null) |
-            select(.value.version | startswith("1.0.0")) |
             select(.value.network.asn_privacy == 0) |
             select(.value.http.error == null) |
+            select(.value.uptime.uptimeDay >= 99) |
             select(.value.tls.grade == "A+" or .value.tls.grade == "A") |
             select(.value.http.grade == "A+" or .value.http.grade == "A") |
             select(.value.html.grade == "V" or .value.html.grade == "F") |
+            select(.key | contains(".i2p") | not) |
             .key
-          ] | sort' > searx-tmp.json
+          ] | sort' > searxng-tmp.json
 
-        cat searx-tmp.json | jq .
+        cat searxng-tmp.json | jq .
 
-        jq --slurpfile searx searx-tmp.json \
-          '( .[] | select(.type == "searx") )
-          .instances |= $searx[0]' services.json > services-tmp.json
+        jq --slurpfile searxng searxng-tmp.json \
+          '( .[] | select(.type == "searxng") )
+          .instances |= $searxng[0]' services-full.json > services-tmp.json
 
         apply_update
 
         # ==============================================================
-        # Simply Translate update
+        # Nitter update
         # ==============================================================
-        curl -s https://simple-web.org/instances/simplytranslate > st-out
+        curl -s https://status.d420.de/api/v1/instances | \
+          jq '[
+            .hosts |
+            to_entries[] |
+            select(.value.healthy == true) |
+            .value.url ] |
+            sort' > nitter-tmp.json
+
+        jq --slurpfile nitter nitter-tmp.json \
+          '( .[] | select(.type == "nitter") )
+          .instances |= $nitter[0]' services-full.json > services-tmp.json
+
+        apply_update
+
+        # ==============================================================
+        # SimplyTranslate update
+        # ==============================================================
+        curl -s https://codeberg.org/SimpleWeb/Website/raw/branch/master/config.json | jq -r '.projects | map(select(.id == "simplytranslate"))[0].instances | .[]' > st-out
         sed -i -e 's/^/https:\/\//' st-out
         jq -ncR '[inputs]' <<< "$(cat st-out | sort)" > st-tmp.json
 
         jq --slurpfile simplytranslate st-tmp.json \
           '( .[] | select(.type == "simplytranslate") )
-          .instances |= $simplytranslate[0]' services.json > services-tmp.json
+          .instances |= $simplytranslate[0]' services-full.json > services-tmp.json
 
         apply_update
 
@@ -83,32 +135,174 @@ jobs:
 
         jq --slurpfile whoogle whoogle-tmp.json \
           '( .[] | select(.type == "whoogle") )
-          .instances |= $whoogle[0]' services.json > services-tmp.json
+          .instances |= $whoogle[0]' services-full.json > services-tmp.json
 
         apply_update
-        
+
         # ==============================================================
         # Invidious update
         # ==============================================================
         curl -s https://api.invidious.io/instances.json | \
           jq '[
-            .[] | 
-            select(.[1].type | contains("https")) ] | 
-            [.[][1].uri] | 
+            .[] |
+            select(.[1].type | contains("https")) ] |
+            [.[][1].uri] |
             sort' > invidious-tmp.json
         jq --slurpfile invidious invidious-tmp.json \
           '( .[] | select(.type == "invidious") )
-          .instances |= $invidious[0]' services.json > services-tmp.json
-          
+          .instances |= $invidious[0]' services-full.json > services-tmp.json
+
         apply_update
 
+        # ==============================================================
+        # Scribe update
+        # ==============================================================
+        curl -s https://git.sr.ht/~edwardloveall/scribe/blob/main/docs/instances.json > scribe-tmp.json
+        jq --slurpfile scribe scribe-tmp.json \
+          '( .[] | select(.type == "scribe") )
+          .instances |= $scribe[0]' services-full.json > services-tmp.json
+
+        apply_update
+
+        # ==============================================================
+        # teddit update
+        # Note: no longer maintained
+        # ==============================================================
+        #curl -s https://codeberg.org/teddit/teddit/raw/branch/main/instances.json | \
+          #jq '[
+            #.[] |
+            #select(.url | contains("https")) |
+            #.url ] |
+            #sort' > teddit-tmp.json
+        #jq --slurpfile teddit teddit-tmp.json \
+          #'( .[] | select(.type == "teddit") )
+          #.instances |= $teddit[0]' services-full.json > services-tmp.json
+
+        #apply_update
+
+        # ==============================================================
+        # Wikiless update
+        # NOTE: Codeberg has removed the Wikiless repo until the maintainer
+        # makes changes. See https://orenom.fi for updates.
+        # ==============================================================
+        #curl -s https://codeberg.org/orenom/Wikiless/raw/branch/main/instances.json | \
+          #jq '[
+            #.[] |
+            #select((.url | contains(".onion") | not) and (.url | contains("https"))).url ] |
+            #sort' > wikiless-tmp.json
+        #jq --slurpfile wikiless wikiless-tmp.json \
+          #'( .[] | select(.type == "wikiless") )
+          #.instances |= $wikiless[0]' services-full.json > services-tmp.json
+
+        #apply_update
+
+        # ==============================================================
+        # libreddit + redlib update
+        # ==============================================================
+        curl -s https://raw.githubusercontent.com/redlib-org/redlib-instances/main/instances.json | \
+          jq '[
+            .instances[] |
+            select(.url) |
+            .url ] |
+            sort' > libreddit-tmp.json
+
+        jq --slurpfile libreddit libreddit-tmp.json \
+          '( .[] | select(.type == "libreddit") )
+          .instances |= $libreddit[0]' services-full.json > services-tmp.json
+
+        mv services-tmp.json services-full.json
+
+        jq --slurpfile redlib libreddit-tmp.json \
+          '( .[] | select(.type == "redlib") )
+          .instances |= $redlib[0]' services-full.json > services-tmp.json
+
+        apply_update
+
+        # ==============================================================
+        # BreezeWiki update
+        # ==============================================================
+        curl -s https://docs.breezewiki.com/files/instances.json | \
+          jq '[.[] | .instance]' > breezewiki-tmp.json
+        jq --slurpfile breezewiki breezewiki-tmp.json \
+          '(.[] | select(.type == "breezewiki") )
+          .instances |= $breezewiki[0]' services-full.json > services-tmp.json
+
+        apply_update
+
+        # ==============================================================
+        # GotHub update
+        # ==============================================================
+
+        curl -s https://codeberg.org/gothub/gothub-instances/raw/branch/master/instances.json | \
+          jq '[.[] | .link]' > gothub-tmp.json
+        jq --slurpfile gothub gothub-tmp.json \
+          '(.[] | select(.type == "gothub") )
+          .instances |= $gothub[0]' services-full.json > services-tmp.json
+
+          apply_update
+
+        # ==============================================================
+        # LibreX/LibreY update
+        # ==============================================================
+        curl -s https://raw.githubusercontent.com/Ahwxorg/LibreY/main/instances.json | \
+          jq '[
+            .instances |
+            .[] |
+            .clearnet] |
+            sort' > librey-tmp.json
+
+        jq --slurpfile librey librey-tmp.json \
+          '( .[] | select(.type == "librey") )
+          .instances |= $librey[0]' services-full.json > services-tmp.json
+
+        apply_update
+
+        # ==============================================================
+        # Rimgo update
+        # ==============================================================
+
+        curl -s https://rimgo.codeberg.page/api.json | \
+          jq '[
+            .clearnet[] |
+            .url] |
+            sort' > rimgo-tmp.json
+
+        jq --slurpfile rimgo rimgo-tmp.json \
+          '(.[] | select(.type == "rimgo") )
+          .instances |= $rimgo[0]' services-full.json > services-tmp.json
+
+          apply_update
+
+        # ==============================================================
+        # Tent update
+        # ==============================================================
+
+        curl -s https://forgejo.sny.sh/sun/Tent/raw/branch/main/instances.json | \
+          jq '[
+            .[] |
+            select(.type == "http") |
+            .url] |
+            sort' > tent-tmp.json
+
+        jq --slurpfile tent tent-tmp.json \
+          '(.[] | select(.type == "tent") )
+          .instances |= $tent[0]' services-full.json > services-tmp.json
+
+          apply_update
+
         # ==============================================================
         # TODO: Update instances for other services
         # ==============================================================
 
+        # ==============================================================
+        # Filter out Cloudflared instances from services-full.json into
+        # services.json
+        # ==============================================================
+        ./tools/un-cloudflare.sh
+
         # ==============================================================
         # Push changes
         # ==============================================================
-        git add services.json
+        git add services.json services-full.json
         git commit -m '[CI] Auto update instances' || exit 0
         git push

.gitignore

@@ -1,15 +1,4 @@
-/_build
-/cover
-/deps
-/doc
-/.fetch
-erl_crash.dump
-*.ez
-*.beam
-/config/*.secret.exs
-.elixir_ls/
-
-# Ignore results from update script
-.update-result*
-
-*.rdb
+badger-db
+farside
+deploy.sh
+out

README.md

@@ -1,8 +1,35 @@
-![Farside](img/farside.svg)
+<div align="center" style="margin-bottom: 10px;">
+<img src="https://benbusby.com/assets/images/farside.svg" alt="Farside">
+</div>
+<br>
+
+<div align="center">
 
 [![Latest Release](https://img.shields.io/github/v/release/benbusby/farside?label=Release)](https://github.com/benbusby/farside/releases)
 [![MIT License](https://img.shields.io/github/license/benbusby/earthbound-themes.svg)](http://opensource.org/licenses/MIT)
-[![Elixir CI](https://github.com/benbusby/privacy-revolver/actions/workflows/elixir.yml/badge.svg)](https://github.com/benbusby/privacy-revolver/actions/workflows/elixir.yml)
+[![Tests](https://github.com/benbusby/farside/actions/workflows/tests.yml/badge.svg)](https://github.com/benbusby/farside/actions/workflows/tests.yml)
+
+<table>
+  <tr>
+    <td><a href="https://sr.ht/~benbusby/farside">SourceHut</a></td>
+    <td><a href="https://github.com/benbusby/farside">GitHub</a></td>
+  </tr>
+</table>
+
+</div>
+
+Contents
+1. [About](#about)
+2. [Demo](#demo)
+3. [How It Works](#how-it-works)
+4. [Cloudflare](#regarding-cloudflare)
+5. [Development](#development)
+    1. [Environment Variables](#environment-variables)
+6. [Search Integration](#search-integration)
+    1. [Kagi](#kagi)
+    2. [Whoogle Search](#whoogle-search)
+
+## About
 
 A redirecting service for FOSS alternative frontends.
 
@@ -13,25 +40,66 @@ available public instances for a particular service, while also helping to
 distribute traffic more evenly across all instances and avoid performance
 bottlenecks and rate-limiting.
 
+Farside also integrates smoothly with basic redirector extensions in most
+browsers. For a simple example setup,
+[refer to the wiki](https://github.com/benbusby/farside/wiki/Browser-Extension).
+
 ## Demo
 
 Farside's links work with the following structure: `farside.link/<service>/<path>`
 
 For example:
 
-| Service | Page | Farside Link |
-| -- | -- | -- |
-| [Libreddit](https://github.com/spikecodes/libreddit) | /r/popular | https://farside.link/libreddit/r/popular
-| [Teddit](https://codeberg.org/teddit/teddit) | /r/popular | https://farside.link/teddit/r/popular
-| [Nitter](https://github.com/zedeus/nitter) | User Profile | https://farside.link/nitter/josevalim
-| [Invidious](https://github.com/iv-org/invidious) | Home Page | https://farside.link/invidious
-| [Piped](https://github.com/TeamPiped/Piped) | Video Page | https://farside.link/piped/watch?v=eBGIQ7ZuuiU
-| [Bibliogram](https://sr.ht/~cadence/bibliogram/) | User Profile | https://farside.link/bibliogram/u/kbdfans
-| [Whoogle](https://github.com/benbusby/whoogle-search) | Search "Elixir" | https://farside.link/whoogle/search?q=elixir&lang_interface=en
-| [Searx](https://github.com/searx/searx) | Search "Redis" | https://farside.link/searx/search?q=redis
-| [SimplyTranslate](https://git.sr.ht/~metalune/simplytranslate_web) | Translate "hola" | https://farside.link/simplytranslate/?engine=google&text=hola
-| [Lingva](https://github.com/TheDavidDelta/lingva-translate) | Translate "bonjour" | https://farside.link/lingva/auto/en/bonjour
-| [Rimgo](https://codeberg.org/video-prize-ranch/rimgo) | View photo album | https://farside.link/rimgo/a/H8M4rcp
+<table>
+    <tr>
+        <td>Service</td>
+        <td>Page</td>
+        <td>Farside Link</td>
+    </tr>
+    <tr>
+        <td><a href="https://sr.ht/~edwardloveall/Scribe/">Scribe</a></td>
+        <td>View Medium post</td>
+        <td><a href="https://farside.link/scribe/@ftrain/big-data-small-effort-b62607a43a8c">https://farside.link/scribe/@ftrain/big-data-small-effort-b62607a43a8c</a></td>
+    </tr>
+    <tr>
+        <td><a href="https://github.com/spikecodes/libreddit">Libreddit</a></td>
+        <td>/r/popular</td>
+        <td><a href="https://farside.link/libreddit/r/popular">https://farside.link/libreddit/r/popular</a></td>
+    </tr>
+    <tr>
+        <td><a href="https://gitdab.com/cadence/breezewiki">BreezeWiki</a></td>
+        <td>Balatro Wiki</td>
+        <td><a href="https://farside.link/breezewiki/balatrogame">https://farside.link/https://balatrogame.fandom.com</a></td>
+    </tr>
+    <tr>
+        <td><a href="https://github.com/searxng/searxng">SearXNG</a></td>
+        <td>Search "EFF"</td>
+        <td><a href="https://farside.link/searxng/search?q=EFF">https://farside.link/searxng/search?q=EFF</a></td>
+    </tr>
+    <tr>
+        <td><a href="https://codeberg.org/ManeraKai/simplytranslate">SimplyTranslate</a></td>
+        <td>Translate "hola"</td>
+        <td><a href="https://farside.link/simplytranslate/?engine=google&text=hola">https://farside.link/simplytranslate/?engine=google&text=hola</a></td>
+    </tr>
+    <tr>
+        <td><a href="https://github.com/TheDavidDelta/lingva-translate">Lingva</a></td>
+        <td>Translate "bonjour"</td>
+        <td><a href="https://farside.link/lingva/auto/en/bonjour">https://farside.link/lingva/auto/en/bonjour</a></td>
+    </tr>
+    <tr>
+        <td><a href="https://codeberg.org/video-prize-ranch/rimgo">Rimgo</a></td>
+        <td>View photo album</td>
+        <td><a href="https://farside.link/rimgo/a/H8M4rcp">https://farside.link/rimgo/a/H8M4rcp</a></td>
+    </tr>
+</table>
+
+<sup>Note: This table doesn't include all available services. For a complete list of supported frontends, see: https://farside.link</sup>
+
+Farside also accepts URLs to "parent" services, and will redirect to an appropriate front end service, for example:
+
+- https://farside.link/https://balatrogame.fandom.com/wiki/Abandoned_Deck will redirect to a [BreezeWiki](https://gitdab.com/cadence/breezewiki) instance
+- https://farside.link/reddit.com/r/popular will redirect to a [Libreddit](https://github.com/spikecodes/libreddit) or [Teddit](https://codeberg.org/teddit/teddit) instance
+- etc.
 
 ## How It Works
 
@@ -46,13 +114,14 @@ Farside's routing is very minimal, with only the following routes:
 
 - `/`
   - The app home page, displaying all live instances for every service
-- `/ping`
-  - A passthrough "ping" to redis to ensure both app and redis are working
 - `/:service/*glob`
   - The main endpoint for redirecting a user to a working instance of a
     particular service with the specified path
   - Ex: `/libreddit/r/popular` would navigate to `<libreddit instance
     URL>/r/popular`
+    - If the service provided is actually a URL to a "parent" service
+      (i.e. "youtube.com" instead of "piped" or "invidious"), Farside
+      will determine the correct frontend to use for the specified URL.
   - Note that a path is not required. `/libreddit` for example will still
     redirect the user to a working libreddit instance
 - `/_/:service/*glob`
@@ -64,8 +133,8 @@ Farside's routing is very minimal, with only the following routes:
   - *Note: Uses Javascript to preserve the page in history*
 
 When a service is requested with the `/:service/...` endpoint, Farside requests
-the list of working instances from Redis and returns a random one from the list
-and adds that instance as a new entry in Redis to remove from subsequent
+the list of working instances from the db and returns a random one from the list
+and adds that instance as a new entry in the db to remove from subsequent
 requests for that service. For example:
 
 A user navigates to `/nitter` and is redirected to `nitter.net`. The next user
@@ -80,18 +149,78 @@ ensure better distribution of traffic to available instances for each service.
 Farside also has built-in IP ratelimiting for all requests, enforcing only one
 request per second per IP.
 
+## Regarding Cloudflare
+Instances for each supported service that are deployed behind Cloudflare are
+not included when using [farside.link](https://farside.link). If you would like
+to also access instances that use Cloudflare (in addition to instances that do
+not), you can either use [cf.farside.link](https://cf.farside.link) instead, or
+deploy your own instance of Farside and set
+`FARSIDE_SERVICES_JSON=services-full.json` when running.
+
+If you do decide to use [cf.farside.link](https://cf.farside.link) or use the
+full instance list provided by `services-full.json`, please be aware that
+Cloudflare takes steps to block site visitors using Tor (and some VPNs), and
+that their mission to centralize the entire web behind their service ultimately
+goes against what Farside is trying to solve. Use at your own discretion.
+
 ## Development
 
-- Install [redis](https://redis.io)
-- Install [elixir](https://elixir-lang.org/install.html)
-- Start redis: `redis-server /usr/local/etc/redis.conf`
-- Install dependencies: `mix deps.get`
-- Initialize redis contents: `mix run -e Farside.Instances.sync`
-- Run Farside: `mix run --no-halt`
-  - Uses localhost:4001
+- Install [Go](https://go.dev/doc/install)
+- Compile with `go build`
 
 ### Environment Variables
 
-| Name | Purpose |
-| -- | -- |
-| FARSIDE_TEST | If enabled, bypasses the instance availability check and adds all instances to the pool. |
+<table>
+    <tr>
+        <td>Name</td>
+        <td>Purpose</td>
+    </tr>
+    <tr>
+        <td>FARSIDE_TEST</td>
+        <td>If enabled, bypasses the instance availability check and adds all instances to the pool</td>
+    </tr>
+    <tr>
+        <td>FARSIDE_PORT</td>
+        <td>The port to run Farside on (default: `4001`)</td>
+    </tr>
+    <tr>
+        <td>FARSIDE_DB_DIR</td>
+        <td>The path to the directory to use for storing instance data (default: `./`)</td>
+    </tr>
+    <tr>
+        <td>FARSIDE_CF_ENABLED</td>
+        <td>Set to 1 to enable redirecting to instances behind cloudflare</td>
+    </tr>
+    <tr>
+        <td>FARSIDE_CRON</td>
+        <td>Set to 0 to deactivate the periodic instance availability check</td>
+    </tr>
+</table>
+
+## Search Integration
+
+### Kagi
+
+https://kagi.com
+
+On the settings page, go to `Search > Advanced > Open Redirects` and setup your redirects.
+
+With the exception of BreezeWiki, most redirect rules can just extract the path of the
+link you're visiting and append them to whichever Farside redirect you want to use.
+
+For example:
+
+##### Medium -> Scribe
+
+`^https://medium.com/(.*)|https://farside.link/scribe/$1`
+
+##### Fandom -> BreezeWiki
+
+`^https://([^/]+).fandom.com/(.*)|https://farside.link/breezewiki/$1/$2`
+
+### Whoogle Search
+
+https://github.com/benbusby/whoogle-search
+
+Whoogle automatically routes eligible links through Farside when the `Replace
+Social Media Links` option in the home page settings menu is enabled.

config/config.exs

@@ -1,22 +0,0 @@
-import Config
-
-config :farside,
-  port: 4001,
-  redis_conn: "redis://localhost:6379",
-  update_file: ".update-results",
-  service_prefix: "service-",
-  fallback_suffix: "-fallback",
-  previous_suffix: "-previous",
-  services_json: "services.json",
-  index: "index.eex",
-  route: "route.eex",
-  headers: [
-    {"User-Agent", "Mozilla/5.0 (Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0"},
-    {"Accept", "text/html"},
-    {"Accept-Language", "en-US,en;q=0.5"},
-    {"Accept-Encoding", "gzip, deflate, br"}
-  ],
-  queries: [
-    "weather",
-    "time"
-  ]

cross_compile.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+
+mkdir -p $dir/out/
+rm -f $dir/out/*
+
+platforms=(
+    "linux/arm"
+    "linux/amd64"
+    "linux/arm64"
+    "linux/386"
+    "darwin/amd64"
+    "darwin/arm64"
+    "windows/386"
+    "windows/amd64"
+    "windows/arm64")
+
+for platform in "${platforms[@]}"
+do
+    platform_split=(${platform//\// })
+    GOOS=${platform_split[0]}
+    GOARCH=${platform_split[1]}
+    output_name="farside"
+
+    tar_name="farside_${GOOS}_${GOARCH}.tar.gz"
+    if [ $GOOS = "darwin" ]; then
+        tar_name="farside_macOS_${GOARCH}.tar.gz"
+    fi
+
+    if [ $GOOS = "windows" ]; then
+        output_name+=".exe"
+    fi
+
+    compile_cmd="GOOS=$GOOS GOARCH=$GOARCH go build -ldflags='-s -w' -o $output_name ."
+    echo "└ $compile_cmd"
+    eval $compile_cmd
+    if [ $? -ne 0 ]; then
+        echo "An error has occurred! Aborting the script execution..."
+        exit 1
+    fi
+
+    tar -czvf out/$tar_name $output_name
+    rm -f $output_name
+done

db/cron.go

@@ -0,0 +1,153 @@
+package db
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/benbusby/farside/services"
+	"github.com/robfig/cron/v3"
+)
+
+const defaultPrimary = "https://farside.link/state"
+const defaultCFPrimary = "https://cf.farside.link/state"
+
+var LastUpdate time.Time
+var skipInstanceChecks = []string{
+	"searx",
+	"searxng",
+}
+
+func InitCronTasks() {
+	log.Println("Initializing cron tasks...")
+	updateServiceList()
+
+	cronDisabled := os.Getenv("FARSIDE_CRON")
+	if len(cronDisabled) == 0 || cronDisabled == "1" {
+		c := cron.New()
+		c.AddFunc("@every 10m", queryServiceInstances)
+		c.AddFunc("@daily", updateServiceList)
+		c.Start()
+	}
+
+	queryServiceInstances()
+}
+
+func updateServiceList() {
+	fileName := services.GetServicesFileName()
+	_, _ = services.FetchServicesFile(fileName)
+	services.InitializeServices()
+}
+
+func queryServiceInstances() {
+	log.Println("Starting instance queries...")
+
+	isPrimary := os.Getenv("FARSIDE_PRIMARY")
+	if len(isPrimary) == 0 || isPrimary != "1" {
+		remoteServices, err := fetchInstancesFromPrimary()
+		if err != nil {
+			log.Println("Unable to fetch instances from primary", err)
+		}
+
+		for _, service := range remoteServices {
+			SetInstances(service.Type, service.Instances)
+		}
+
+		LastUpdate = time.Now().UTC()
+		return
+	}
+
+	for _, service := range services.ServiceList {
+		canSkip := slices.Contains[[]string, string](skipInstanceChecks, service.Type)
+
+		fmt.Printf("===== %s =====\n", service.Type)
+		var instances []string
+		for _, instance := range service.Instances {
+			testURL := strings.ReplaceAll(
+				service.TestURL,
+				"<%=query%>",
+				"current+weather")
+			available := queryServiceInstance(
+				instance,
+				testURL,
+				canSkip)
+
+			if available {
+				instances = append(instances, instance)
+			}
+		}
+
+		SetInstances(service.Type, instances)
+	}
+
+	LastUpdate = time.Now().UTC()
+}
+
+func fetchInstancesFromPrimary() ([]services.Service, error) {
+	primaryURL := defaultPrimary
+	useCF := os.Getenv("FARSIDE_CF_ENABLED")
+	if len(useCF) > 0 && useCF == "1" {
+		primaryURL = defaultCFPrimary
+	}
+
+	resp, err := http.Get(primaryURL)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	bodyBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var serviceList []services.Service
+	err = json.Unmarshal(bodyBytes, &serviceList)
+	return serviceList, err
+}
+
+func queryServiceInstance(instance, testURL string, canSkipCheck bool) bool {
+	testMode := os.Getenv("FARSIDE_TEST")
+	if len(testMode) > 0 && testMode == "1" {
+		return true
+	}
+
+	if canSkipCheck {
+		fmt.Printf("    [INFO] Adding %s\n", instance)
+		return true
+	}
+
+	ua := "Mozilla/5.0 (compatible; Farside/1.0.0; +https://farside.link)"
+	url := instance + testURL
+
+	req, err := http.NewRequest(http.MethodGet, url, nil)
+	if err != nil {
+		fmt.Println("    [ERRO] Failed to create new http request!", err)
+		return false
+	}
+
+	req.Header.Set("User-Agent", ua)
+	client := &http.Client{
+		Timeout: 10 * time.Second,
+	}
+	resp, err := client.Do(req)
+
+	if err != nil {
+		fmt.Println("    [ERRO] Error fetching instance:", err)
+		return false
+	} else if resp.StatusCode != http.StatusOK {
+		fmt.Printf("    [WARN] Received non-200 status for %s\n", url)
+		return false
+	} else {
+		fmt.Printf("    [INFO] Received 200 status for %s\n", url)
+	}
+
+	return true
+}

db/db.go

@@ -0,0 +1,158 @@
+package db
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"math/rand"
+	"os"
+	"slices"
+	"strings"
+	"time"
+
+	"github.com/benbusby/farside/services"
+	"github.com/dgraph-io/badger/v4"
+)
+
+var (
+	badgerDB     *badger.DB
+	selectionMap map[string]string
+
+	cachedServiceList []services.Service
+	cacheUpdated      time.Time
+)
+
+func InitializeDB() error {
+	var err error
+
+	dbDir := os.Getenv("FARSIDE_DB_DIR")
+	if len(dbDir) == 0 {
+		dbDir = "./badger-db"
+	}
+
+	badgerDB, err = badger.Open(badger.DefaultOptions(dbDir))
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func SetInstances(service string, instances []string) error {
+	instancesBytes, err := json.Marshal(instances)
+	if err != nil {
+		return err
+	}
+
+	err = badgerDB.Update(func(txn *badger.Txn) error {
+		err := txn.Set([]byte(service), instancesBytes)
+		return err
+	})
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func GetInstance(service, path string) (string, error) {
+	instances, err := GetAllInstances(service)
+	if err != nil || len(instances) == 0 {
+		if err != nil {
+			log.Println("DB err:", err)
+		}
+
+		link, ok := services.FallbackMap[service]
+		if !ok {
+			return "", errors.New("invalid service")
+		}
+
+		return link, nil
+	}
+
+	previous, ok := selectionMap[service]
+	if ok && len(instances) > 2 {
+		instances = slices.DeleteFunc(instances, func(i string) bool {
+			return i == previous
+		})
+	}
+
+	index := rand.Intn(len(instances))
+	value := instances[index]
+	selectionMap[service] = value
+
+	if len(path) > 0 {
+		value = strings.TrimSuffix(value, "/")
+		value = fmt.Sprintf("%s/%s", value, path)
+	}
+
+	return value, nil
+}
+
+func GetAllInstances(service string) ([]string, error) {
+	var instances []string
+	err := badgerDB.View(func(txn *badger.Txn) error {
+		item, err := txn.Get([]byte(service))
+		if err != nil {
+			return err
+		}
+
+		err = item.Value(func(val []byte) error {
+			err := json.Unmarshal(val, &instances)
+			return err
+		})
+
+		return err
+	})
+
+	return instances, err
+}
+
+func GetServiceList() []services.Service {
+	if cacheUpdated.Add(5 * time.Minute).After(time.Now().UTC()) {
+		return cachedServiceList
+	}
+
+	canCache := true
+
+	var serviceList []services.Service
+	for _, service := range services.ServiceList {
+		instances, err := GetAllInstances(service.Type)
+		if err != nil {
+			canCache = false
+			instances = []string{service.Fallback}
+		}
+
+		storedService := services.Service{
+			Type:      service.Type,
+			Instances: instances,
+		}
+
+		serviceList = append(serviceList, storedService)
+	}
+
+	if canCache {
+		cachedServiceList = serviceList
+		cacheUpdated = time.Now().UTC()
+	}
+
+	return serviceList
+}
+
+func CloseDB() error {
+	log.Println("Closing database...")
+	err := badgerDB.Close()
+	if err != nil {
+		log.Println("Error closing database", err)
+		return err
+	}
+
+	log.Println("Database closed!")
+	return nil
+}
+
+func init() {
+	selectionMap = make(map[string]string)
+}

db/db_test.go

@@ -0,0 +1,60 @@
+package db
+
+import (
+	"log"
+	"os"
+	"slices"
+	"testing"
+)
+
+func TestMain(m *testing.M) {
+	err := InitializeDB()
+	if err != nil {
+		log.Fatalln("Failed to initialize database")
+	}
+
+	exitCode := m.Run()
+
+	_ = CloseDB()
+	os.Exit(exitCode)
+}
+
+func TestDatabase(t *testing.T) {
+	var (
+		service = "test"
+		siteA   = "a.com"
+		siteB   = "b.com"
+		siteC   = "c.com"
+	)
+	instances := []string{siteA, siteB, siteC}
+	err := SetInstances(service, instances)
+	if err != nil {
+		t.Fatalf("Failed to set instances: %v\n", err)
+	}
+
+	dbInstances, err := GetAllInstances(service)
+	if err != nil {
+		t.Fatalf("Failed to retrieve instances: %v\n", err)
+	}
+
+	for _, instance := range instances {
+		idx := slices.Index(dbInstances, instance)
+		if idx < 0 {
+			t.Fatalf("Failed to find instance in list")
+		}
+	}
+
+	firstInstance, err := GetInstance(service, "")
+	if err != nil {
+		t.Fatalf("Failed to fetch single instance: %v\n", err)
+	}
+
+	secondInstance, err := GetInstance(service, "")
+	if err != nil {
+		t.Fatalf("Failed to fetch single instance (second): %v\n", err)
+	} else if firstInstance == secondInstance {
+		t.Fatalf("Same instance was selected twice")
+	}
+
+	_ = CloseDB()
+}

go.mod

@@ -0,0 +1,29 @@
+module github.com/benbusby/farside
+
+go 1.23.4
+
+require (
+	github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/charmbracelet/lipgloss v0.10.0 // indirect
+	github.com/dgraph-io/badger/v4 v4.5.0 // indirect
+	github.com/dgraph-io/ristretto/v2 v2.0.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-logfmt/logfmt v0.6.0 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/google/flatbuffers v24.3.25+incompatible // indirect
+	github.com/klauspost/compress v1.17.11 // indirect
+	github.com/lucasb-eyer/go-colorful v1.2.0 // indirect
+	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
+	github.com/muesli/reflow v0.3.0 // indirect
+	github.com/muesli/termenv v0.15.2 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/robfig/cron/v3 v3.0.0 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
+	golang.org/x/net v0.31.0 // indirect
+	golang.org/x/sys v0.27.0 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
+)

go.sum

@@ -0,0 +1,141 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
+github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/charmbracelet/lipgloss v0.10.0 h1:KWeXFSexGcfahHX+54URiZGkBFazf70JNMtwg/AFW3s=
+github.com/charmbracelet/lipgloss v0.10.0/go.mod h1:Wig9DSfvANsxqkRsqj6x87irdy123SR4dOXlKa91ciE=
+github.com/charmbracelet/log v0.4.0 h1:G9bQAcx8rWA2T3pWvx7YtPTPwgqpk7D68BX21IRW8ZM=
+github.com/charmbracelet/log v0.4.0/go.mod h1:63bXt/djrizTec0l11H20t8FDSvA4CRZJ1KH22MdptM=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgraph-io/badger/v4 v4.5.0 h1:TeJE3I1pIWLBjYhIYCA1+uxrjWEoJXImFBMEBVSm16g=
+github.com/dgraph-io/badger/v4 v4.5.0/go.mod h1:ysgYmIeG8dS/E8kwxT7xHyc7MkmwNYLRoYnFbr7387A=
+github.com/dgraph-io/ristretto/v2 v2.0.0 h1:l0yiSOtlJvc0otkqyMaDNysg8E9/F/TYZwMbxscNOAQ=
+github.com/dgraph-io/ristretto/v2 v2.0.0/go.mod h1:FVFokF2dRqXyPyeMnK1YDy8Fc6aTe0IKgbcd03CYeEk=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4=
+github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI=
+github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
+github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
+github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
+github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
+github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98=
+github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s=
+github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8=
+github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo=
+github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E=
+github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
+go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
+golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
+golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
+golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI=
+google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

index.eex

@@ -1,53 +0,0 @@
-<head>
-  <title>Farside</title>
-  <style>
-    html {
-      font-family: monospace;
-      font-size: 16px;
-      color: #66397C;
-    }
-    #parent-div {
-      text-align: center;
-    }
-    #child-div {
-      text-align: left;
-      width: 50%;
-      display: inline-block;
-    }
-    hr {
-      border: 1px dashed;
-    }
-    a:link, a:visited {
-      color: #66397C; 
-    }
-    @media only screen and (max-width: 1000px) {
-      #child-div {
-        width: 90%;
-      }
-    }
-    ul {
-      margin: 10px;
-    }
-  </style>
-</head>
-<body>
-  <div id="parent-div">
-    <div id="child-div">
-      <h1>Farside | <a href="https://github.com/benbusby/farside">View on GitHub</a></h1>
-      <hr>
-      <h3>Last synced <%= last_updated %> UTC</h2>
-      <div>
-        <ul>
-          <%= for {service, instance_list} <- services do %>
-            <li><a href="/<%= service %>"><%= service %></a></li>
-            <ul>
-              <%= for url <- instance_list do %>
-                <li><a href="<%= url %>"><%= url %></a></li>
-              <% end%>
-            </ul>
-          <% end %>
-        </ul>
-      </div>
-    </div>
-  </div>
-</body>

lib/farside.ex

@@ -1,90 +0,0 @@
-defmodule Farside do
-  @service_prefix Application.fetch_env!(:farside, :service_prefix)
-  @fallback_suffix Application.fetch_env!(:farside, :fallback_suffix)
-  @previous_suffix Application.fetch_env!(:farside, :previous_suffix)
-
-  def get_services_map do
-    {:ok, service_list} = Redix.command(:redix, ["KEYS", "#{@service_prefix}*"])
-
-    # Match service name to list of available instances
-    Enum.reduce(service_list, %{}, fn service, acc ->
-      {:ok, instance_list} =
-        Redix.command(
-          :redix,
-          ["LRANGE", service, "0", "-1"]
-        )
-
-      Map.put(
-        acc,
-        String.replace_prefix(
-          service,
-          @service_prefix,
-          ""
-        ),
-        instance_list
-      )
-    end)
-  end
-
-  def pick_instance(service) do
-    {:ok, instances} =
-      Redix.command(
-        :redix,
-        [
-          "LRANGE",
-          "#{@service_prefix}#{service}",
-          "0",
-          "-1"
-        ]
-      )
-
-    # Either pick a random available instance, 
-    # or fall back to the default one
-    instance =
-      if Enum.count(instances) > 0 do
-        if Enum.count(instances) == 1 do
-          # If there's only one instance, just return that one...
-          List.first(instances)
-        else
-          # ...otherwise pick a random one from the list, ensuring
-          # that the same instance is never picked twice in a row.
-          {:ok, previous} =
-            Redix.command(
-              :redix,
-              ["GET", "#{service}#{@previous_suffix}"]
-            )
-
-          instance =
-            Enum.filter(instances, &(&1 != previous))
-            |> Enum.random()
-
-          Redix.command(
-            :redix,
-            ["SET", "#{service}#{@previous_suffix}", instance]
-          )
-
-          instance
-        end
-      else
-        {:ok, result} =
-          Redix.command(
-            :redix,
-            ["GET", "#{service}#{@fallback_suffix}"]
-          )
-
-        result
-      end
-
-    instance
-  end
-
-  def get_last_updated do
-    {:ok, last_updated} =
-      Redix.command(
-        :redix,
-        ["GET", "last_updated"]
-      )
-
-    last_updated
-  end
-end

lib/farside/application.ex

@@ -1,27 +0,0 @@
-defmodule Farside.Application do
-  @farside_port Application.fetch_env!(:farside, :port)
-  @redis_conn Application.fetch_env!(:farside, :redis_conn)
-  @moduledoc false
-
-  use Application
-
-  @impl true
-  def start(_type, _args) do
-    children = [
-      Plug.Cowboy.child_spec(
-        scheme: :http,
-        plug: Farside.Router,
-        options: [
-          port: @farside_port
-        ]
-      ),
-      {PlugAttack.Storage.Ets, name: Farside.Throttle.Storage, clean_period: 60_000},
-      {Redix, {@redis_conn, [name: :redix]}},
-      Farside.Scheduler,
-      Farside.Server
-    ]
-
-    opts = [strategy: :one_for_one, name: Farside.Supervisor]
-    Supervisor.start_link(children, opts)
-  end
-end

lib/farside/instances.ex

@@ -1,109 +0,0 @@
-defmodule Farside.Instances do
-  @fallback_suffix Application.fetch_env!(:farside, :fallback_suffix)
-  @update_file Application.fetch_env!(:farside, :update_file)
-  @services_json Application.fetch_env!(:farside, :services_json)
-  @service_prefix Application.fetch_env!(:farside, :service_prefix)
-  @headers Application.fetch_env!(:farside, :headers)
-  @queries Application.fetch_env!(:farside, :queries)
-  @debug_header "======== "
-  @debug_spacer "         "
-
-  def sync() do
-    File.rename(@update_file, "#{@update_file}-prev")
-    update()
-
-    # Add UTC time of last update
-    Redix.command(:redix, [
-      "SET",
-      "last_updated",
-      Calendar.strftime(DateTime.utc_now(), "%c")
-    ])
-  end
-
-  def request(url) do
-    cond do
-      System.get_env("FARSIDE_TEST") ->
-        :good
-
-      true ->
-        HTTPoison.get(url, @headers)
-        |> then(&elem(&1, 1))
-        |> Map.get(:status_code)
-        |> case do
-          n when n < 400 ->
-            IO.puts("#{@debug_spacer}✓ [#{n}]")
-            :good
-
-          n ->
-            IO.puts("#{@debug_spacer}x [#{(n && n) || "error"}]")
-            :bad
-        end
-    end
-  end
-
-  def update() do
-    {:ok, file} = File.read(@services_json)
-    {:ok, json} = Poison.decode(file, as: [%Service{}])
-
-    # Loop through all instances and check each for availability
-    for service <- json do
-      IO.puts("#{@debug_header}#{service.type}")
-
-      result =
-        Enum.filter(service.instances, fn instance_url ->
-          request_url =
-            instance_url <>
-              EEx.eval_string(
-                service.test_url,
-                query: Enum.random(@queries)
-              )
-
-          IO.puts("#{@debug_spacer}#{request_url}")
-
-          request(request_url) == :good
-        end)
-
-      add_to_redis(service, result)
-      log_results(service.type, result)
-    end
-  end
-
-  def add_to_redis(service, instances) do
-    # Remove previous list of instances
-    Redix.command(:redix, [
-      "DEL",
-      "#{@service_prefix}#{service.type}"
-    ])
-
-    # Update with new list of available instances
-    Redix.command(
-      :redix,
-      [
-        "LPUSH",
-        "#{@service_prefix}#{service.type}"
-      ] ++ instances
-    )
-
-    # Set fallback to one of the available instances,
-    # or the default instance if all are "down"
-    if Enum.count(instances) > 0 do
-      Redix.command(:redix, [
-        "SET",
-        "#{service.type}#{@fallback_suffix}",
-        Enum.random(instances)
-      ])
-    else
-      Redix.command(:redix, [
-        "SET",
-        "#{service.type}#{@fallback_suffix}",
-        service.fallback
-      ])
-    end
-  end
-
-  def log_results(service_name, results) do
-    {:ok, file} = File.open(@update_file, [:append, {:delayed_write, 100, 20}])
-    IO.write(file, "#{service_name}: #{inspect(results)}\n")
-    File.close(file)
-  end
-end

lib/farside/router.ex

@@ -1,62 +0,0 @@
-defmodule Farside.Router do
-  @index Application.fetch_env!(:farside, :index)
-  @route Application.fetch_env!(:farside, :route)
-
-  use Plug.Router
-
-  plug(Farside.Throttle)
-  plug(:match)
-  plug(:dispatch)
-
-  get "/" do
-    resp =
-      EEx.eval_file(
-        @index,
-        last_updated: Farside.get_last_updated(),
-        services: Farside.get_services_map()
-      )
-
-    send_resp(conn, 200, resp)
-  end
-
-  get "/ping" do
-    # Useful for app healthcheck
-    {:ok, resp} = Redix.command(:redix, ["PING"])
-    send_resp(conn, 200, resp)
-  end
-
-  get "/_/:service/*glob" do
-    r_path = String.slice(conn.request_path, 2..-1)
-
-    resp =
-      EEx.eval_file(
-        @route,
-        service: service,
-        instance_url: r_path
-      )
-
-    send_resp(conn, 200, resp)
-  end
-
-  get "/:service/*glob" do
-    path = Enum.join(glob, "/")
-    instance = Farside.pick_instance(service)
-
-    params =
-      cond do
-        String.length(conn.query_string) > 0 ->
-          "?#{conn.query_string}"
-
-        true ->
-          ""
-      end
-
-    # Redirect to the available instance
-    conn
-    |> Plug.Conn.resp(:found, "")
-    |> Plug.Conn.put_resp_header(
-      "location",
-      "#{instance}/#{path}#{params}"
-    )
-  end
-end

lib/farside/scheduler.ex

@@ -1,3 +0,0 @@
-defmodule Farside.Scheduler do
-  use Quantum, otp_app: :farside
-end

lib/farside/server.ex

@@ -1,22 +0,0 @@
-defmodule Farside.Server do
-  use GenServer
-  import Crontab.CronExpression
-
-  def init(init_arg) do
-    {:ok, init_arg}
-  end
-
-  def start_link(arg) do
-    if System.get_env("FARSIDE_TEST") do
-      IO.puts("Skipping sync job setup...")
-    else
-      Farside.Scheduler.new_job()
-      |> Quantum.Job.set_name(:sync)
-      |> Quantum.Job.set_schedule(~e[*/5 * * * *])
-      |> Quantum.Job.set_task(fn -> Farside.Instances.sync() end)
-      |> Farside.Scheduler.add_job()
-    end
-
-    GenServer.start_link(__MODULE__, arg)
-  end
-end

lib/farside/throttle.ex

@@ -1,21 +0,0 @@
-defmodule Farside.Throttle do
-  import Plug.Conn
-  use PlugAttack
-
-  rule "throttle per ip", conn do
-    # throttle to 1 request per second
-    throttle(conn.remote_ip,
-      period: 1_000,
-      limit: 1,
-      storage: {PlugAttack.Storage.Ets, Farside.Throttle.Storage}
-    )
-  end
-
-  def allow_action(conn, _data, _opts), do: conn
-
-  def block_action(conn, _data, _opts) do
-    conn
-    |> send_resp(:forbidden, "Exceeded rate limit\n")
-    |> halt
-  end
-end

lib/service.ex

@@ -1,6 +0,0 @@
-defmodule Service do
-  defstruct type: nil,
-            test_url: nil,
-            fallback: nil,
-            instances: []
-end

main.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"github.com/benbusby/farside/db"
+	"github.com/benbusby/farside/server"
+	"github.com/benbusby/farside/services"
+)
+
+func main() {
+	err := db.InitializeDB()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	go func() {
+		err = services.InitializeServices()
+		if err != nil {
+			log.Println("Error intializing services", err)
+		}
+	}()
+
+	go db.InitCronTasks()
+
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, os.Interrupt, syscall.SIGTERM)
+
+	go func() {
+		<-signalChan
+		_ = db.CloseDB()
+		os.Exit(0)
+	}()
+
+	server.RunServer()
+}

mix.exs

@@ -1,34 +0,0 @@
-defmodule Farside.MixProject do
-  use Mix.Project
-
-  def project do
-    [
-      app: :farside,
-      version: "0.1.0",
-      elixir: "~> 1.8",
-      start_permanent: Mix.env() == :prod,
-      deps: deps()
-    ]
-  end
-
-  # Run "mix help compile.app" to learn about applications.
-  def application do
-    [
-      extra_applications: [:logger],
-      mod: {Farside.Application, []}
-    ]
-  end
-
-  # Run "mix help deps" to learn about dependencies.
-  defp deps do
-    [
-      {:httpoison, "~> 1.8"},
-      {:jason, "~> 1.1"},
-      {:plug_attack, "~> 0.4.2"},
-      {:plug_cowboy, "~> 2.0"},
-      {:poison, "~> 5.0"},
-      {:quantum, "~> 3.0"},
-      {:redix, "~> 1.1"}
-    ]
-  end
-end

mix.lock

@@ -1,29 +0,0 @@
-%{
-  "certifi": {:hex, :certifi, "2.8.0", "d4fb0a6bb20b7c9c3643e22507e42f356ac090a1dcea9ab99e27e0376d695eba", [:rebar3], [], "hexpm", "6ac7efc1c6f8600b08d625292d4bbf584e14847ce1b6b5c44d983d273e1097ea"},
-  "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"},
-  "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
-  "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"},
-  "crontab": {:hex, :crontab, "1.1.10", "dc9bb1f4299138d47bce38341f5dcbee0aa6c205e864fba7bc847f3b5cb48241", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "1347d889d1a0eda997990876b4894359e34bfbbd688acbb0ba28a2795ca40685"},
-  "gen_stage": {:hex, :gen_stage, "1.1.2", "b1656cd4ba431ed02c5656fe10cb5423820847113a07218da68eae5d6a260c23", [:mix], [], "hexpm", "9e39af23140f704e2b07a3e29d8f05fd21c2aaf4088ff43cb82be4b9e3148d02"},
-  "hackney": {:hex, :hackney, "1.18.0", "c4443d960bb9fba6d01161d01cd81173089686717d9490e5d3606644c48d121f", [:rebar3], [{:certifi, "~>2.8.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "9afcda620704d720db8c6a3123e9848d09c87586dc1c10479c42627b905b5c5e"},
-  "httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"},
-  "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
-  "jason": {:hex, :jason, "1.2.2", "ba43e3f2709fd1aa1dce90aaabfd039d000469c05c56f0b8e31978e03fa39052", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "18a228f5f0058ee183f29f9eae0805c6e59d61c3b006760668d8d18ff0d12179"},
-  "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"},
-  "mime": {:hex, :mime, "2.0.2", "0b9e1a4c840eafb68d820b0e2158ef5c49385d17fb36855ac6e7e087d4b1dcc5", [:mix], [], "hexpm", "e6a3f76b4c277739e36c2e21a2c640778ba4c3846189d5ab19f97f126df5f9b7"},
-  "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"},
-  "mustache": {:hex, :mustache, "0.3.1", "4c6ee79b13aae954035fe31b83c94480ddc7b536d09c44d4c65e61a9ead38d6b", [:mix], [], "hexpm", "8dc92b9b92a0d7449628f4fc981f8018a16a5b8c9907249e59db461482dac143"},
-  "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"},
-  "phoenix_view": {:hex, :phoenix_view, "1.0.0", "fea71ecaaed71178b26dd65c401607de5ec22e2e9ef141389c721b3f3d4d8011", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "82be3e2516f5633220246e2e58181282c71640dab7afc04f70ad94253025db0c"},
-  "plug": {:hex, :plug, "1.12.1", "645678c800601d8d9f27ad1aebba1fdb9ce5b2623ddb961a074da0b96c35187d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d57e799a777bc20494b784966dc5fbda91eb4a09f571f76545b72a634ce0d30b"},
-  "plug_attack": {:hex, :plug_attack, "0.4.3", "88e6c464d68b1491aa083a0347d59d58ba71a7e591a7f8e1b675e8c7792a0ba8", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "9ed6fb8a6f613a36040f2875130a21187126c5625092f24bc851f7f12a8cbdc1"},
-  "plug_cowboy": {:hex, :plug_cowboy, "2.5.2", "62894ccd601cf9597e2c23911ff12798a8a18d237e9739f58a6b04e4988899fe", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ea6e87f774c8608d60c8d34022a7d073bd7680a0a013f049fc62bf35efea1044"},
-  "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"},
-  "poison": {:hex, :poison, "5.0.0", "d2b54589ab4157bbb82ec2050757779bfed724463a544b6e20d79855a9e43b24", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "11dc6117c501b80c62a7594f941d043982a1bd05a1184280c0d9166eb4d8d3fc"},
-  "quantum": {:hex, :quantum, "3.4.0", "5a53c3c52b0d55f2323940232ba6ab4c98e7e14c73dfacbba3a1ed799b037ce5", [:mix], [{:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.14 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d0eb64957d3dc49c8ed730cc2203108334226496535965b8dfa3f3dbcf430f87"},
-  "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
-  "redix": {:hex, :redix, "1.1.4", "d66fc83d2d4f136c838568d1ec8b0c1a72acfcecfac88a40f86f60aaee883c93", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "515eff055b7de8967e835f4de22a6cfe8311bc1b8fe72f48200238fb43f6a803"},
-  "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},
-  "telemetry": {:hex, :telemetry, "1.0.0", "0f453a102cdf13d506b7c0ab158324c337c41f1cc7548f0bc0e130bbf0ae9452", [:rebar3], [], "hexpm", "73bc09fa59b4a0284efb4624335583c528e07ec9ae76aca96ea0673850aec57a"},
-  "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},
-}

route.eex

@@ -1,10 +0,0 @@
-<head>
-  <title>Farside Redirect - <%= service %></title>
-  <meta http-equiv="refresh" content="1; url=<%= instance_url %>">
-  <script>
-    history.pushState({page: 1}, "Farside Redirect");
-  </script>
-</head>
-<body>
-  <span>Redirecting to <%= service %> instance...
-</body>

server/index.html

@@ -0,0 +1,66 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+  <title>Farside</title>
+  <style>
+    html {
+      font-family: monospace;
+      font-size: 16px;
+      color: #66397C;
+    }
+    #parent-div {
+      text-align: center;
+    }
+    #child-div {
+      text-align: left;
+      width: 50%;
+      display: inline-block;
+    }
+    hr {
+      border: 1px dashed;
+    }
+    a:link, a:visited {
+      color: #66397C;
+    }
+    @media only screen and (max-width: 1000px) {
+      #child-div {
+        width: 90%;
+      }
+    }
+    ul {
+      margin: 10px;
+    }
+    @media (prefers-color-scheme: dark) {
+      html {
+        color: #fff;
+        background: #121517;
+      }
+      a:link, a:visited {
+        color: #AA8AC1;
+      }
+    }
+  </style>
+</head>
+<body>
+  <div id="parent-div">
+    <div id="child-div">
+      <h1>Farside [<a href="https://sr.ht/~benbusby/farside">SourceHut</a>, <a href="https://github.com/benbusby/farside">GitHub</a>]</h1>
+      <hr>
+      <h3>Updated: {{ .LastUpdated }}</h2>
+      <div>
+        <ul>
+		{{ range $i, $service := .ServiceList }}
+		<li><a href="/{{ $service.Type }}">{{ $service.Type }}</a></li>
+		<ul>
+			{{ range $j, $instance := $service.Instances }}
+			<li><a href="{{ $instance }}">{{ $instance }}</li>
+			{{ end  }}
+		</ul>
+		{{ end }}
+	</ul>
+      </div>
+    </div>
+  </div>
+</body>
+

server/route.html

@@ -0,0 +1,10 @@
+<head>
+	<title>Farside Redirect</title>
+	<meta http-equiv="refresh" content="1; url={{ .InstanceURL }}">
+	<script>
+		history.pushState({page: 1}, "Farside Redirect");
+	</script>
+</head>
+<body>
+	<span>Redirecting to {{ .InstanceURL }}...</span>
+</body>

server/server.go

@@ -0,0 +1,155 @@
+package server
+
+import (
+	_ "embed"
+	"encoding/json"
+	"fmt"
+	"html/template"
+	"log"
+	"net/http"
+	"net/url"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/benbusby/farside/db"
+	"github.com/benbusby/farside/services"
+)
+
+//go:embed index.html
+var indexHTML string
+
+//go:embed route.html
+var routeHTML string
+
+type indexData struct {
+	LastUpdated time.Time
+	ServiceList []services.Service
+}
+
+type routeData struct {
+	InstanceURL string
+}
+
+func home(w http.ResponseWriter, r *http.Request) {
+	serviceList := db.GetServiceList()
+	data := indexData{
+		LastUpdated: db.LastUpdate,
+		ServiceList: serviceList,
+	}
+
+	tmpl, err := template.New("").Parse(indexHTML)
+	if err != nil {
+		log.Println(err)
+		http.Error(w, "Error parsing template", http.StatusInternalServerError)
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/html")
+
+	err = tmpl.Execute(w, data)
+	if err != nil {
+		log.Println(err)
+		http.Error(w, "Error executing template", http.StatusInternalServerError)
+	}
+}
+
+func state(w http.ResponseWriter, r *http.Request) {
+	storedServices := db.GetServiceList()
+	jsonData, _ := json.Marshal(storedServices)
+	w.Header().Set("Content-Type", "application/json")
+	_, _ = w.Write(jsonData)
+}
+
+func baseRouting(w http.ResponseWriter, r *http.Request) {
+	routing(w, r, false, r.URL.RawQuery)
+}
+
+func jsRouting(w http.ResponseWriter, r *http.Request) {
+	r.URL.Path = strings.Replace(r.URL.Path, "/_", "", 1)
+	routing(w, r, true, r.URL.RawQuery)
+}
+
+func routing(w http.ResponseWriter, r *http.Request, jsEnabled bool, query string) {
+	value := r.PathValue("routing")
+	if len(value) == 0 {
+		value = r.URL.Path
+	}
+
+	parsedURL, _ := url.Parse(value)
+	path := strings.TrimPrefix(parsedURL.Path, "/")
+	segments := strings.Split(path, "/")
+
+	if len(segments[0]) == 0 {
+		http.Redirect(w, r, "", http.StatusTemporaryRedirect)
+	}
+
+	target, err := services.MatchRequest(segments[0])
+	if err != nil {
+		errMsg := fmt.Sprintf("No routing found for '%s'", segments[0])
+		log.Printf("Error during match request: %v\n", err)
+		http.Error(w, errMsg, http.StatusBadRequest)
+		return
+	}
+
+	var servicePath string
+	if target == "breezewiki" {
+		// Breezewiki requires the subdomain of the instance to be
+		// preserved for correct routing
+		splitDomain := strings.Split(path, ".")
+		if len(splitDomain) > 2 {
+			servicePath = strings.Split(path, ".")[0]
+		}
+	}
+
+	instance, err := db.GetInstance(target, servicePath)
+	if err != nil {
+		log.Printf("Error fetching instance from db: %v\n", err)
+		http.Error(
+			w,
+			"Error fetching instance for "+target,
+			http.StatusInternalServerError)
+		return
+	}
+
+	if len(segments) > 1 {
+		targetPath := strings.Join(segments[1:], "/")
+		instance = instance + "/" + targetPath
+	}
+
+	w.Header().Set("Cache-Control", "no-store, no-cache, must-revalidate, max-age=0")
+	w.Header().Set("Pragma", "no-cache")
+	w.Header().Set("Expires", "0")
+
+	instance += fmt.Sprintf("?%s", query)
+	if jsEnabled {
+		data := routeData{
+			InstanceURL: instance,
+		}
+		tmpl, _ := template.New("").Parse(routeHTML)
+		w.Header().Set("Content-Type", "text/html")
+		_ = tmpl.Execute(w, data)
+	} else {
+		http.Redirect(w, r, instance, http.StatusFound)
+	}
+}
+
+func RunServer() {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/{$}", home)
+	mux.HandleFunc("/state/{$}", state)
+	mux.HandleFunc("/{routing...}", baseRouting)
+	mux.HandleFunc("/_/{routing...}", jsRouting)
+
+	port := os.Getenv("FARSIDE_PORT")
+	if len(port) == 0 {
+		port = "4001"
+	}
+
+	log.Println("Starting server on http://localhost:" + port)
+
+	err := http.ListenAndServe(":"+port, mux)
+	if err != nil {
+		log.Fatal(err)
+	}
+}

server/server_test.go

@@ -0,0 +1,80 @@
+package server
+
+import (
+	"io"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/benbusby/farside/db"
+)
+
+const breezewikiTestSite = "https://breezewikitest.com"
+
+func TestMain(m *testing.M) {
+	err := db.InitializeDB()
+	if err != nil {
+		log.Fatalln("Failed to initialize database", err)
+	}
+
+	err = db.SetInstances("breezewiki", []string{breezewikiTestSite})
+	if err != nil {
+		log.Fatalln("Failed to set instances in db")
+	}
+
+	exitCode := m.Run()
+
+	_ = db.CloseDB()
+	os.Exit(exitCode)
+}
+
+func TestBaseRouting(t *testing.T) {
+	req := httptest.NewRequest(http.MethodGet, "/fandom.com", nil)
+	w := httptest.NewRecorder()
+
+	baseRouting(w, req)
+
+	res := w.Result()
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusFound {
+		t.Fatalf("Incorrect resp code (%d) in base routing", res.StatusCode)
+	}
+
+	expectedHost, _ := url.Parse(breezewikiTestSite)
+	redirect, err := res.Location()
+	if err != nil {
+		t.Fatalf("Error retrieving direct from request: %v\n", err)
+	} else if redirect.Host != expectedHost.Host {
+		t.Fatalf("Incorrect redirect site -- expected: %s, actual: %s\n",
+			expectedHost.Host,
+			redirect.Host)
+	}
+}
+
+func TestJSRouting(t *testing.T) {
+	req := httptest.NewRequest(http.MethodGet, "/_/fandom.com", nil)
+	w := httptest.NewRecorder()
+
+	jsRouting(w, req)
+
+	res := w.Result()
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		t.Fatalf("Incorrect resp code (%d) in base routing", res.StatusCode)
+	}
+
+	data, err := io.ReadAll(res.Body)
+	if err != nil {
+		t.Fatalf("Error reading response body: %v", err)
+	}
+
+	if !strings.Contains(string(data), breezewikiTestSite) {
+		t.Fatalf("%s not found in response body (%s)", breezewikiTestSite, string(data))
+	}
+}

services-full.json

@@ -0,0 +1,569 @@
+[
+  {
+    "type": "libreddit",
+    "test_url": "/r/popular",
+    "fallback": "https://redlib.freedit.eu",
+    "instances": [
+      "https://libreddit.privacydev.net",
+      "https://red.arancia.click",
+      "https://red.artemislena.eu",
+      "https://red.ngn.tf",
+      "https://reddit.nerdvpn.de",
+      "https://redlib.baczek.me",
+      "https://redlib.catsarch.com",
+      "https://redlib.ducks.party",
+      "https://redlib.kittywi.re",
+      "https://redlib.nadeko.net",
+      "https://redlib.perennialte.ch",
+      "https://redlib.privacy.com.de",
+      "https://redlib.privacyredirect.com",
+      "https://redlib.private.coffee",
+      "https://redlib.seasi.dev",
+      "https://rl.bloat.cat"
+    ]
+  },
+  {
+    "type": "redlib",
+    "test_url": "/r/popular",
+    "fallback": "https://redlib.freedit.eu",
+    "instances": [
+      "https://libreddit.privacydev.net",
+      "https://red.arancia.click",
+      "https://red.artemislena.eu",
+      "https://red.ngn.tf",
+      "https://reddit.nerdvpn.de",
+      "https://redlib.baczek.me",
+      "https://redlib.catsarch.com",
+      "https://redlib.ducks.party",
+      "https://redlib.kittywi.re",
+      "https://redlib.nadeko.net",
+      "https://redlib.perennialte.ch",
+      "https://redlib.privacy.com.de",
+      "https://redlib.privacyredirect.com",
+      "https://redlib.private.coffee",
+      "https://redlib.seasi.dev",
+      "https://rl.bloat.cat"
+    ]
+  },
+  {
+    "type": "teddit",
+    "test_url": "/r/popular",
+    "fallback": "https://teddit.net",
+    "instances": [
+      "https://i.opnxng.com",
+      "https://incogsnoo.com",
+      "https://rdt.trom.tf",
+      "https://reddit.lol",
+      "https://snoo.ioens.is",
+      "https://t.sneed.network",
+      "https://td.vern.cc",
+      "https://teddit.adminforge.de",
+      "https://teddit.bus-hit.me",
+      "https://teddit.domain.glass",
+      "https://teddit.froth.zone",
+      "https://teddit.garudalinux.org",
+      "https://teddit.ggc-project.de",
+      "https://teddit.hostux.net",
+      "https://teddit.httpjames.space",
+      "https://teddit.laserdisc.tokyo",
+      "https://teddit.net",
+      "https://teddit.no-logs.com",
+      "https://teddit.privacytools.io",
+      "https://teddit.projectsegfau.lt",
+      "https://teddit.pussthecat.org",
+      "https://teddit.rawbit.ninja",
+      "https://teddit.sethforprivacy.com",
+      "https://teddit.tinfoil-hat.net",
+      "https://teddit.tokhmi.xyz",
+      "https://teddit.xbdm.fun",
+      "https://teddit.zaggy.nl"
+    ]
+  },
+  {
+    "type": "proxigram",
+    "test_url": "/p/Ch2WRmiLMjj",
+    "fallback": "https://ig.opnxng.com",
+    "instances": [
+      "https://ig.opnxng.com",
+      "https://proxigram.lunar.icu",
+      "https://gram.whatever.social",
+      "https://ig.snine.nl",
+      "https://ig.floppa.one",
+      "https://proxigram.kyun.li"
+    ]
+  },
+  {
+    "type": "invidious",
+    "test_url": "/watch?v=eBGIQ7ZuuiU",
+    "fallback": "https://invidious.snopyta.org",
+    "instances": [
+      "https://inv.nadeko.net"
+    ]
+  },
+  {
+    "type": "piped",
+    "test_url": "/watch?v=dQw4w9WgXcQ|/streams/dQw4w9WgXcQ",
+    "fallback": "https://piped.video",
+    "instances": [
+      "https://piped.video|https://pipedapi.kavin.rocks",
+      "https://piped.tokhmi.xyz|https://pipedapi.tokhmi.xyz",
+      "https://piped.moomoo.me|https://pipedapi.moomoo.me",
+      "https://piped.mha.fi|https://api-piped.mha.fi",
+      "https://efy.piped.pages.dev|https://pipedapi.aeong.one",
+      "https://watch.whatever.social|https://watchapi.whatever.social",
+      "https://piped.garudalinux.org|https://piped-api.garudalinux.org",
+      "https://watch.leptons.xyz|https://pipedapi.leptons.xyz",
+      "https://piped.lunar.icu|https://piped-api.lunar.icu",
+      "https://piped.projectsegfau.lt|https://api.piped.projectsegfau.lt",
+      "https://piped.privacydev.net|https://api.piped.privacydev.net",
+      "https://piped.smnz.de|https://pipedapi.smnz.de",
+      "https://piped.adminforge.de|https://pipedapi.adminforge.de",
+      "https://piped.qdi.fi|https://pipedapi.qdi.fi",
+      "https://piped.hostux.net|https://piped-api.hostux.net",
+      "https://pd.vern.cc|https://pdapi.vern.cc",
+      "https://piped.colinslegacy.com|https://pipedapi.colinslegacy.com",
+      "https://piped.ducks.party|https://pipedapi.ducks.party"
+    ]
+  },
+  {
+    "type": "nitter",
+    "test_url": "/jack/status/20",
+    "fallback": "https://nitter.net",
+    "instances": [
+      "https://lightbrd.com",
+      "https://nitter.net",
+      "https://nitter.poast.org",
+      "https://nitter.privacydev.net",
+      "https://nitter.privacyredirect.com",
+      "https://nitter.space",
+      "https://xcancel.com"
+    ]
+  },
+  {
+    "type": "scribe",
+    "test_url": "/@ftrain/big-data-small-effort-b62607a43a8c",
+    "fallback": "https://scribe.rip",
+    "instances": [
+      "https://scribe.rip",
+      "https://scribe.nixnet.services",
+      "https://scribe.citizen4.eu",
+      "https://scribe.bus-hit.me",
+      "https://scribe.froth.zone",
+      "https://scribe.privacydev.net",
+      "https://scribe.rawbit.ninja",
+      "https://sc.vern.cc",
+      "https://m.opnxng.com",
+      "https://scribe.manasiwibi.com",
+      "https://scribe.r4fo.com",
+      "https://scribe.privacyredirect.com"
+    ]
+  },
+  {
+    "type": "simplytranslate",
+    "test_url": "/?engine=google&text=simplicado",
+    "fallback": "https://simplytranslate.org",
+    "instances": [
+      "https://simplytranslate.esmailelbob.xyz",
+      "https://simplytranslate.leemoon.network",
+      "https://simplytranslate.manerakai.com",
+      "https://simplytranslate.org",
+      "https://simplytranslate.pussthecat.org",
+      "https://st.odyssey346.dev",
+      "https://st.privacydev.net",
+      "https://st.tokhmi.xyz",
+      "https://tl.vern.cc",
+      "https://translate.bus-hit.me",
+      "https://translate.josias.dev",
+      "https://translate.northboot.xyz",
+      "https://translate.priv.pw",
+      "https://translate.riverside.rocks",
+      "https://translate.slipfox.xyz",
+      "https://translate.tiekoetter.com"
+    ]
+  },
+  {
+    "type": "lingva",
+    "test_url": "/auto/en/hola",
+    "fallback": "https://lingva.ml",
+    "instances": [
+      "https://lingva.ml",
+      "https://lingva.garudalinux.org",
+      "https://lingva.frontendfriendly.xyz",
+      "https://lingva.retiolus.net",
+      "https://translate.plausibility.cloud",
+      "https://lingva.lunar.icu",
+      "https://translate.dr460nf1r3.org",
+      "https://translate.whateveritworks.org",
+      "https://translate.sapti.me",
+      "https://lingva.fuk.world",
+      "https://translate.colinslegacy.com",
+      "https://translate.catswords.net"
+    ]
+  },
+  {
+    "type": "rimgo",
+    "test_url": "/a/H8M4rcp",
+    "fallback": "https://rimgo.projectsegfau.lt",
+    "instances": [
+      "https://i.habedieeh.re",
+      "https://imgur.010032.xyz",
+      "https://imgur.artemislena.eu",
+      "https://imgur.fsky.io",
+      "https://imgur.nerdvpn.de",
+      "https://ri.nadeko.net",
+      "https://rimgo.4o1x5.dev",
+      "https://rimgo.aketawi.space",
+      "https://rimgo.astrial.org",
+      "https://rimgo.bloat.cat",
+      "https://rimgo.bus-hit.me",
+      "https://rimgo.canine.tools",
+      "https://rimgo.catsarch.com",
+      "https://rimgo.darkness.services",
+      "https://rimgo.drgns.space",
+      "https://rimgo.ducks.party",
+      "https://rimgo.eu.projectsegfau.lt",
+      "https://rimgo.fascinated.cc",
+      "https://rimgo.frontendfriendly.xyz",
+      "https://rimgo.frylo.net",
+      "https://rimgo.gitro.xyz",
+      "https://rimgo.hostux.net",
+      "https://rimgo.in.projectsegfau.lt",
+      "https://rimgo.lunar.icu",
+      "https://rimgo.nohost.network",
+      "https://rimgo.perennialte.ch",
+      "https://rimgo.privacyredirect.com",
+      "https://rimgo.projectsegfau.lt",
+      "https://rimgo.pussthecat.org",
+      "https://rimgo.quantenzitrone.eu",
+      "https://rimgo.reallyaweso.me",
+      "https://rimgo.thebunny.zone",
+      "https://rimgo.totaldarkness.net",
+      "https://rimgo.us.projectsegfau.lt",
+      "https://rmgur.com"
+    ]
+  },
+  {
+    "type": "whoogle",
+    "test_url": "/search?cookies_disabled=1&q=<%=query%>",
+    "fallback": "https://search.sethforprivacy.com",
+    "instances": [
+      "https://search.garudalinux.org",
+      "https://search.sethforprivacy.com",
+      "https://wg.vern.cc",
+      "https://whoogle.4040940.xyz",
+      "https://whoogle.lunar.icu",
+      "https://whoogle.privacydev.net"
+    ]
+  },
+  {
+    "type": "searx",
+    "test_url": "/search?q=<%=query%>",
+    "fallback": "https://searx.be",
+    "instances": [
+      "https://search.stinpriza.org",
+      "https://searx.dresden.network",
+      "https://searx.gnu.style",
+      "https://searx.mastodontech.de",
+      "https://searx.mxchange.org",
+      "https://searx.nixnet.services",
+      "https://searx.roflcopter.fr",
+      "https://searx.ru",
+      "https://searx.zapashcanon.fr"
+    ]
+  },
+  {
+    "type": "searxng",
+    "test_url": "/search?q=<%=query%>",
+    "fallback": "https://searx.be",
+    "instances": [
+      "https://baresearch.org",
+      "https://kantan.cat",
+      "https://northboot.xyz",
+      "https://opnxng.com",
+      "https://paulgo.io",
+      "https://priv.au",
+      "https://s.datuan.dev",
+      "https://s.mble.dk",
+      "https://search.080609.xyz",
+      "https://search.canine.tools",
+      "https://search.catboy.house",
+      "https://search.einfachzocken.eu",
+      "https://search.gcomm.ch",
+      "https://search.im-in.space",
+      "https://search.ipv6s.net",
+      "https://search.leptons.xyz",
+      "https://search.mdosch.de",
+      "https://search.nerdvpn.de",
+      "https://search.ononoki.org",
+      "https://search.projectsegfau.lt",
+      "https://search.rhscz.eu",
+      "https://search.rowie.at",
+      "https://search.sapti.me",
+      "https://search.tinfoil.ch",
+      "https://search.url4irl.com",
+      "https://searx.ankha.ac",
+      "https://searx.be",
+      "https://searx.dresden.network",
+      "https://searx.foobar.vip",
+      "https://searx.foss.family",
+      "https://searx.namejeff.xyz",
+      "https://searx.oloke.xyz",
+      "https://searx.ox2.fr",
+      "https://searx.perennialte.ch",
+      "https://searx.rhscz.eu",
+      "https://searx.tiekoetter.com",
+      "https://searxng.f24o.zip",
+      "https://searxng.shreven.org",
+      "https://searxng.site",
+      "https://searxng.world",
+      "https://seek.fyi"
+    ]
+  },
+  {
+    "type": "wikiless",
+    "test_url": "/wiki/Wikipedia?lang=en",
+    "fallback": "https://wikiless.org",
+    "instances": [
+      "https://wiki.owo.si",
+      "https://wiki.adminforge.de",
+      "https://wiki.froth.zone",
+      "https://wiki.privacytools.io",
+      "https://wiki.slipfox.xyz",
+      "https://wikiless.esmailelbob.xyz",
+      "https://wikiless.funami.tech",
+      "https://wikiless.lunar.icu",
+      "https://wikiless.northboot.xyz",
+      "https://wikiless.org",
+      "https://wikiless.rawbit.ninja",
+      "https://wikiless.sethforprivacy.com",
+      "https://wikiless.tiekoetter.com",
+      "https://wl.vern.cc"
+    ]
+  },
+  {
+    "type": "librarian",
+    "test_url": "",
+    "fallback": "https://lbry.bcow.xyz",
+    "instances": [
+      "https://lbry.bcow.xyz",
+      "https://odysee.076.ne.jp",
+      "https://librarian.pussthecat.org",
+      "https://lbry.mutahar.rocks",
+      "https://librarian.esmailelbob.xyz",
+      "https://lbry.vern.cc"
+    ]
+  },
+  {
+    "type": "proxitok",
+    "test_url": "/discover",
+    "fallback": "https://proxitok.pabloferreiro.es",
+    "instances": [
+      "https://proxitok.pabloferreiro.es",
+      "https://proxitok.pussthecat.org",
+      "https://proxitok.privacydev.net",
+      "https://tok.habedieeh.re",
+      "https://proxitok.esmailelbob.xyz",
+      "https://tok.artemislena.eu",
+      "https://tok.adminforge.de",
+      "https://tik.hostux.net",
+      "https://proxitok.lunar.icu",
+      "https://proxitok.privacy.com.de",
+      "https://tiktok.chauvet.pro",
+      "https://cringe.whateveritworks.org"
+    ]
+  },
+  {
+    "type": "quetre",
+    "test_url": "/How-does-the-Z-boson-decay",
+    "fallback": "https://quetre.iket.me",
+    "instances": [
+      "https://quetre.iket.me",
+      "https://quora.vern.cc",
+      "https://quetre.pussthecat.org",
+      "https://quetre.tokhmi.xyz",
+      "https://quetre.projectsegfau.lt",
+      "https://quetre.esmailelbob.xyz",
+      "https://quetre.privacydev.net",
+      "https://ask.habedieeh.re",
+      "https://quetre.marcopisco.com",
+      "https://quetre.blackdrgn.nl",
+      "https://quetre.pufe.org",
+      "https://quetre.lunar.icu",
+      "https://que.wilbvr.me",
+      "https://questions.whateveritworks.org",
+      "https://quetre.fascinated.cc",
+      "https://quetre.catsarch.com",
+      "https://quetre.frontendfriendly.xyz",
+      "https://quetre.ducks.party"
+    ]
+  },
+  {
+    "type": "libremdb",
+    "test_url": "/title/tt0133093",
+    "fallback": "https://libremdb.herokuapp.com",
+    "instances": [
+      "https://libremdb.herokuapp.com",
+      "https://libremdb.pussthecat.org",
+      "https://libremdbeu.herokuapp.com",
+      "https://libremdb.esmailelbob.xyz",
+      "https://libremdb.iket.me",
+      "https://ld.vern.cc",
+      "https://binge.whatever.social",
+      "https://binge.whateveritworks.social",
+      "https://libremdb.lunar.icu",
+      "https://lmdb.hostux.net",
+      "https://libremdb.nerdyfam.tech",
+      "https://libremdb.fascinated.cc",
+      "https://libremdb.catsarch.com",
+      "https://libremdb.frontendfriendly.xyz",
+      "https://libremdb.ducks.party"
+    ]
+  },
+  {
+    "type": "dumb",
+    "test_url": "/Naughty-boy-la-la-la-lyrics",
+    "fallback": "https://dumb.privacydev.net",
+    "instances": [
+      "https://dumb.privacydev.net",
+      "https://dm.vern.cc",
+      "https://sing.whatever.social",
+      "https://dumb.nunosempere.com",
+      "https://dumb.lunar.icu",
+      "https://dumb.esmailelbob.xyz",
+      "https://dumb.ducks.party"
+    ]
+  },
+  {
+    "type": "breezewiki",
+    "test_url": "/balatrogame/wiki/Abandoned_Deck",
+    "fallback": "https://breezewiki.com",
+    "instances": [
+      "https://breezewiki.com",
+      "https://antifandom.com",
+      "https://breezewiki.pussthecat.org",
+      "https://bw.hamstro.dev",
+      "https://bw.projectsegfau.lt",
+      "https://breeze.hostux.net",
+      "https://bw.artemislena.eu",
+      "https://nerd.whatever.social",
+      "https://breezewiki.frontendfriendly.xyz",
+      "https://breeze.nohost.network",
+      "https://breeze.whateveritworks.org",
+      "https://z.opnxng.com",
+      "https://breezewiki.hyperreal.coffee",
+      "https://breezewiki.catsarch.com",
+      "https://breeze.mint.lgbt",
+      "https://breezewiki.woodland.cafe",
+      "https://breezewiki.nadeko.net",
+      "https://fandom.reallyaweso.me",
+      "https://breezewiki.4o1x5.dev",
+      "https://breezewiki.r4fo.com",
+      "https://breezewiki.private.coffee",
+      "https://fan.blitzw.in",
+      "http://bw.skunky7dhv7nohsoalpwe3sxfz3fbkad7r3wk632riye25vqm3meqead.onion",
+      "http://breezewiki.pjsfkvpxlinjamtawaksbnnaqs2fc2mtvmozrzckxh7f3kis6yea25ad.onion",
+      "http://breezewiki.nadekonfkhwlxwwk4ycbvq42zvcjmvo5iakl4tajojjwxd4a5dcetuyd.onion",
+      "http://breezewiki.catsarchywsyuss6jdxlypsw5dc7owd5u5tr6bujxb7o6xw2hipqehyd.onion",
+      "http://breezewiki.r4focoma7gu2zdwwcjjad47ysxt634lg73sxmdbkdozanwqslho5ohyd.onion",
+      "http://breezewiki.coffee2m3bjsrrqqycx6ghkxrnejl2q6nl7pjw2j4clchjj6uk5zozad.onion"
+    ]
+  },
+  {
+    "type": "gothub",
+    "test_url": "/benbusby/farside",
+    "fallback": "https://gothub.projectsegfau.lt",
+    "instances": [
+      "https://gothub.lunar.icu",
+      "https://g.opnxng.com",
+      "https://gh.owo.si",
+      "https://gothub.projectsegfau.lt",
+      "https://gothub.r4fo.com",
+      "https://gothub.dev.projectsegfau.lt",
+      "https://gh.phreedom.club",
+      "https://gothub.ducks.party"
+    ]
+  },
+  {
+    "type": "anonymousoverflow",
+    "test_url": "/questions/6591213/how-do-i-rename-a-local-git-branch",
+    "fallback": "https://code.whatever.social",
+    "instances": [
+      "https://ao.owo.si",
+      "https://code.whatever.social",
+      "https://ao.vern.cc",
+      "https://overflow.smnz.de",
+      "https://overflow.lunar.icu",
+      "https://overflow.adminforge.de",
+      "https://overflow.hostux.net",
+      "https://overflow.projectsegfau.lt",
+      "https://code.xbdm.fun",
+      "https://overflow.fascinated.cc",
+      "https://ao.bloatcat.tk",
+      "https://anonoverflow.frontendfriendly.xyz",
+      "https://overflow.ducks.party"
+    ]
+  },
+  {
+    "type": "4get",
+    "test_url": "/ami4get",
+    "fallback": "https://4get.ca",
+    "instances": [
+      "https://4g.ggtyler.dev",
+      "https://4g.opnxng.com",
+      "https://4get.ca",
+      "https://4get.dcs0.hu",
+      "https://4get.etenie.pl",
+      "https://4get.hbubli.cc",
+      "https://4get.kizuki.lol",
+      "https://4get.konakona.moe",
+      "https://4get.lunar.icu",
+      "https://4get.lvkaszus.pl",
+      "https://4get.perennialte.ch",
+      "https://4get.plunked.party",
+      "https://4get.psily.garden",
+      "https://4get.seitan-ayoub.lol",
+      "https://4get.sijh.net",
+      "https://4get.silly.computer",
+      "https://4get.zzls.xyz",
+      "https://4getus.zzls.xyz",
+      "https://4get.ducks.party",
+      "https://s.futureofthe.tech"
+    ]
+  },
+  {
+    "type": "librey",
+    "test_url": "/search.php?q=<%=query%>",
+    "fallback": "https://search.ahwx.org",
+    "instances": [
+      "https://libre.blitzw.in",
+      "https://librex.nohost.network",
+      "https://librey.4o1x5.dev",
+      "https://librey.baczek.me",
+      "https://librey.darkness.services",
+      "https://librey.nube-gran.de",
+      "https://librey.org",
+      "https://librey.sny.sh",
+      "https://ly.owo.si",
+      "https://search.davidovski.xyz",
+      "https://search.funami.tech",
+      "https://search.liv.town",
+      "https://search.revvy.de",
+      "https://search.technicalvoid.dev",
+      "https://search.uwabaki.party",
+      "https://serp.catswords.net"
+    ]
+  },
+  {
+    "type": "tent",
+    "test_url": "/search.php?query=<%=query%>",
+    "fallback": "https://tent.sny.sh",
+    "instances": [
+      "https://bandcamp.lurkmore.com",
+      "https://tent.bloat.cat",
+      "https://tent.deep-swarm.xyz",
+      "https://tent.qunn.link",
+      "https://tent.sny.sh",
+      "https://tn.vern.cc"
+    ]
+  }
+]

services.json

@@ -2,37 +2,45 @@
   {
     "type": "libreddit",
     "test_url": "/r/popular",
-    "fallback": "https://libredd.it",
+    "fallback": "https://redlib.freedit.eu",
     "instances": [
-      "https://libreddit.albony.xyz",
-      "https://libredd.it",
-      "https://libreddit.spike.codes",
-      "https://libreddit.dothq.co",
-      "https://libreddit.kavin.rocks",
-      "https://libreddit.40two.app",
-      "https://reddit.invak.id",
-      "https://reddit.phii.me",
-      "https://lr.riverside.rocks",
-      "https://libreddit.silkky.cloud",
-      "https://libreddit.database.red",
-      "https://libreddit.exonip.de",
-      "https://libreddit.domain.glass",
-      "https://libreddit.sugoma.tk",
-      "https://libreddit.trevorthalacker.com",
-      "https://reddit.artemislena.eu",
-      "https://r.nf",
-      "https://libreddit.awesomehub.io",
-      "https://libreddit.some-things.org",
-      "https://reddit.stuehieyr.com",
-      "https://lr.mint.lgbt",
-      "https://libreddit.alefvanoon.xyz",
-      "https://libreddit.igna.rocks",
-      "https://libreddit.autarkic.org",
-      "https://libreddit.flux.industries",
-      "https://libreddit.drivet.xyz",
-      "https://lr.oversold.host",
-      "https://libreddit.de",
-      "https://libreddit.sethforprivacy.com"
+      "https://libreddit.privacydev.net",
+      "https://red.arancia.click",
+      "https://red.artemislena.eu",
+      "https://red.ngn.tf",
+      "https://reddit.nerdvpn.de",
+      "https://redlib.baczek.me",
+      "https://redlib.catsarch.com",
+      "https://redlib.ducks.party",
+      "https://redlib.kittywi.re",
+      "https://redlib.nadeko.net",
+      "https://redlib.privacy.com.de",
+      "https://redlib.privacyredirect.com",
+      "https://redlib.private.coffee",
+      "https://redlib.seasi.dev",
+      "https://rl.bloat.cat"
+    ]
+  },
+  {
+    "type": "redlib",
+    "test_url": "/r/popular",
+    "fallback": "https://redlib.freedit.eu",
+    "instances": [
+      "https://libreddit.privacydev.net",
+      "https://red.arancia.click",
+      "https://red.artemislena.eu",
+      "https://red.ngn.tf",
+      "https://reddit.nerdvpn.de",
+      "https://redlib.baczek.me",
+      "https://redlib.catsarch.com",
+      "https://redlib.ducks.party",
+      "https://redlib.kittywi.re",
+      "https://redlib.nadeko.net",
+      "https://redlib.privacy.com.de",
+      "https://redlib.privacyredirect.com",
+      "https://redlib.private.coffee",
+      "https://redlib.seasi.dev",
+      "https://rl.bloat.cat"
     ]
   },
   {
@@ -40,35 +48,41 @@
     "test_url": "/r/popular",
     "fallback": "https://teddit.net",
     "instances": [
-      "https://teddit.ggc-project.de",
-      "https://teddit.kavin.rocks",
-      "https://teddit.zaggy.nl",
-      "https://teddit.namazso.eu",
-      "https://teddit.nautolan.racing",
-      "https://teddit.tinfoil-hat.net",
-      "https://teddit.domain.glass",
-      "https://snoo.ioens.is",
-      "https://teddit.httpjames.space",
-      "https://teddit.alefvanoon.xyz",
+      "https://i.opnxng.com",
       "https://incogsnoo.com",
-      "https://teddit.pussthecat.org",
+      "https://rdt.trom.tf",
       "https://reddit.lol",
-      "https://teddit.sethforprivacy.com"
+      "https://snoo.ioens.is",
+      "https://td.vern.cc",
+      "https://teddit.adminforge.de",
+      "https://teddit.bus-hit.me",
+      "https://teddit.froth.zone",
+      "https://teddit.garudalinux.org",
+      "https://teddit.ggc-project.de",
+      "https://teddit.hostux.net",
+      "https://teddit.laserdisc.tokyo",
+      "https://teddit.net",
+      "https://teddit.no-logs.com",
+      "https://teddit.projectsegfau.lt",
+      "https://teddit.pussthecat.org",
+      "https://teddit.sethforprivacy.com",
+      "https://teddit.tinfoil-hat.net",
+      "https://teddit.tokhmi.xyz",
+      "https://teddit.xbdm.fun",
+      "https://teddit.zaggy.nl"
     ]
   },
   {
-    "type": "bibliogram",
-    "test_url": "/u/officialrickastley",
-    "fallback": "https://bibliogram.art",
+    "type": "proxigram",
+    "test_url": "/p/Ch2WRmiLMjj",
+    "fallback": "https://ig.opnxng.com",
     "instances": [
-      "https://bibliogram.art",
-      "https://bibliogram.snopyta.org",
-      "https://bibliogram.pussthecat.org",
-      "https://bibliogram.1d4.us",
-      "https://insta.trom.tf",
-      "https://bibliogram.hamster.dance",
-      "https://bib.actionsack.com",
-      "https://biblio.alefvanoon.xyz"
+      "https://ig.opnxng.com",
+      "https://proxigram.lunar.icu",
+      "https://gram.whatever.social",
+      "https://ig.snine.nl",
+      "https://ig.floppa.one",
+      "https://proxigram.kyun.li"
     ]
   },
   {
@@ -76,73 +90,38 @@
     "test_url": "/watch?v=eBGIQ7ZuuiU",
     "fallback": "https://invidious.snopyta.org",
     "instances": [
-      "https://inv.cthd.icu",
-      "https://inv.riverside.rocks",
-      "https://invidio.xamh.de",
-      "https://invidious-us.kavin.rocks",
-      "https://invidious.kavin.rocks",
-      "https://invidious.namazso.eu",
-      "https://invidious.osi.kr",
-      "https://invidious.snopyta.org",
-      "https://vid.puffyan.us",
-      "https://yewtu.be",
-      "https://youtube.076.ne.jp",
-      "https://yt.artemislena.eu"
+      "https://inv.nadeko.net"
     ]
   },
   {
     "type": "piped",
-    "test_url": "/watch?v=eBGIQ7ZuuiU",
-    "fallback": "https://piped.kavin.rocks",
+    "test_url": "/watch?v=dQw4w9WgXcQ|/streams/dQw4w9WgXcQ",
+    "fallback": "https://piped.video",
     "instances": [
-      "https://piped.kavin.rocks",
-      "https://piped.silkky.cloud",
-      "https://piped.mutahar.rocks",
-      "https://piped.albony.xyz"
+      "https://watch.whatever.social|https://watchapi.whatever.social",
+      "https://piped.garudalinux.org|https://piped-api.garudalinux.org",
+      "https://piped.lunar.icu|https://piped-api.lunar.icu",
+      "https://piped.projectsegfau.lt|https://api.piped.projectsegfau.lt",
+      "https://piped.privacydev.net|https://api.piped.privacydev.net",
+      "https://piped.smnz.de|https://pipedapi.smnz.de",
+      "https://piped.adminforge.de|https://pipedapi.adminforge.de",
+      "https://piped.qdi.fi|https://pipedapi.qdi.fi",
+      "https://piped.hostux.net|https://piped-api.hostux.net",
+      "https://pd.vern.cc|https://pdapi.vern.cc",
+      "https://piped.colinslegacy.com|https://pipedapi.colinslegacy.com",
+      "https://piped.ducks.party|https://pipedapi.ducks.party"
     ]
   },
   {
     "type": "nitter",
-    "test_url": "/twitter",
+    "test_url": "/jack/status/20",
     "fallback": "https://nitter.net",
     "instances": [
       "https://nitter.net",
-      "https://nitter.42l.fr",
-      "https://nitter.pussthecat.org",
-      "https://nitter.nixnet.services",
-      "https://nitter.fdn.fr",
-      "https://nitter.1d4.us",
-      "https://nitter.kavin.rocks",
-      "https://nitter.vxempire.xyz",
-      "https://nitter.unixfox.eu",
-      "https://nitter.domain.glass",
-      "https://nitter.eu",
-      "https://nitter.namazso.eu",
-      "https://nitter.mailstation.de",
-      "https://nitter.actionsack.com",
-      "https://nitter.cattube.org",
-      "https://birdsite.xanny.family",
-      "https://nitter.hu",
-      "https://nitter.exonip.de",
-      "https://twitr.gq",
-      "https://nitter.moomoo.me",
-      "https://bird.trom.tf",
-      "https://nitter.it",
-      "https://twitter.censors.us",
-      "https://nitter.grimneko.de",
-      "https://nitter.koyu.space",
-      "https://nitter.alefvanoon.xyz",
-      "https://nitter.ir",
-      "https://nitter.autarkic.org",
-      "https://n.0x0.st",
-      "https://n.hyperborea.cloud",
-      "https://nitter.ca",
-      "https://twitter.076.ne.jp",
-      "https://lu-nitter.resolv.ee",
-      "https://is-nitter.resolv.ee",
-      "https://cy-nitter.resolv.ee",
-      "https://tweet.lambda.dance",
-      "https://nitter.sethforprivacy.com"
+      "https://nitter.poast.org",
+      "https://nitter.privacydev.net",
+      "https://nitter.privacyredirect.com",
+      "https://xcancel.com"
     ]
   },
   {
@@ -152,7 +131,15 @@
     "instances": [
       "https://scribe.rip",
       "https://scribe.nixnet.services",
-      "https://scribe.citizen4.eu"
+      "https://scribe.citizen4.eu",
+      "https://scribe.bus-hit.me",
+      "https://scribe.froth.zone",
+      "https://scribe.privacydev.net",
+      "https://sc.vern.cc",
+      "https://m.opnxng.com",
+      "https://scribe.manasiwibi.com",
+      "https://scribe.r4fo.com",
+      "https://scribe.privacyredirect.com"
     ]
   },
   {
@@ -160,14 +147,22 @@
     "test_url": "/?engine=google&text=simplicado",
     "fallback": "https://simplytranslate.org",
     "instances": [
-      "https://manerakai.asuscomm.com:447",
+      "https://simplytranslate.esmailelbob.xyz",
+      "https://simplytranslate.leemoon.network",
+      "https://simplytranslate.manerakai.com",
       "https://simplytranslate.org",
       "https://simplytranslate.pussthecat.org",
-      "https://st.alefvanoon.xyz",
+      "https://st.odyssey346.dev",
+      "https://st.privacydev.net",
+      "https://st.tokhmi.xyz",
+      "https://tl.vern.cc",
       "https://translate.bus-hit.me",
       "https://translate.josias.dev",
-      "https://translate.namazso.eu",
-      "https://translate.riverside.rocks"
+      "https://translate.northboot.xyz",
+      "https://translate.priv.pw",
+      "https://translate.riverside.rocks",
+      "https://translate.slipfox.xyz",
+      "https://translate.tiekoetter.com"
     ]
   },
   {
@@ -175,38 +170,66 @@
     "test_url": "/auto/en/hola",
     "fallback": "https://lingva.ml",
     "instances": [
-      "https://lingva.ml",
-      "https://translate.alefvanoon.xyz",
-      "https://translate.igna.rocks",
-      "https://lingva.pussthecat.org"
+      "https://lingva.frontendfriendly.xyz",
+      "https://lingva.retiolus.net",
+      "https://translate.plausibility.cloud",
+      "https://lingva.lunar.icu",
+      "https://translate.dr460nf1r3.org",
+      "https://translate.whateveritworks.org",
+      "https://translate.sapti.me",
+      "https://translate.colinslegacy.com",
+      "https://translate.catswords.net"
     ]
   },
   {
     "type": "rimgo",
     "test_url": "/a/H8M4rcp",
-    "fallback": "https://i.bcow.xyz",
+    "fallback": "https://rimgo.projectsegfau.lt",
     "instances": [
-      "https://i.bcow.xyz",
+      "https://i.habedieeh.re",
+      "https://imgur.010032.xyz",
+      "https://imgur.artemislena.eu",
+      "https://imgur.fsky.io",
+      "https://imgur.nerdvpn.de",
+      "https://ri.nadeko.net",
+      "https://rimgo.4o1x5.dev",
+      "https://rimgo.astrial.org",
+      "https://rimgo.bloat.cat",
+      "https://rimgo.bus-hit.me",
+      "https://rimgo.canine.tools",
+      "https://rimgo.catsarch.com",
+      "https://rimgo.darkness.services",
+      "https://rimgo.drgns.space",
+      "https://rimgo.ducks.party",
+      "https://rimgo.eu.projectsegfau.lt",
+      "https://rimgo.frontendfriendly.xyz",
+      "https://rimgo.frylo.net",
+      "https://rimgo.gitro.xyz",
+      "https://rimgo.hostux.net",
+      "https://rimgo.in.projectsegfau.lt",
+      "https://rimgo.lunar.icu",
+      "https://rimgo.nohost.network",
+      "https://rimgo.perennialte.ch",
+      "https://rimgo.privacyredirect.com",
+      "https://rimgo.projectsegfau.lt",
       "https://rimgo.pussthecat.org",
-      "https://img.riverside.rocks",
+      "https://rimgo.quantenzitrone.eu",
+      "https://rimgo.reallyaweso.me",
+      "https://rimgo.thebunny.zone",
       "https://rimgo.totaldarkness.net",
-      "https://rimgo.bus-hit.me"
+      "https://rimgo.us.projectsegfau.lt",
+      "https://rmgur.com"
     ]
   },
   {
     "type": "whoogle",
     "test_url": "/search?cookies_disabled=1&q=<%=query%>",
-    "fallback": "https://whoogle.fossho.st",
+    "fallback": "https://search.sethforprivacy.com",
     "instances": [
-      "https://s.alefvanoon.xyz",
-      "https://search.albony.xyz",
-      "https://search.exonip.de",
-      "https://search.garudalinux.org",
       "https://search.sethforprivacy.com",
-      "https://whoogle.dcs0.hu",
-      "https://whoogle.fossho.st",
-      "https://whooglesearch.net",
-      "https://www.whooglesearch.ml"
+      "https://wg.vern.cc",
+      "https://whoogle.lunar.icu",
+      "https://whoogle.privacydev.net"
     ]
   },
   {
@@ -214,39 +237,302 @@
     "test_url": "/search?q=<%=query%>",
     "fallback": "https://searx.be",
     "instances": [
-      "https://anon.sx",
-      "https://northboot.xyz",
-      "https://paulgo.io",
-      "https://procurx.pt",
-      "https://s.zhaocloud.net",
-      "https://search.asynchronousexchange.com",
-      "https://search.bus-hit.me",
-      "https://search.mdosch.de",
-      "https://search.ononoki.org",
-      "https://searx.bar",
-      "https://searx.be",
-      "https://searx.divided-by-zero.eu",
-      "https://searx.fmac.xyz",
+      "https://search.stinpriza.org",
+      "https://searx.dresden.network",
       "https://searx.gnu.style",
-      "https://searx.hummel-web.at",
-      "https://searx.mha.fi",
-      "https://searx.nevrlands.de",
-      "https://searx.prvcy.eu",
-      "https://searx.rasp.fr",
+      "https://searx.mastodontech.de",
+      "https://searx.mxchange.org",
+      "https://searx.nixnet.services",
+      "https://searx.roflcopter.fr",
       "https://searx.ru",
-      "https://searx.sp-codes.de",
-      "https://searx.stuehieyr.com",
-      "https://searx.theanonymouse.xyz",
+      "https://searx.zapashcanon.fr"
+    ]
+  },
+  {
+    "type": "searxng",
+    "test_url": "/search?q=<%=query%>",
+    "fallback": "https://searx.be",
+    "instances": [
+      "https://baresearch.org",
+      "https://kantan.cat",
+      "https://northboot.xyz",
+      "https://opnxng.com",
+      "https://paulgo.io",
+      "https://priv.au",
+      "https://s.datuan.dev",
+      "https://s.mble.dk",
+      "https://search.080609.xyz",
+      "https://search.canine.tools",
+      "https://search.catboy.house",
+      "https://search.einfachzocken.eu",
+      "https://search.gcomm.ch",
+      "https://search.im-in.space",
+      "https://search.ipv6s.net",
+      "https://search.leptons.xyz",
+      "https://search.mdosch.de",
+      "https://search.nerdvpn.de",
+      "https://search.ononoki.org",
+      "https://search.projectsegfau.lt",
+      "https://search.rhscz.eu",
+      "https://search.rowie.at",
+      "https://search.sapti.me",
+      "https://search.tinfoil.ch",
+      "https://search.url4irl.com",
+      "https://searx.ankha.ac",
+      "https://searx.be",
+      "https://searx.dresden.network",
+      "https://searx.foobar.vip",
+      "https://searx.foss.family",
+      "https://searx.namejeff.xyz",
+      "https://searx.oloke.xyz",
+      "https://searx.ox2.fr",
+      "https://searx.perennialte.ch",
+      "https://searx.rhscz.eu",
       "https://searx.tiekoetter.com",
-      "https://searx.tux.land",
-      "https://searx.tuxcloud.net",
-      "https://searx.webheberg.info",
-      "https://searx.xyz",
-      "https://searx.zackptg5.com",
-      "https://serx.cf",
-      "https://swag.pw",
-      "https://sx.fedi.tech",
-      "https://www.webrats.xyz"
+      "https://searxng.f24o.zip",
+      "https://searxng.shreven.org",
+      "https://searxng.site",
+      "https://searxng.world",
+      "https://seek.fyi"
+    ]
+  },
+  {
+    "type": "wikiless",
+    "test_url": "/wiki/Wikipedia?lang=en",
+    "fallback": "https://wikiless.org",
+    "instances": [
+      "https://wiki.owo.si",
+      "https://wiki.adminforge.de",
+      "https://wiki.froth.zone",
+      "https://wiki.slipfox.xyz",
+      "https://wikiless.esmailelbob.xyz",
+      "https://wikiless.funami.tech",
+      "https://wikiless.lunar.icu",
+      "https://wikiless.northboot.xyz",
+      "https://wikiless.org",
+      "https://wikiless.sethforprivacy.com",
+      "https://wikiless.tiekoetter.com",
+      "https://wl.vern.cc"
+    ]
+  },
+  {
+    "type": "librarian",
+    "test_url": "",
+    "fallback": "https://lbry.bcow.xyz",
+    "instances": [
+      "https://lbry.bcow.xyz",
+      "https://odysee.076.ne.jp",
+      "https://librarian.pussthecat.org",
+      "https://lbry.mutahar.rocks",
+      "https://librarian.esmailelbob.xyz",
+      "https://lbry.vern.cc"
+    ]
+  },
+  {
+    "type": "proxitok",
+    "test_url": "/discover",
+    "fallback": "https://proxitok.pabloferreiro.es",
+    "instances": [
+      "https://proxitok.pabloferreiro.es",
+      "https://proxitok.pussthecat.org",
+      "https://proxitok.privacydev.net",
+      "https://tok.habedieeh.re",
+      "https://proxitok.esmailelbob.xyz",
+      "https://tok.artemislena.eu",
+      "https://tok.adminforge.de",
+      "https://tik.hostux.net",
+      "https://proxitok.lunar.icu",
+      "https://proxitok.privacy.com.de",
+      "https://tiktok.chauvet.pro",
+      "https://cringe.whateveritworks.org"
+    ]
+  },
+  {
+    "type": "quetre",
+    "test_url": "/How-does-the-Z-boson-decay",
+    "fallback": "https://quetre.iket.me",
+    "instances": [
+      "https://quetre.iket.me",
+      "https://quora.vern.cc",
+      "https://quetre.pussthecat.org",
+      "https://quetre.tokhmi.xyz",
+      "https://quetre.projectsegfau.lt",
+      "https://quetre.esmailelbob.xyz",
+      "https://quetre.privacydev.net",
+      "https://ask.habedieeh.re",
+      "https://quetre.marcopisco.com",
+      "https://quetre.blackdrgn.nl",
+      "https://quetre.pufe.org",
+      "https://quetre.lunar.icu",
+      "https://que.wilbvr.me",
+      "https://questions.whateveritworks.org",
+      "https://quetre.catsarch.com",
+      "https://quetre.frontendfriendly.xyz",
+      "https://quetre.ducks.party"
+    ]
+  },
+  {
+    "type": "libremdb",
+    "test_url": "/title/tt0133093",
+    "fallback": "https://libremdb.herokuapp.com",
+    "instances": [
+      "https://libremdb.herokuapp.com",
+      "https://libremdb.pussthecat.org",
+      "https://libremdbeu.herokuapp.com",
+      "https://libremdb.esmailelbob.xyz",
+      "https://libremdb.iket.me",
+      "https://ld.vern.cc",
+      "https://binge.whatever.social",
+      "https://binge.whateveritworks.social",
+      "https://libremdb.lunar.icu",
+      "https://lmdb.hostux.net",
+      "https://libremdb.catsarch.com",
+      "https://libremdb.frontendfriendly.xyz",
+      "https://libremdb.ducks.party"
+    ]
+  },
+  {
+    "type": "dumb",
+    "test_url": "/Naughty-boy-la-la-la-lyrics",
+    "fallback": "https://dumb.privacydev.net",
+    "instances": [
+      "https://dumb.privacydev.net",
+      "https://dm.vern.cc",
+      "https://sing.whatever.social",
+      "https://dumb.nunosempere.com",
+      "https://dumb.lunar.icu",
+      "https://dumb.esmailelbob.xyz",
+      "https://dumb.ducks.party"
+    ]
+  },
+  {
+    "type": "breezewiki",
+    "test_url": "/balatrogame/wiki/Abandoned_Deck",
+    "fallback": "https://breezewiki.com",
+    "instances": [
+      "https://breezewiki.com",
+      "https://antifandom.com",
+      "https://breezewiki.pussthecat.org",
+      "https://bw.hamstro.dev",
+      "https://bw.projectsegfau.lt",
+      "https://breeze.hostux.net",
+      "https://bw.artemislena.eu",
+      "https://nerd.whatever.social",
+      "https://breezewiki.frontendfriendly.xyz",
+      "https://breeze.nohost.network",
+      "https://breeze.whateveritworks.org",
+      "https://z.opnxng.com",
+      "https://breezewiki.hyperreal.coffee",
+      "https://breezewiki.catsarch.com",
+      "https://breeze.mint.lgbt",
+      "https://breezewiki.woodland.cafe",
+      "https://breezewiki.nadeko.net",
+      "https://fandom.reallyaweso.me",
+      "https://breezewiki.4o1x5.dev",
+      "https://breezewiki.r4fo.com",
+      "https://breezewiki.private.coffee",
+      "https://fan.blitzw.in",
+      "http://bw.skunky7dhv7nohsoalpwe3sxfz3fbkad7r3wk632riye25vqm3meqead.onion",
+      "http://breezewiki.pjsfkvpxlinjamtawaksbnnaqs2fc2mtvmozrzckxh7f3kis6yea25ad.onion",
+      "http://breezewiki.nadekonfkhwlxwwk4ycbvq42zvcjmvo5iakl4tajojjwxd4a5dcetuyd.onion",
+      "http://breezewiki.catsarchywsyuss6jdxlypsw5dc7owd5u5tr6bujxb7o6xw2hipqehyd.onion",
+      "http://breezewiki.r4focoma7gu2zdwwcjjad47ysxt634lg73sxmdbkdozanwqslho5ohyd.onion",
+      "http://breezewiki.coffee2m3bjsrrqqycx6ghkxrnejl2q6nl7pjw2j4clchjj6uk5zozad.onion"
+    ]
+  },
+  {
+    "type": "gothub",
+    "test_url": "/benbusby/farside",
+    "fallback": "https://gothub.projectsegfau.lt",
+    "instances": [
+      "https://gothub.lunar.icu",
+      "https://g.opnxng.com",
+      "https://gh.owo.si",
+      "https://gothub.projectsegfau.lt",
+      "https://gothub.r4fo.com",
+      "https://gothub.dev.projectsegfau.lt",
+      "https://gh.phreedom.club",
+      "https://gothub.ducks.party"
+    ]
+  },
+  {
+    "type": "anonymousoverflow",
+    "test_url": "/questions/6591213/how-do-i-rename-a-local-git-branch",
+    "fallback": "https://code.whatever.social",
+    "instances": [
+      "https://ao.owo.si",
+      "https://code.whatever.social",
+      "https://ao.vern.cc",
+      "https://overflow.smnz.de",
+      "https://overflow.lunar.icu",
+      "https://overflow.adminforge.de",
+      "https://overflow.hostux.net",
+      "https://overflow.projectsegfau.lt",
+      "https://code.xbdm.fun",
+      "https://ao.bloatcat.tk",
+      "https://anonoverflow.frontendfriendly.xyz",
+      "https://overflow.ducks.party"
+    ]
+  },
+  {
+    "type": "4get",
+    "test_url": "/ami4get",
+    "fallback": "https://4get.ca",
+    "instances": [
+      "https://4g.ggtyler.dev",
+      "https://4g.opnxng.com",
+      "https://4get.ca",
+      "https://4get.dcs0.hu",
+      "https://4get.etenie.pl",
+      "https://4get.hbubli.cc",
+      "https://4get.kizuki.lol",
+      "https://4get.konakona.moe",
+      "https://4get.lunar.icu",
+      "https://4get.lvkaszus.pl",
+      "https://4get.perennialte.ch",
+      "https://4get.psily.garden",
+      "https://4get.seitan-ayoub.lol",
+      "https://4get.silly.computer",
+      "https://4get.zzls.xyz",
+      "https://4getus.zzls.xyz",
+      "https://4get.ducks.party",
+      "https://s.futureofthe.tech"
+    ]
+  },
+  {
+    "type": "librey",
+    "test_url": "/search.php?q=<%=query%>",
+    "fallback": "https://search.ahwx.org",
+    "instances": [
+      "https://libre.blitzw.in",
+      "https://librex.nohost.network",
+      "https://librey.4o1x5.dev",
+      "https://librey.baczek.me",
+      "https://librey.darkness.services",
+      "https://librey.nube-gran.de",
+      "https://librey.org",
+      "https://librey.sny.sh",
+      "https://ly.owo.si",
+      "https://search.davidovski.xyz",
+      "https://search.funami.tech",
+      "https://search.liv.town",
+      "https://search.revvy.de",
+      "https://search.technicalvoid.dev",
+      "https://search.uwabaki.party",
+      "https://serp.catswords.net"
+    ]
+  },
+  {
+    "type": "tent",
+    "test_url": "/search.php?query=<%=query%>",
+    "fallback": "https://tent.sny.sh",
+    "instances": [
+      "https://bandcamp.lurkmore.com",
+      "https://tent.bloat.cat",
+      "https://tent.deep-swarm.xyz",
+      "https://tent.qunn.link",
+      "https://tent.sny.sh",
+      "https://tn.vern.cc"
     ]
   }
 ]

services/mappings.go

@@ -0,0 +1,143 @@
+package services
+
+import (
+	"errors"
+	"math/rand"
+	"regexp"
+	"strings"
+)
+
+type RegexMapping struct {
+	Pattern *regexp.Regexp
+	Targets []string
+}
+
+var regexMap = []RegexMapping{
+	{
+		// YouTube
+		Pattern: regexp.MustCompile(`youtu(\.be|be\.com)|invidious|piped`),
+		Targets: []string{"piped", "invidious"},
+	},
+	{
+		// Twitter / X
+		Pattern: regexp.MustCompile(`twitter\.com|x\.com|nitter`),
+		Targets: []string{"nitter"},
+	},
+	{
+		// Reddit
+		Pattern: regexp.MustCompile(`reddit\.com|libreddit|redlib|teddit`),
+		Targets: []string{"libreddit", "redlib", "teddit"},
+	},
+	{
+		// Google Search
+		Pattern: regexp.MustCompile(`google\.com|whoogle|searx|searxng`),
+		Targets: []string{"whoogle", "searxng"},
+	},
+	{
+		// Instagram
+		Pattern: regexp.MustCompile(`instagram\.com|proxigram`),
+		Targets: []string{"proxigram"},
+	},
+	{
+		// Wikipedia
+		Pattern: regexp.MustCompile(`wikipedia\.org|wikiless`),
+		Targets: []string{"wikiless"},
+	},
+	{
+		// Medium
+		Pattern: regexp.MustCompile(`medium\.com|scribe`),
+		Targets: []string{"scribe"},
+	},
+	{
+		// Odysee
+		Pattern: regexp.MustCompile(`odysee\.com|librarian`),
+		Targets: []string{"librarian"},
+	},
+	{
+		// Imgur
+		Pattern: regexp.MustCompile(`imgur\.com|rimgo`),
+		Targets: []string{"rimgo"},
+	},
+	{
+		// Google Translate
+		Pattern: regexp.MustCompile(`translate\.google\.com|lingva|simplytranslate`),
+		Targets: []string{"lingva", "simplytranslate"},
+	},
+	{
+		// TikTok
+		Pattern: regexp.MustCompile(`tiktok\.com|proxitok`),
+		Targets: []string{"proxitok"},
+	},
+	{
+		// Fandom
+		Pattern: regexp.MustCompile(`.*fandom\.com|breezewiki`),
+		Targets: []string{"breezewiki"},
+	},
+	{
+		// IMDB
+		Pattern: regexp.MustCompile(`imdb\.com|libremdb`),
+		Targets: []string{"libremdb"},
+	},
+	{
+		// Quora
+		Pattern: regexp.MustCompile(`quora\.com|quetre`),
+		Targets: []string{"quetre"},
+	},
+	{
+		// GitHub
+		Pattern: regexp.MustCompile(`github\.com|gothub`),
+		Targets: []string{"gothub"},
+	},
+	{
+		// StackOverflow
+		Pattern: regexp.MustCompile(`stackoverflow\.com|anonymousoverflow`),
+		Targets: []string{"anonymousoverflow"},
+	},
+	{
+		// Genius
+		Pattern: regexp.MustCompile(`genius\.com|dumb`),
+		Targets: []string{"dumb"},
+	},
+	{
+		// 4get
+		// Note: Could be used for redirecting other search engine
+		// requests, but would need special handling
+		Pattern: regexp.MustCompile("4get"),
+		Targets: []string{"4get"},
+	},
+	{
+		// LibreY
+		// Note: Could be used for redirecting other search engine
+		// requests, but would need special handling
+		Pattern: regexp.MustCompile("librex|librey"),
+		Targets: []string{"librey"},
+	},
+	{
+		// Tent
+		// Note: This is a Bandcamp alternative, but the endpoints are
+		// completely different than Bandcamp, so 1-to-1 mapping of URLs
+		// is not possible without some additional work
+		Pattern: regexp.MustCompile("tent"),
+		Targets: []string{"tent"},
+	},
+}
+
+func MatchRequest(service string) (string, error) {
+
+	for _, mapping := range regexMap {
+		hasMatch := mapping.Pattern.MatchString(service)
+		if !hasMatch {
+			continue
+		}
+
+		if !strings.Contains(service, ".") {
+			return service, nil
+		}
+
+		index := rand.Intn(len(mapping.Targets))
+		value := mapping.Targets[index]
+		return value, nil
+	}
+
+	return "", errors.New("no match found")
+}

services/services.go

@@ -0,0 +1,92 @@
+package services
+
+import (
+	"encoding/json"
+	"io"
+	"net/http"
+	"os"
+)
+
+var (
+	ServiceList []Service
+	FallbackMap map[string]string
+)
+
+const (
+	baseRepoLink = "https://git.sr.ht/~benbusby/farside/blob/main/"
+
+	noCFServicesJSON = "services.json"
+	fullServicesJSON = "services-full.json"
+)
+
+type Service struct {
+	Type      string   `json:"type"`
+	TestURL   string   `json:"test_url,omitempty"`
+	Fallback  string   `json:"fallback,omimtempty"`
+	Instances []string `json:"instances"`
+}
+
+func ingestServicesList(servicesBytes []byte) error {
+	err := json.Unmarshal(servicesBytes, &ServiceList)
+	return err
+}
+
+func GetServicesFileName() string {
+	cloudflareEnabled := false
+
+	cfEnabledVar := os.Getenv("FARSIDE_CF_ENABLED")
+	if len(cfEnabledVar) > 0 && cfEnabledVar == "1" {
+		cloudflareEnabled = true
+	}
+
+	serviceJSON := noCFServicesJSON
+	if cloudflareEnabled {
+		serviceJSON = fullServicesJSON
+	}
+
+	return serviceJSON
+}
+
+func FetchServicesFile(serviceJSON string) ([]byte, error) {
+	resp, err := http.Get(baseRepoLink + serviceJSON)
+	if err != nil {
+		return nil, err
+	}
+
+	defer resp.Body.Close()
+
+	bodyBytes, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	err = os.WriteFile(serviceJSON, bodyBytes, 0666)
+	if err != nil {
+		return nil, err
+	}
+
+	return bodyBytes, nil
+}
+
+func InitializeServices() error {
+	serviceJSON := GetServicesFileName()
+	fileBytes, err := os.ReadFile(serviceJSON)
+	if err != nil {
+		fileBytes, err = FetchServicesFile(serviceJSON)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = ingestServicesList(fileBytes)
+	if err != nil {
+		return err
+	}
+
+	FallbackMap = make(map[string]string)
+	for _, serviceElement := range ServiceList {
+		FallbackMap[serviceElement.Type] = serviceElement.Fallback
+	}
+
+	return nil
+}

test/farside_test.exs

@@ -1,75 +0,0 @@
-defmodule FarsideTest do
-  @services_json Application.fetch_env!(:farside, :services_json)
-
-  use ExUnit.Case
-  use Plug.Test
-
-  alias Farside.Router
-
-  @opts Router.init([])
-
-  def test_conn(path) do
-    :timer.sleep(1000)
-
-    :get
-    |> conn(path, "")
-    |> Router.call(@opts)
-  end
-
-  test "throttle" do
-    :get
-    |> conn("/", "")
-    |> Router.call(@opts)
-
-    throttled_conn =
-      :get
-      |> conn("/", "")
-      |> Router.call(@opts)
-
-    assert throttled_conn.state == :sent
-    assert throttled_conn.status == 403
-  end
-
-  test "/" do
-    conn = test_conn("/")
-    assert conn.state == :sent
-    assert conn.status == 200
-  end
-
-  test "/ping" do
-    conn = test_conn("/ping")
-    assert conn.state == :sent
-    assert conn.status == 200
-    assert conn.resp_body == "PONG"
-  end
-
-  test "/:service" do
-    {:ok, file} = File.read(@services_json)
-    {:ok, service_list} = Poison.decode(file, as: [%{}])
-
-    service_names =
-      Enum.map(
-        service_list,
-        fn service -> service["type"] end
-      )
-
-    IO.puts("")
-
-    Enum.map(service_names, fn service_name ->
-      conn = test_conn("/#{service_name}")
-      first_redirect = elem(List.last(conn.resp_headers), 1)
-
-      IO.puts("    /#{service_name} (#1) -- #{first_redirect}")
-      assert conn.state == :set
-      assert conn.status == 302
-
-      conn = test_conn("/#{service_name}")
-      second_redirect = elem(List.last(conn.resp_headers), 1)
-
-      IO.puts("    /#{service_name} (#2) -- #{second_redirect}")
-      assert conn.state == :set
-      assert conn.status == 302
-      assert first_redirect != second_redirect
-    end)
-  end
-end

test/test_helper.exs

@@ -1 +0,0 @@
-ExUnit.start()

tools/un-cloudflare.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+# Remove cloudflare instances from services-full.json
+
+rm -f out.json
+file="services-full.json"
+
+while read -r line; do
+    if [[ "$line" == "\"https://"* ]]; then
+        domain=$(echo "$line" | sed -e "s/^\"https:\/\///" -e "s/\",//" -e "s/\"//" | awk -F  '|' '{print $1}')
+        ips=$(dig "$domain" +short || true)
+        cf=0
+        echo "$domain"
+
+        for ip in $ips
+        do
+            echo "    - $ip"
+            resp=$(curl --connect-timeout 5 --max-time 5 -s "$ip")
+
+            # Cloudflare does not allow accessing sites using their IP,
+            # and returns a 1003 error code when attempting to do so. This
+            # allows us to check for sites using Cloudflare for proxying,
+            # rather than just their nameservers.
+            if [[ "$resp" == *"error code: 1003"* ]]; then
+                cf=1
+                echo "    ! Using cloudflare proxy, skipping..."
+                break
+            fi
+        done
+
+        if [ $cf -eq 0 ]; then
+            echo "$line" >> out.json
+        fi
+    else
+        echo "$line" >> out.json
+    fi
+done <$file
+
+# Remove any trailing commas from new instance lists
+sed -i -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json
+
+cat out.json | jq --indent 2 . > services.json
+rm -f out.json