From d0b516cf8d50fe798a55bdb018f600eac3fc3b46 Mon Sep 17 00:00:00 2001
From: Stanley <113554994+l3pr-org@users.noreply.github.com>
Date: Fri, 19 Sep 2025 17:29:47 +0000
Subject: [PATCH] Update main.go

Add helper function to create robots.txt unless explicitly disabled with DisableSearchIndex: false and call the function on server init.

Signed-off-by: Stanley <113554994+l3pr-org@users.noreply.github.com>
---
 main.go | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/main.go b/main.go
index e5f07fa..c66f7f8 100644
--- a/main.go
+++ b/main.go
@@ -134,6 +134,8 @@ func main() {
 
 	r.HandleFunc("/", handleIndex).
 		Methods(http.MethodGet)
+	r.HandleFunc("/robots.txt", handleRobotsTXT).
+		Methods(http.MethodGet)
 	r.PathPrefix("/").HandlerFunc(assetDelivery).
 		Methods(http.MethodGet)
 
@@ -241,6 +243,18 @@ func handleIndex(w http.ResponseWriter, _ *http.Request) {
 	}
 }
 
+func handleRobotsTXT(w http.ResponseWriter, _ *http.Request) {
+	// If explicitly set to false, do not create robots.txt.
+	if cust.DisableSearchIndex != nil && !*cust.DisableSearchIndex {
+		http.NotFound(w, nil)
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	_, _ = w.Write([]byte("User-agent: *\nDisallow: /\n"))
+}
+
 func handleRemoveAcceptEncoding(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		r.Header.Del("Accept-Encoding")