Update main.go

Add helper function to create robots.txt unless explicitly disabled with DisableSearchIndex: false and call the function on server init.

Signed-off-by: Stanley <113554994+l3pr-org@users.noreply.github.com>

main.go

@@ -134,6 +134,8 @@ func main() {
 
 	r.HandleFunc("/", handleIndex).
 		Methods(http.MethodGet)
+	r.HandleFunc("/robots.txt", handleRobotsTXT).
+		Methods(http.MethodGet)
 	r.PathPrefix("/").HandlerFunc(assetDelivery).
 		Methods(http.MethodGet)
 
@@ -241,6 +243,18 @@ func handleIndex(w http.ResponseWriter, _ *http.Request) {
 	}
 }
 
+func handleRobotsTXT(w http.ResponseWriter, _ *http.Request) {
+	// If explicitly set to false, do not create robots.txt.
+	if cust.DisableSearchIndex != nil && !*cust.DisableSearchIndex {
+		http.NotFound(w, nil)
+		return
+	}
+
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	_, _ = w.Write([]byte("User-agent: *\nDisallow: /\n"))
+}
+
 func handleRemoveAcceptEncoding(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		r.Header.Del("Accept-Encoding")