Add automated Issue generation for missing translations (#119)

.github/workflows/test-and-build.yml

@@ -8,6 +8,7 @@ on:
 
 permissions:
   contents: write
+  issues: write
 
 jobs:
   test-and-build:
@@ -70,6 +71,15 @@ jobs:
       - name: Generate (and validate) translations
         run: make translate
 
+      - name: Update Translations Issue
+        uses: JasonEtco/create-an-issue@v2
+        if: github.ref == 'refs/heads/master'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          filename: translate-issue.md
+          update_existing: true
+
       - name: Build release
         run: make publish
         env:

.gitignore

@@ -11,3 +11,4 @@ frontend/webfonts
 frontend/*.woff2
 node_modules
 ots
+translate-issue.md

Makefile

@@ -26,7 +26,7 @@ publish: download_libs generate-inner generate-apidocs
 	bash ./ci/build.sh
 
 translate:
-	cd ci/translate && go run .
+	cd ci/translate && go run . --write-issue-file
 
 # -- Download / refresh external libraries --
 

ci/translate/issue.tpl.md

@@ -0,0 +1,22 @@
+---
+title: Missing Translations
+---
+> As a developer I want my application to have correct translations in all available languages and not to have them to experience mixed translations in their native language and English.
+
+**In order to achieve this we need to fix the following missing translations.**
+
+To do so please either **create a pull-request** updating the `i18n.yaml` in the root of the repository and add the missing translations to the corresponding language or **just leave a comment** below and ping @Luzifer in your comment. He then will integrate the new translation strings and mark your comment hidden after this issue has been automatically updated (kind of a to-do list for translations until we have something better in place).
+
+{{ range $lang, $translation := .Translations -}}
+{{ if MissingTranslations $lang -}}
+### Language: `{{ $lang }}`
+
+Please add the following translations:
+{{ range MissingTranslations $lang }}
+- `{{ . }}`
+  > {{ English . }}
+{{ end }}
+_{{ Ping $lang }}_
+
+{{ end -}}
+{{ end -}}

ci/translate/issuegen.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	_ "embed"
+	"os"
+	"sort"
+	"strings"
+	"text/template"
+
+	"github.com/pkg/errors"
+)
+
+//go:embed issue.tpl.md
+var issueTemplate string
+
+func generateIssue(tf translationFile) error {
+	fm := template.FuncMap{
+		"English":             tplEnglish(tf),
+		"MissingTranslations": tplMissingTranslations(tf),
+		"Ping":                tplPing(tf),
+	}
+
+	tpl, err := template.New("issue").Funcs(fm).Parse(issueTemplate)
+	if err != nil {
+		return errors.Wrap(err, "parsing issue template")
+	}
+
+	f, err := os.Create(cfg.IssueFile)
+	if err != nil {
+		return errors.Wrap(err, "opening issue file")
+	}
+	defer f.Close() //nolint:errcheck // Short-lived fd-leak
+
+	return errors.Wrap(tpl.Execute(f, tf), "executing issue template")
+}
+
+func tplEnglish(tf translationFile) func(string) any {
+	return func(key string) any {
+		return tf.Reference.Translations[key]
+	}
+}
+
+func tplMissingTranslations(tf translationFile) func(string) []string {
+	return func(lang string) []string {
+		missing, _, _ := tf.Translations[lang].Translations.GetErrorKeys(tf.Reference.Translations)
+		sort.Strings(missing)
+		return missing
+	}
+}
+
+func tplPing(tf translationFile) func(string) string {
+	return func(lang string) string {
+		if len(tf.Translations[lang].Translators) == 0 {
+			return "No translators to ping for this language."
+		}
+
+		var pings []string
+		for _, t := range tf.Translations[lang].Translators {
+			pings = append(pings, "@"+t)
+		}
+
+		return strings.Join([]string{"Ping", strings.Join(pings, ", ")}, " ")
+	}
+}

ci/translate/main.go

@@ -20,31 +20,19 @@ import (
 
 const deeplRequestTimeout = 10 * time.Second
 
-type (
-	translation     map[string]any
-	translationFile struct {
-		Reference    translationMapping             `yaml:"reference"`
-		Translations map[string]*translationMapping `yaml:"translations"`
-	}
-	translationMapping struct {
-		DeeplLanguage      string      `yaml:"deeplLanguage,omitempty"`
-		LanguageKey        string      `yaml:"languageKey,omitempty"`
-		Translations       translation `yaml:"translations"`
-		FormalTranslations translation `yaml:"formalTranslations,omitempty"`
-	}
-)
-
 var (
 	cfg = struct {
 		AutoTranslate    bool   `flag:"auto-translate" default:"false" description:"Enable auto-translation through DeepL"`
 		DeeplAPIEndpoint string `flag:"deepl-api-endpoint" default:"https://api-free.deepl.com/v2/translate" description:"DeepL API endpoint to request translations from"`
 		DeeplAPIKey      string `flag:"deepl-api-key" default:"" description:"API key for the DeepL API"`
+		IssueFile        string `flag:"issue-file" default:"../../translate-issue.md" description:"Where to create the translate issue"`
 		OutputFile       string `flag:"output-file,o" default:"../../src/langs/langs.js" description:"Where to put rendered translations"`
 		Template         string `flag:"template" default:"../../src/langs/langs.tpl.js" description:"Template to load for translation JS file"`
 		TranslationFile  string `flag:"translation-file,t" default:"../../i18n.yaml" description:"File to use for translations"`
 		LogLevel         string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"`
 		Verify           bool   `flag:"verify" default:"true" description:"Run verification against translation file"`
 		VersionAndExit   bool   `flag:"version" default:"false" description:"Prints current version and exits"`
+		WriteIssueFile   bool   `flag:"write-issue-file" default:"false" description:"Generates an issue body for missing translations"`
 	}{}
 
 	version = "dev"
@@ -113,6 +101,13 @@ func main() {
 	if err = renderJSFile(tf); err != nil {
 		logrus.WithError(err).Fatal("rendering JS output")
 	}
+
+	if cfg.WriteIssueFile {
+		logrus.Info("writing issue template...")
+		if err = generateIssue(tf); err != nil {
+			logrus.WithError(err).Fatal("generating issue template")
+		}
+	}
 }
 
 func autoTranslate(tf *translationFile) error {
@@ -212,7 +207,11 @@ func fetchTranslation(srcLang, destLang, text string) (string, error) {
 	if err != nil {
 		return "", errors.Wrap(err, "executing request")
 	}
-	defer resp.Body.Close()
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			logrus.WithError(err).Error("closing response body (leaked fd)")
+		}
+	}()
 
 	var payload struct {
 		Translations []struct {
@@ -237,7 +236,7 @@ func loadTranslationFile() (translationFile, error) {
 	if err != nil {
 		return tf, errors.Wrap(err, "opening translation file")
 	}
-	defer f.Close()
+	defer f.Close() //nolint:errcheck // Short-lived fd-leak
 
 	decoder := yaml.NewDecoder(f)
 	decoder.KnownFields(true)
@@ -262,11 +261,11 @@ func renderJSFile(tf translationFile) error {
 	}
 
 	if err = tpl.Execute(f, tf); err != nil {
-		f.Close()
+		f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak
 		return errors.Wrap(err, "rendering js template")
 	}
 
-	f.Close()
+	f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak
 	return errors.Wrap(os.Rename(cfg.OutputFile+".tmp", cfg.OutputFile), "moving file in place")
 }
 
@@ -277,18 +276,13 @@ func saveTranslationFile(tf translationFile) error {
 	}
 
 	encoder := yaml.NewEncoder(f)
-	encoder.SetIndent(2)
+	encoder.SetIndent(2) //nolint:gomnd
 
 	if err = encoder.Encode(tf); err != nil {
-		f.Close()
+		f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak
 		return errors.Wrap(err, "encoding translation file")
 	}
 
-	f.Close()
+	f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak
 	return errors.Wrap(os.Rename(cfg.TranslationFile+".tmp", cfg.TranslationFile), "moving file in place")
 }
-
-func (t translation) ToJSON() (string, error) {
-	j, err := json.Marshal(t)
-	return strings.ReplaceAll(string(j), "'", "\\'"), errors.Wrap(err, "marshalling JSON")
-}

ci/translate/translation.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"encoding/json"
+	"reflect"
+	"strings"
+
+	"github.com/Luzifer/go_helpers/v2/str"
+	"github.com/pkg/errors"
+)
+
+type (
+	translation     map[string]any
+	translationFile struct {
+		Reference    translationMapping             `yaml:"reference"`
+		Translations map[string]*translationMapping `yaml:"translations"`
+	}
+	translationMapping struct {
+		DeeplLanguage      string      `yaml:"deeplLanguage,omitempty"`
+		LanguageKey        string      `yaml:"languageKey,omitempty"`
+		Translators        []string    `yaml:"translators"`
+		Translations       translation `yaml:"translations"`
+		FormalTranslations translation `yaml:"formalTranslations,omitempty"`
+	}
+)
+
+func (t translation) ToJSON() (string, error) {
+	j, err := json.Marshal(t)
+	return strings.ReplaceAll(string(j), "'", "\\'"), errors.Wrap(err, "marshalling JSON")
+}
+
+func (t translation) GetErrorKeys(ref translation) (missing, extra, wrongType []string) {
+	var (
+		keys     []string
+		keyType  = map[string]reflect.Type{}
+		seenKeys []string
+	)
+
+	for k, v := range ref {
+		keys = append(keys, k)
+		keyType[k] = reflect.TypeOf(v)
+	}
+
+	for k, v := range t {
+		if !str.StringInSlice(k, keys) {
+			// Contains extra key, is error
+			extra = append(extra, k)
+			continue // No further checks for that key
+		}
+
+		seenKeys = append(seenKeys, k)
+		if kt := reflect.TypeOf(v); keyType[k] != kt {
+			// Type mismatches (i.e. string vs []string)
+			wrongType = append(wrongType, k)
+			continue
+		}
+	}
+
+	for _, k := range keys {
+		if !str.StringInSlice(k, seenKeys) {
+			missing = append(missing, k)
+		}
+	}
+
+	return missing, extra, wrongType
+}

ci/translate/verify.go

@@ -1,41 +1,35 @@
 package main
 
 import (
-	"reflect"
 	"regexp"
+	"sort"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-
-	"github.com/Luzifer/go_helpers/v2/str"
 )
 
 var langKeyFormat = regexp.MustCompile(`^[a-z]{2}(-[A-Z]{2})?$`)
 
 func verify(tf translationFile) error {
-	var (
+	var err error
-		err     error
-		keys    []string
-		keyType = map[string]reflect.Type{}
-	)
-
-	for k, v := range tf.Reference.Translations {
-		keys = append(keys, k)
-		keyType[k] = reflect.TypeOf(v)
-	}
 
 	if !langKeyFormat.MatchString(tf.Reference.LanguageKey) {
 		return errors.New("reference contains invalid languageKey")
 	}
 
-	if len(keys) == 0 {
+	if len(tf.Reference.Translations) == 0 {
 		return errors.New("reference does not contain translations")
 	}
 
-	logrus.Infof("found %d translation keys in reference", len(keys))
+	logrus.Infof("found %d translation keys in reference", len(tf.Reference.Translations))
 
 	if tf.Reference.FormalTranslations != nil {
-		if verifyTranslationKeys(logrus.NewEntry(logrus.StandardLogger()), tf.Reference.FormalTranslations, keys, keyType, false); err != nil {
+		if verifyTranslationKeys(
+			logrus.NewEntry(logrus.StandardLogger()),
+			tf.Reference.FormalTranslations,
+			tf.Reference.Translations,
+			false,
+		); err != nil {
 			return errors.New("reference contains error in formalTranslations")
 		}
 	}
@@ -54,8 +48,18 @@ func verify(tf translationFile) error {
 			logger.Info("no deeplLanguage is set")
 		}
 
-		hadErrors = hadErrors || verifyTranslationKeys(logger, tm.Translations, keys, keyType, true)
+		hadErrors = hadErrors || verifyTranslationKeys(
-		hadErrors = hadErrors || verifyTranslationKeys(logger, tm.FormalTranslations, keys, keyType, false)
+			logger,
+			tm.Translations,
+			tf.Reference.Translations,
+			true,
+		)
+		hadErrors = hadErrors || verifyTranslationKeys(
+			logger,
+			tm.FormalTranslations,
+			tf.Reference.Translations,
+			false,
+		)
 	}
 
 	if hadErrors {
@@ -64,31 +68,27 @@ func verify(tf translationFile) error {
 	return nil
 }
 
-func verifyTranslationKeys(logger *logrus.Entry, t translation, keys []string, keyType map[string]reflect.Type, warnMissing bool) (hadErrors bool) {
+//revive:disable-next-line:flag-parameter
-	var seenKeys []string
+func verifyTranslationKeys(logger *logrus.Entry, t, ref translation, warnMissing bool) (hadErrors bool) {
+	missing, extra, wrongType := t.GetErrorKeys(ref)
 
-	for k, v := range t {
+	sort.Strings(extra)
-		keyLogger := logger.WithField("translation_key", k)
+	sort.Strings(missing)
-		if !str.StringInSlice(k, keys) {
+	sort.Strings(wrongType)
-			// Contains extra key, is error
-			hadErrors = true
-			keyLogger.Error("extra key found")
-			continue // No further checks for that key
-		}
 
-		seenKeys = append(seenKeys, k)
+	for _, k := range extra {
-		if kt := reflect.TypeOf(v); keyType[k] != kt {
+		logger.WithField("translation_key", k).Error("extra key found")
-			// Type mismatches (i.e. string vs []string)
-			hadErrors = true
-			keyLogger.Errorf("key has invalid type %s != %s", kt, keyType[k])
-		}
 	}
 
-	for _, k := range keys {
+	for _, k := range wrongType {
-		if warnMissing && !str.StringInSlice(k, seenKeys) {
+		logger.WithField("translation_key", k).Error("key has invalid type")
+	}
+
+	if warnMissing {
+		for _, k := range missing {
 			logger.WithField("translation_key", k).Warn("missing translation")
 		}
 	}
 
-	return hadErrors
+	return len(extra)+len(wrongType) > 0
 }

i18n.yaml

@@ -1,6 +1,8 @@
 reference:
   deeplLanguage: en
   languageKey: en
+  translators:
+    - Luzifer
   translations:
     alert-secret-not-found: This is not the secret you are looking for… - If you expected the secret to be here it might be compromised as someone else might have opened the link already.
     alert-something-went-wrong: Something went wrong. I'm very sorry about this…
@@ -43,6 +45,7 @@ reference:
     title-secret-created: Secret created!
 translations:
   ca:
+    translators: []
     translations:
       alert-secret-not-found: Aquest no és el secret que busques… - Si esperaves que el secret estiguera ací, és possible que s'haja vist compromés, ja que una altra persona podria haver obert l'enllaç en comptes de tu.
       alert-something-went-wrong: Alguna cosa ha eixit malament. Ens sap molt greu…
@@ -79,6 +82,8 @@ translations:
       title-secret-created: Secret creat!
   de:
     deeplLanguage: de
+    translators:
+      - Luzifer
     translations:
       alert-secret-not-found: Das ist nicht das Secret, was du suchst… - Falls du diesen Link noch nicht selbst geöffnet hast, könnte das Secret kompromittiert sein, da jemand anderes den Link geöffnet haben könnte.
       alert-something-went-wrong: Irgendwas ging schief. Entschuldigung…
@@ -138,6 +143,7 @@ translations:
       title-new-secret: Ein neues Secret erstellen
   es:
     deeplLanguage: es
+    translators: []
     translations:
       alert-secret-not-found: Este no es el secreto que buscas… - Si esperabas que el secreto estuviera aquí, es posible que se haya visto comprometido, ya que otra persona podría haber abierto el enlace en tu lugar.
       alert-something-went-wrong: Algo ha salido mal. Lo sentimos mucho…
@@ -174,6 +180,7 @@ translations:
       title-secret-created: ¡Secreto creado!
   fr:
     deeplLanguage: fr
+    translators: []
     translations:
       alert-secret-not-found: Ce secret n'est pas celui que vous cherchez… - Si vous comptiez trouvez ce secret ici, il a pu être compromis car quelqu'un a probablement déjà ouvert le lien.
       alert-something-went-wrong: Un problème est survenu. Nous en sommes désolés…
@@ -207,6 +214,7 @@ translations:
       title-secret-created: Secret créé!
   lv:
     deeplLanguage: lv
+    translators: []
     translations:
       alert-secret-not-found: <strong>Ziņa nav atrasta!</strong>&hellip; - Ja ievadītā saite ir pareiza, tad ir beidzies ziņas glabāšanas laiks, vai arī tā jau vienreiz ir atvērta.
       alert-something-went-wrong: Neparedzēta sistēmas kļūda. Atvainojiet par sagādātajām neērtībām&hellip;
@@ -240,6 +248,7 @@ translations:
       title-secret-created: Ziņa nošifrēta!
   nl:
     deeplLanguage: nl
+    translators: []
     translations:
       alert-secret-not-found: De gegevens die je zocht bestaan niet (meer)&hellip; - Als je hier informatie verwachtte dan is de link mogelijk al door iemand anders bekeken!
       alert-something-went-wrong: Er ging iets verkeerd, sorry&hellip;
@@ -273,6 +282,7 @@ translations:
       title-secret-created: Vertrouwelijke info opgeslaan!
   pl:
     deeplLanguage: pl
+    translators: []
     translations:
       alert-secret-not-found: TO nie jest sekret, którego szukasz&hellip; - Jeśli spodziewałeś się tu sekretu, to może być on zagrożony, ponieważ ktoś inny mógł już otworzyć ten link.
       alert-something-went-wrong: Coś poszło nie tak. Bardzo mi przykro&hellip;
@@ -309,6 +319,7 @@ translations:
       title-secret-created: Sekret utworzony!
   pt-BR:
     deeplLanguage: pt-BR
+    translators: []
     translations:
       alert-secret-not-found: Esta não é o segredo que você está procurando… - Se você esperava que o segredo estaria aqui, ele pode ter sido comprometido por alguém que já acessou o link.
       alert-something-went-wrong: Desculpe, algo deu errado…
@@ -342,6 +353,7 @@ translations:
       title-secret-created: Segredo criado!
   ru:
     deeplLanguage: ru
+    translators: []
     translations:
       alert-secret-not-found: Секрет недоступен&hellip; - Помните, он может быть скомпрометирован. Возможно кто-то другой уже открыл вашу ссылку.
       alert-something-went-wrong: Что-то пошло не так. Приносим свои извинения&hellip;
@@ -375,6 +387,7 @@ translations:
       title-secret-created: Секрет создан!
   sv:
     deeplLanguage: sv
+    translators: []
     translations:
       alert-secret-not-found: Hemlighet hittades inte&hellip; - Om du förväntade dig att hemligheten skulle finnas här kan den vara röjd då någon annan kan ha öppnat denna länk tidigare.
       alert-something-went-wrong: Något gick fel. Jag ber om ursäkt för detta!&hellip;
@@ -408,6 +421,7 @@ translations:
       title-secret-created: Hemlighet skapad!
   tr:
     deeplLanguage: tr
+    translators: []
     translations:
       alert-secret-not-found: Aradığınız sır bu değil… - Sırrın burada olmasını bekliyorsanız, bu link başkası tarafından açılmış ve sırrınız tehlikede olabilir.
       alert-something-went-wrong: Bir şeyler ters gitti. Bunun için çok üzgünüm…
@@ -441,6 +455,7 @@ translations:
       title-secret-created: Sır oluşturuldu!
   uk:
     deeplLanguage: uk
+    translators: []
     translations:
       alert-secret-not-found: Це не секрет, який ви шукаєте&hellip; - Якщо ви очікували, що секрет буде тут, він міг бути скомпрометований, оскільки хтось інший міг уже відкрити посилання.
       alert-something-went-wrong: Щось пішло не так. Ми дуже шкодуємо про це&hellip;
@@ -476,6 +491,7 @@ translations:
       title-secret-create-disabled: Створення секрету вимкнено…
       title-secret-created: Секрет створений!
   zh:
+    translators: []
     translations:
       alert-secret-not-found: 该秘密不存在 - 如果这与您的预期不符，那么该链接可能已经泄露，且您的秘密已经被其他人查看了。
       alert-something-went-wrong: 运行异常，对此我深感抱歉…
@@ -511,6 +527,7 @@ translations:
       title-secret-create-disabled: 创建秘密被禁止…
       title-secret-created: 秘密已创建！
   zh-TW:
+    translators: []
     translations:
       alert-secret-not-found: 這不是您正在尋找的機密&hellip; - 如果您期望機密會出現在這裡，它可能已經被泄漏了，因為可能有其他人已經打開了此連結。
       alert-something-went-wrong: 看樣子出了一些問題，對此我非常抱歉&hellip;