Add issue generator, add translators

Signed-off-by: Knut Ahlers <knut@ahlers.me>

ci/translate/issue.tpl

@@ -0,0 +1,20 @@
+---
+title: Missing Translations
+---
+> As a developer I want my application to have correct translations in all available languages and not to have them to experience mixed translations in their native language and English.
+
+In order to achieve this we need to fix the following missing translations:
+
+{{ range $lang, $translation := .Translations -}}
+{{ if MissingTranslations $lang -}}
+### Language: `{{ $lang }}`
+
+Please add the following translations:
+{{ range MissingTranslations $lang }}
+- `{{ . }}`
+  > {{ English . }}
+{{ end }}
+_{{ Ping $lang }}_
+
+{{ end -}}
+{{ end -}}

ci/translate/issuegen.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	_ "embed"
+	"os"
+	"sort"
+	"strings"
+	"text/template"
+
+	"github.com/pkg/errors"
+)
+
+//go:embed issue.tpl
+var issueTemplate string
+
+func generateIssue(tf translationFile) error {
+	fm := template.FuncMap{
+		"English":             tplEnglish(tf),
+		"MissingTranslations": tplMissingTranslations(tf),
+		"Ping":                tplPing(tf),
+	}
+
+	tpl, err := template.New("issue").Funcs(fm).Parse(issueTemplate)
+	if err != nil {
+		return errors.Wrap(err, "parsing issue template")
+	}
+
+	f, err := os.Create(cfg.IssueFile)
+	if err != nil {
+		return errors.Wrap(err, "opening issue file")
+	}
+	defer f.Close() //nolint:errcheck // Short-lived fd-leak
+
+	return errors.Wrap(tpl.Execute(f, tf), "executing issue template")
+}
+
+func tplEnglish(tf translationFile) func(string) any {
+	return func(key string) any {
+		return tf.Reference.Translations[key]
+	}
+}
+
+func tplMissingTranslations(tf translationFile) func(string) []string {
+	return func(lang string) []string {
+		missing, _, _ := tf.Translations[lang].Translations.GetErrorKeys(tf.Reference.Translations)
+		sort.Strings(missing)
+		return missing
+	}
+}
+
+func tplPing(tf translationFile) func(string) string {
+	return func(lang string) string {
+		if len(tf.Translations[lang].Translators) == 0 {
+			return "No translators to ping for this language."
+		}
+
+		var pings []string
+		for _, t := range tf.Translations[lang].Translators {
+			pings = append(pings, "@"+t)
+		}
+
+		return strings.Join([]string{"Ping", strings.Join(pings, ", ")}, " ")
+	}
+}

ci/translate/main.go

@@ -20,31 +20,19 @@ import (
 
 const deeplRequestTimeout = 10 * time.Second
 
-type (
-	translation     map[string]any
-	translationFile struct {
-		Reference    translationMapping             `yaml:"reference"`
-		Translations map[string]*translationMapping `yaml:"translations"`
-	}
-	translationMapping struct {
-		DeeplLanguage      string      `yaml:"deeplLanguage,omitempty"`
-		LanguageKey        string      `yaml:"languageKey,omitempty"`
-		Translations       translation `yaml:"translations"`
-		FormalTranslations translation `yaml:"formalTranslations,omitempty"`
-	}
-)
-
 var (
 	cfg = struct {
 		AutoTranslate    bool   `flag:"auto-translate" default:"false" description:"Enable auto-translation through DeepL"`
 		DeeplAPIEndpoint string `flag:"deepl-api-endpoint" default:"https://api-free.deepl.com/v2/translate" description:"DeepL API endpoint to request translations from"`
 		DeeplAPIKey      string `flag:"deepl-api-key" default:"" description:"API key for the DeepL API"`
+		IssueFile        string `flag:"issue-file" default:"../../translate-issue.md" description:"Where to create the translate issue"`
 		OutputFile       string `flag:"output-file,o" default:"../../src/langs/langs.js" description:"Where to put rendered translations"`
 		Template         string `flag:"template" default:"../../src/langs/langs.tpl.js" description:"Template to load for translation JS file"`
 		TranslationFile  string `flag:"translation-file,t" default:"../../i18n.yaml" description:"File to use for translations"`
 		LogLevel         string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"`
 		Verify           bool   `flag:"verify" default:"true" description:"Run verification against translation file"`
 		VersionAndExit   bool   `flag:"version" default:"false" description:"Prints current version and exits"`
+		WriteIssueFile   bool   `flag:"write-issue-file" default:"false" description:"Generates an issue body for missing translations"`
 	}{}
 
 	version = "dev"
@@ -113,6 +101,13 @@ func main() {
 	if err = renderJSFile(tf); err != nil {
 		logrus.WithError(err).Fatal("rendering JS output")
 	}
+
+	if cfg.WriteIssueFile {
+		logrus.Info("writing issue template...")
+		if err = generateIssue(tf); err != nil {
+			logrus.WithError(err).Fatal("generating issue template")
+		}
+	}
 }
 
 func autoTranslate(tf *translationFile) error {
@@ -212,7 +207,11 @@ func fetchTranslation(srcLang, destLang, text string) (string, error) {
 	if err != nil {
 		return "", errors.Wrap(err, "executing request")
 	}
-	defer resp.Body.Close()
+	defer func() {
+		if err := resp.Body.Close(); err != nil {
+			logrus.WithError(err).Error("closing response body (leaked fd)")
+		}
+	}()
 
 	var payload struct {
 		Translations []struct {
@@ -237,7 +236,7 @@ func loadTranslationFile() (translationFile, error) {
 	if err != nil {
 		return tf, errors.Wrap(err, "opening translation file")
 	}
-	defer f.Close()
+	defer f.Close() //nolint:errcheck // Short-lived fd-leak
 
 	decoder := yaml.NewDecoder(f)
 	decoder.KnownFields(true)
@@ -262,11 +261,11 @@ func renderJSFile(tf translationFile) error {
 	}
 
 	if err = tpl.Execute(f, tf); err != nil {
-		f.Close()
+		f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak
 		return errors.Wrap(err, "rendering js template")
 	}
 
-	f.Close()
+	f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak
 	return errors.Wrap(os.Rename(cfg.OutputFile+".tmp", cfg.OutputFile), "moving file in place")
 }
 
@@ -277,18 +276,13 @@ func saveTranslationFile(tf translationFile) error {
 	}
 
 	encoder := yaml.NewEncoder(f)
-	encoder.SetIndent(2)
+	encoder.SetIndent(2) //nolint:gomnd
 
 	if err = encoder.Encode(tf); err != nil {
-		f.Close()
+		f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak
 		return errors.Wrap(err, "encoding translation file")
 	}
 
-	f.Close()
+	f.Close() //nolint:errcheck,gosec,revive // Short-lived fd-leak
 	return errors.Wrap(os.Rename(cfg.TranslationFile+".tmp", cfg.TranslationFile), "moving file in place")
 }
-
-func (t translation) ToJSON() (string, error) {
-	j, err := json.Marshal(t)
-	return strings.ReplaceAll(string(j), "'", "\\'"), errors.Wrap(err, "marshalling JSON")
-}

ci/translate/translation.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"encoding/json"
+	"reflect"
+	"strings"
+
+	"github.com/Luzifer/go_helpers/v2/str"
+	"github.com/pkg/errors"
+)
+
+type (
+	translation     map[string]any
+	translationFile struct {
+		Reference    translationMapping             `yaml:"reference"`
+		Translations map[string]*translationMapping `yaml:"translations"`
+	}
+	translationMapping struct {
+		DeeplLanguage      string      `yaml:"deeplLanguage,omitempty"`
+		LanguageKey        string      `yaml:"languageKey,omitempty"`
+		Translators        []string    `yaml:"translators"`
+		Translations       translation `yaml:"translations"`
+		FormalTranslations translation `yaml:"formalTranslations,omitempty"`
+	}
+)
+
+func (t translation) ToJSON() (string, error) {
+	j, err := json.Marshal(t)
+	return strings.ReplaceAll(string(j), "'", "\\'"), errors.Wrap(err, "marshalling JSON")
+}
+
+func (t translation) GetErrorKeys(ref translation) (missing, extra, wrongType []string) {
+	var (
+		keys     []string
+		keyType  = map[string]reflect.Type{}
+		seenKeys []string
+	)
+
+	for k, v := range ref {
+		keys = append(keys, k)
+		keyType[k] = reflect.TypeOf(v)
+	}
+
+	for k, v := range t {
+		if !str.StringInSlice(k, keys) {
+			// Contains extra key, is error
+			extra = append(extra, k)
+			continue // No further checks for that key
+		}
+
+		seenKeys = append(seenKeys, k)
+		if kt := reflect.TypeOf(v); keyType[k] != kt {
+			// Type mismatches (i.e. string vs []string)
+			wrongType = append(wrongType, k)
+			continue
+		}
+	}
+
+	for _, k := range keys {
+		if !str.StringInSlice(k, seenKeys) {
+			missing = append(missing, k)
+		}
+	}
+
+	return missing, extra, wrongType
+}

ci/translate/verify.go

@@ -1,41 +1,35 @@
 package main
 
 import (
-	"reflect"
 	"regexp"
+	"sort"
 
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
-
-	"github.com/Luzifer/go_helpers/v2/str"
 )
 
 var langKeyFormat = regexp.MustCompile(`^[a-z]{2}(-[A-Z]{2})?$`)
 
 func verify(tf translationFile) error {
-	var (
-		err     error
-		keys    []string
-		keyType = map[string]reflect.Type{}
-	)
-
-	for k, v := range tf.Reference.Translations {
-		keys = append(keys, k)
-		keyType[k] = reflect.TypeOf(v)
-	}
+	var err error
 
 	if !langKeyFormat.MatchString(tf.Reference.LanguageKey) {
 		return errors.New("reference contains invalid languageKey")
 	}
 
-	if len(keys) == 0 {
+	if len(tf.Reference.Translations) == 0 {
 		return errors.New("reference does not contain translations")
 	}
 
-	logrus.Infof("found %d translation keys in reference", len(keys))
+	logrus.Infof("found %d translation keys in reference", len(tf.Reference.Translations))
 
 	if tf.Reference.FormalTranslations != nil {
-		if verifyTranslationKeys(logrus.NewEntry(logrus.StandardLogger()), tf.Reference.FormalTranslations, keys, keyType, false); err != nil {
+		if verifyTranslationKeys(
+			logrus.NewEntry(logrus.StandardLogger()),
+			tf.Reference.FormalTranslations,
+			tf.Reference.Translations,
+			false,
+		); err != nil {
 			return errors.New("reference contains error in formalTranslations")
 		}
 	}
@@ -54,8 +48,18 @@ func verify(tf translationFile) error {
 			logger.Info("no deeplLanguage is set")
 		}
 
-		hadErrors = hadErrors || verifyTranslationKeys(logger, tm.Translations, keys, keyType, true)
-		hadErrors = hadErrors || verifyTranslationKeys(logger, tm.FormalTranslations, keys, keyType, false)
+		hadErrors = hadErrors || verifyTranslationKeys(
+			logger,
+			tm.Translations,
+			tf.Reference.Translations,
+			true,
+		)
+		hadErrors = hadErrors || verifyTranslationKeys(
+			logger,
+			tm.FormalTranslations,
+			tf.Reference.Translations,
+			false,
+		)
 	}
 
 	if hadErrors {
@@ -64,31 +68,27 @@ func verify(tf translationFile) error {
 	return nil
 }
 
-func verifyTranslationKeys(logger *logrus.Entry, t translation, keys []string, keyType map[string]reflect.Type, warnMissing bool) (hadErrors bool) {
-	var seenKeys []string
+//revive:disable-next-line:flag-parameter
+func verifyTranslationKeys(logger *logrus.Entry, t, ref translation, warnMissing bool) (hadErrors bool) {
+	missing, extra, wrongType := t.GetErrorKeys(ref)
 
-	for k, v := range t {
-		keyLogger := logger.WithField("translation_key", k)
-		if !str.StringInSlice(k, keys) {
-			// Contains extra key, is error
-			hadErrors = true
-			keyLogger.Error("extra key found")
-			continue // No further checks for that key
-		}
+	sort.Strings(extra)
+	sort.Strings(missing)
+	sort.Strings(wrongType)
 
-		seenKeys = append(seenKeys, k)
-		if kt := reflect.TypeOf(v); keyType[k] != kt {
-			// Type mismatches (i.e. string vs []string)
-			hadErrors = true
-			keyLogger.Errorf("key has invalid type %s != %s", kt, keyType[k])
-		}
+	for _, k := range extra {
+		logger.WithField("translation_key", k).Error("extra key found")
 	}
 
-	for _, k := range keys {
-		if warnMissing && !str.StringInSlice(k, seenKeys) {
+	for _, k := range wrongType {
+		logger.WithField("translation_key", k).Error("key has invalid type")
+	}
+
+	if warnMissing {
+		for _, k := range missing {
 			logger.WithField("translation_key", k).Warn("missing translation")
 		}
 	}
 
-	return hadErrors
+	return len(extra)+len(wrongType) > 0
 }