diff --git a/go-xgettext/main.go b/go-xgettext/main.go index 5f89abe..64d7342 100644 --- a/go-xgettext/main.go +++ b/go-xgettext/main.go @@ -34,7 +34,6 @@ import ( "go/parser" "go/token" "io" - "io/ioutil" "log" "os" "sort" @@ -50,11 +49,11 @@ var ( noLocation = flag.Bool("no-location", false, "Do not write '#: filename:line' lines.") msgIDBugsAddress = flag.String("msgid-bugs-address", "EMAIL", "set report address for msgid bugs.") packageName = flag.String("package-name", "", "Set package name in output.") + deterministic = flag.Bool("deterministic", false, "Generate deterministic output (e.g. no timestamps)") - keyword = flag.String("keyword", "gettext.Gettext", "Look for WORD as the keyword for singular strings.") - keywordPlural = flag.String("keyword-plural", "gettext.NGettext", "Look for WORD as the keyword for plural strings.") - keywordContextual = flag.String("keyword-contextual", "gettext.CGettext", "Look for WORD as the keyword for contextual strings.") - keywordPluralContextual = flag.String("keyword-plural-contextual", "gettext.CNGettext", "Look for WORD as the keyword for plural contextual strings.") + keyword = flag.String("keyword", "gettext.Gettext", "Look for WORD as the keyword for singular strings.") + keywordPlural = flag.String("keyword-plural", "gettext.NGettext", "Look for WORD as the keyword for plural strings.") + keywordContextual = flag.String("keyword-contextual", "gettext.CGettext", "Look for WORD as the keyword for contextual strings.") skipArgs = flag.Int("skip-args", 0, "Number of arguments to skip in gettext function call before considering a text message argument.") @@ -69,25 +68,41 @@ const ( ) type keywordDef struct { - Type string `json:"type"` - Name string `json:"name"` - SkipArgs int `json:"skipArgs"` + Type string `json:"type"` + Name string `json:"name"` + SkipArgs int `json:"skipArgs"` + FormatHint string `json:"formatHint"` + FormatHintArgs int `json:"formatHintArgs"` + ForceContext string `json:"forceContext"` } type keywords map[string]*keywordDef -type allKeywordsConfig []*keywordDef +type msgKey struct { + msgctxt string + msgtext string +} -type msgID struct { +type msgData struct { msgidPlural string - msgctxt string comment string fname string line int formatHint string } -var msgIDs map[string][]msgID +type msgKeyList []msgKey + +func (l msgKeyList) Len() int { return len(l) } +func (l msgKeyList) Less(i, j int) bool { + if l[i].msgctxt != l[j].msgctxt { + return l[i].msgctxt < l[j].msgctxt + } + return l[i].msgtext < l[j].msgtext +} +func (l msgKeyList) Swap(i, j int) { l[i], l[j] = l[j], l[i] } + +var msgIDs map[msgKey][]msgData func formatComment(com string) string { out := "" @@ -132,28 +147,28 @@ func findCommentsForTranslation(fset *token.FileSet, f *ast.File, posCall token. } func constructValue(val interface{}) (string, error) { - switch val.(type) { + switch val := val.(type) { case *ast.BasicLit: - return val.(*ast.BasicLit).Value, nil + return val.Value, nil // this happens for constructs like: // gettext.Gettext("foo" + "bar") case *ast.BinaryExpr: // we only support string concat - if val.(*ast.BinaryExpr).Op != token.ADD { + if val.Op != token.ADD { return "", nil } - left, err := constructValue(val.(*ast.BinaryExpr).X) + left, err := constructValue(val.X) if err != nil { return "", err } // strip right " (or `) left = left[0 : len(left)-1] - right, err := constructValue(val.(*ast.BinaryExpr).Y) + right, err := constructValue(val.Y) if err != nil { return "", err } // strip left " (or `) - right = right[1:len(right)] + right = right[1:] return left + right, nil default: return "", fmt.Errorf("unknown type: %v", val) @@ -179,7 +194,7 @@ func parseFunExpr(path string, expr ast.Expr) string { func inspectNodeForTranslations(k keywords, fset *token.FileSet, f *ast.File, n ast.Node) bool { switch x := n.(type) { case *ast.CallExpr: - var i18nStr, i18nStrPlural, i18nCtxt string + var i18nStr, i18nStrPlural, i18nCtxt, formatHint string var err error name := parseFunExpr("", x.Fun) if name == "" { @@ -187,31 +202,65 @@ func inspectNodeForTranslations(k keywords, fset *token.FileSet, f *ast.File, n } if keyword, ok := k[name]; ok { idx := keyword.SkipArgs - switch keyword.Type { - case kTypeSingular: - i18nStr, err = constructValue(x.Args[idx]) - case kTypePlural: - i18nStr, err = constructValue(x.Args[idx]) - if err != nil { - break - } - i18nStrPlural, err = constructValue(x.Args[idx+1]) - case kTypeContextual: - i18nCtxt, err = constructValue(x.Args[idx]) - if err != nil { + formatHint = keyword.FormatHint + for i := 0; i < keyword.FormatHintArgs; i++ { + if idx >= len(x.Args) { + err = fmt.Errorf("not enough arguments") break } - i18nStr, err = constructValue(x.Args[idx+1]) - case kTypePluralContextual: - i18nCtxt, err = constructValue(x.Args[idx]) + var argVal string + argVal, err = constructValue(x.Args[idx]) if err != nil { break } - i18nStr, err = constructValue(x.Args[idx+1]) - if err != nil { - break + // strip leading and trailing " (or `) + argVal = argVal[1 : len(argVal)-1] + formatHint = fmt.Sprintf("%s,%s", formatHint, argVal) + idx++ + } + if idx >= len(x.Args) { + err = fmt.Errorf("not enough arguments") + } else { + switch keyword.Type { + case kTypeSingular: + i18nCtxt = fmt.Sprintf("%q", keyword.ForceContext) + i18nStr, err = constructValue(x.Args[idx]) + case kTypePlural: + i18nCtxt = fmt.Sprintf("%q", keyword.ForceContext) + if idx+1 >= len(x.Args) { + err = fmt.Errorf("not enough arguments") + break + } + i18nStr, err = constructValue(x.Args[idx]) + if err != nil { + break + } + i18nStrPlural, err = constructValue(x.Args[idx+1]) + case kTypeContextual: + if idx+1 >= len(x.Args) { + err = fmt.Errorf("not enough arguments") + break + } + i18nCtxt, err = constructValue(x.Args[idx]) + if err != nil { + break + } + i18nStr, err = constructValue(x.Args[idx+1]) + case kTypePluralContextual: + if idx+2 >= len(x.Args) { + err = fmt.Errorf("not enough arguments") + break + } + i18nCtxt, err = constructValue(x.Args[idx]) + if err != nil { + break + } + i18nStr, err = constructValue(x.Args[idx+1]) + if err != nil { + break + } + i18nStrPlural, err = constructValue(x.Args[idx+2]) } - i18nStrPlural, err = constructValue(x.Args[idx+2]) } } if err != nil { @@ -223,19 +272,15 @@ func inspectNodeForTranslations(k keywords, fset *token.FileSet, f *ast.File, n break } - // FIXME: too simplistic(?), no %% is considered - formatHint := "" - if strings.Contains(i18nStr, "%") || strings.Contains(i18nStrPlural, "%") { - // well, not quite correct but close enough - formatHint = "c-format" - } + i18nCtxt = formatI18nStr(i18nCtxt) + i18nStr = formatI18nStr(i18nStr) + i18nStrPlural = formatI18nStr(i18nStrPlural) - msgidStr := formatI18nStr(i18nStr) posCall := fset.Position(n.Pos()) - msgIDs[msgidStr] = append(msgIDs[msgidStr], msgID{ + k := msgKey{i18nCtxt, i18nStr} + msgIDs[k] = append(msgIDs[k], msgData{ formatHint: formatHint, - msgidPlural: formatI18nStr(i18nStrPlural), - msgctxt: formatI18nStr(i18nCtxt), + msgidPlural: i18nStrPlural, fname: posCall.Filename, line: posCall.Line, comment: findCommentsForTranslation(fset, f, posCall), @@ -263,7 +308,7 @@ func formatI18nStr(s string) string { func processFiles(args []string) error { // go over the input files - msgIDs = make(map[string][]msgID) + msgIDs = make(map[msgKey][]msgData) fset := token.NewFileSet() for _, fname := range args { @@ -278,7 +323,7 @@ func processFiles(args []string) error { func parseKeywords() (keywords, error) { k := make(keywords) if *keywordCfg != "" { - data, err := ioutil.ReadFile(*keywordCfg) + data, err := os.ReadFile(*keywordCfg) if err != nil { return nil, err } @@ -310,7 +355,7 @@ func parseKeywords() (keywords, error) { } func processSingleGoSource(fset *token.FileSet, fname string) error { - fnameContent, err := ioutil.ReadFile(fname) + fnameContent, err := os.ReadFile(fname) if err != nil { panic(err) } @@ -349,42 +394,38 @@ msgid "" msgstr "Project-Id-Version: %s\n" "Report-Msgid-Bugs-To: %s\n" "POT-Creation-Date: %s\n" - "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" - "Last-Translator: FULL NAME \n" - "Language-Team: LANGUAGE \n" - "Language: \n" "MIME-Version: 1.0\n" - "Content-Type: text/plain; charset=CHARSET\n" + "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" `, *packageName, *msgIDBugsAddress, formatTime()) fmt.Fprintf(out, "%s", header) // yes, this is the way to do it in go - sortedKeys := []string{} + var sortedKeys msgKeyList for k := range msgIDs { sortedKeys = append(sortedKeys, k) } if *sortOutput { - sort.Strings(sortedKeys) + sort.Sort(sortedKeys) } // FIXME: use template here? - for _, k := range sortedKeys { - msgidList := msgIDs[k] - for _, msgid := range msgidList { + for _, key := range sortedKeys { + msgList := msgIDs[key] + for _, msgid := range msgList { if *addComments || *addCommentsTag != "" { fmt.Fprintf(out, "%s", msgid.comment) } } if !*noLocation { fmt.Fprintf(out, "#:") - for _, msgid := range msgidList { + for _, msgid := range msgList { fmt.Fprintf(out, " %s:%d", msgid.fname, msgid.line) } fmt.Fprintf(out, "\n") } - msgid := msgidList[0] + msgid := msgList[0] if msgid.formatHint != "" { fmt.Fprintf(out, "#, %s\n", msgid.formatHint) } @@ -395,10 +436,10 @@ msgstr "Project-Id-Version: %s\n" // cleanup too aggressive splitting (empty "" lines) return strings.TrimSuffix(out, "\"\n \"") } - if msgid.msgctxt != "" { - fmt.Fprintf(out, "msgctxt \"%v\"\n", formatOutput(msgid.msgctxt)) + if key.msgctxt != "" { + fmt.Fprintf(out, "msgctxt \"%v\"\n", formatOutput(key.msgctxt)) } - fmt.Fprintf(out, "msgid \"%v\"\n", formatOutput(k)) + fmt.Fprintf(out, "msgid \"%v\"\n", formatOutput(key.msgtext)) if msgid.msgidPlural != "" { fmt.Fprintf(out, "msgid_plural \"%v\"\n", formatOutput(msgid.msgidPlural)) fmt.Fprintf(out, "msgstr[0] \"\"\n") @@ -408,7 +449,6 @@ msgstr "Project-Id-Version: %s\n" } fmt.Fprintf(out, "\n") } - } func main() { @@ -420,6 +460,12 @@ func main() { flag.PrintDefaults() os.Exit(0) } + if *deterministic { + *sortOutput = true + formatTime = func() string { + return "1970-01-01 00:00+0000" + } + } if err := processFiles(args); err != nil { log.Fatalf("processFiles failed with: %s", err) diff --git a/go-xgettext/main_test.go b/go-xgettext/main_test.go index eada037..8f632b7 100644 --- a/go-xgettext/main_test.go +++ b/go-xgettext/main_test.go @@ -29,7 +29,6 @@ package main import ( "bytes" "fmt" - "io/ioutil" "os" "path/filepath" "testing" @@ -48,7 +47,7 @@ var _ = Suite(&xgettextTestSuite{}) // test helper func makeGoSourceFile(c *C, content []byte) string { fname := filepath.Join(c.MkDir(), "foo.go") - err := ioutil.WriteFile(fname, []byte(content), 0644) + err := os.WriteFile(fname, []byte(content), 0644) c.Assert(err, IsNil) return fname @@ -98,8 +97,8 @@ func main() { err := processFiles([]string{fname}) c.Assert(err, IsNil) - c.Assert(msgIDs, DeepEquals, map[string][]msgID{ - "foo": []msgID{ + c.Assert(msgIDs, DeepEquals, map[msgKey][]msgData{ + {"", "foo"}: { { comment: "#. TRANSLATORS: foo comment\n", fname: fname, @@ -123,8 +122,8 @@ func main() { err := processFiles([]string{fname}) c.Assert(err, IsNil) - c.Assert(msgIDs, DeepEquals, map[string][]msgID{ - "foo": []msgID{ + c.Assert(msgIDs, DeepEquals, map[msgKey][]msgData{ + {"", "foo"}: { { comment: "#. TRANSLATORS: foo comment\n", fname: fname, @@ -149,18 +148,14 @@ msgid "" msgstr "Project-Id-Version: snappy\n" "Report-Msgid-Bugs-To: snappy-devel@lists.ubuntu.com\n" "POT-Creation-Date: 2015-06-30 14:48+0200\n" - "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" - "Last-Translator: FULL NAME \n" - "Language-Team: LANGUAGE \n" - "Language: \n" "MIME-Version: 1.0\n" - "Content-Type: text/plain; charset=CHARSET\n" + "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" ` func (s *xgettextTestSuite) TestWriteOutputSimple(c *C) { - msgIDs = map[string][]msgID{ - "foo": []msgID{ + msgIDs = map[msgKey][]msgData{ + {"", "foo"}: { { fname: "fname", line: 2, @@ -182,8 +177,8 @@ msgstr "" } func (s *xgettextTestSuite) TestWriteOutputMultiple(c *C) { - msgIDs = map[string][]msgID{ - "foo": []msgID{ + msgIDs = map[msgKey][]msgData{ + {"", "foo"}: { { fname: "fname", line: 2, @@ -211,8 +206,8 @@ msgstr "" } func (s *xgettextTestSuite) TestWriteOutputNoComment(c *C) { - msgIDs = map[string][]msgID{ - "foo": []msgID{ + msgIDs = map[msgKey][]msgData{ + {"", "foo"}: { { fname: "fname", line: 2, @@ -232,8 +227,8 @@ msgstr "" } func (s *xgettextTestSuite) TestWriteOutputNoLocation(c *C) { - msgIDs = map[string][]msgID{ - "foo": []msgID{ + msgIDs = map[msgKey][]msgData{ + {"", "foo"}: { { fname: "fname", line: 2, @@ -254,8 +249,8 @@ msgstr "" } func (s *xgettextTestSuite) TestWriteOutputFormatHint(c *C) { - msgIDs = map[string][]msgID{ - "foo": []msgID{ + msgIDs = map[msgKey][]msgData{ + {"", "foo"}: { { fname: "fname", line: 2, @@ -278,8 +273,8 @@ msgstr "" } func (s *xgettextTestSuite) TestWriteOutputPlural(c *C) { - msgIDs = map[string][]msgID{ - "foo": []msgID{ + msgIDs = map[msgKey][]msgData{ + {"", "foo"}: { { msgidPlural: "plural", fname: "fname", @@ -303,14 +298,14 @@ msgstr[1] "" } func (s *xgettextTestSuite) TestWriteOutputSorted(c *C) { - msgIDs = map[string][]msgID{ - "aaa": []msgID{ + msgIDs = map[msgKey][]msgData{ + {"", "aaa"}: { { fname: "fname", line: 2, }, }, - "zzz": []msgID{ + {"", "zzz"}: { { fname: "fname", line: 2, @@ -370,7 +365,7 @@ func main() { main() // verify its what we expect - got, err := ioutil.ReadFile(outName) + got, err := os.ReadFile(outName) c.Assert(err, IsNil) expected := fmt.Sprintf(`%s #: %[2]s:9 @@ -391,7 +386,6 @@ msgstr[0] "" msgstr[1] "" #: %[2]s:14 -#, c-format msgid "zz %%s" msgstr "" @@ -410,8 +404,8 @@ func main() { err := processFiles([]string{fname}) c.Assert(err, IsNil) - c.Assert(msgIDs, DeepEquals, map[string][]msgID{ - "foo\\nbar\\nbaz": []msgID{ + c.Assert(msgIDs, DeepEquals, map[msgKey][]msgData{ + {"", "foo\\nbar\\nbaz"}: { { comment: "#. TRANSLATORS: foo comment\n", fname: fname, @@ -445,8 +439,8 @@ msgstr "" } func (s *xgettextTestSuite) TestWriteOutputMultilines(c *C) { - msgIDs = map[string][]msgID{ - "foo\\nbar\\nbaz": []msgID{ + msgIDs = map[msgKey][]msgData{ + {"", "foo\\nbar\\nbaz"}: { { fname: "fname", line: 2, @@ -469,14 +463,14 @@ msgstr "" } func (s *xgettextTestSuite) TestWriteOutputTidy(c *C) { - msgIDs = map[string][]msgID{ - "foo\\nbar\\nbaz": []msgID{ + msgIDs = map[msgKey][]msgData{ + {"", "foo\\nbar\\nbaz"}: { { fname: "fname", line: 2, }, }, - "zzz\\n": []msgID{ + {"", "zzz\\n"}: { { fname: "fname", line: 4, diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..618a9e7 --- /dev/null +++ b/go.mod @@ -0,0 +1,17 @@ +module github.com/joyteam/gettext + +go 1.19 + +require ( + github.com/stretchr/testify v1.9.0 + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/rogpeppe/go-internal v1.12.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a1dc9ac --- /dev/null +++ b/go.sum @@ -0,0 +1,23 @@ +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=