scripts/manage_translations.py
#!/usr/bin/env python
#
# This Python file contains utility scripts to manage Django translations.
# It has to be run inside the django git root directory.
#
# The following commands are available:
#
# * update_catalogs: check for new strings in core and contrib catalogs, and
# output how much strings are new/changed.
#
# * lang_stats: output statistics for each catalog/language combination
#
# * fetch: fetch translations from transifex.com
#
# Each command support the --languages and --resources options to limit their
# operation to the specified language or resource. For example, to get stats
# for Spanish in contrib.admin, run:
#
# $ python scripts/manage_translations.py lang_stats --language=es --resources=admin
import os
from argparse import ArgumentParser
from subprocess import run
import django
from django.conf import settings
from django.core.management import call_command
HAVE_JS = ["admin"]
def _get_locale_dirs(resources, include_core=True):
"""
Return a tuple (contrib name, absolute path) for all locale directories,
optionally including the django core catalog.
If resources list is not None, filter directories matching resources content.
"""
contrib_dir = os.path.join(os.getcwd(), "django", "contrib")
dirs = []
# Collect all locale directories
for contrib_name in os.listdir(contrib_dir):
path = os.path.join(contrib_dir, contrib_name, "locale")
if os.path.isdir(path):
dirs.append((contrib_name, path))
if contrib_name in HAVE_JS:
dirs.append(("%s-js" % contrib_name, path))
if include_core:
dirs.insert(0, ("core", os.path.join(os.getcwd(), "django", "conf", "locale")))
# Filter by resources, if any
if resources is not None:
res_names = [d[0] for d in dirs]
dirs = [ld for ld in dirs if ld[0] in resources]
if len(resources) > len(dirs):
print(
"You have specified some unknown resources. "
"Available resource names are: %s" % (", ".join(res_names),)
)
exit(1)
return dirs
def _tx_resource_for_name(name):
"""Return the Transifex resource name"""
if name == "core":
return "django.core"
else:
return "django.contrib-%s" % name
def _check_diff(cat_name, base_path):
"""
Output the approximate number of changed/added strings in the en catalog.
"""
po_path = "%(path)s/en/LC_MESSAGES/django%(ext)s.po" % {
"path": base_path,
"ext": "js" if cat_name.endswith("-js") else "",
}
p = run(
"git diff -U0 %s | egrep '^[-+]msgid' | wc -l" % po_path,
capture_output=True,
shell=True,
)
num_changes = int(p.stdout.strip())
print("%d changed/added messages in '%s' catalog." % (num_changes, cat_name))
def update_catalogs(resources=None, languages=None):
"""
Update the en/LC_MESSAGES/django.po (main and contrib) files with
new/updated translatable strings.
"""
settings.configure()
django.setup()
if resources is not None:
print("`update_catalogs` will always process all resources.")
contrib_dirs = _get_locale_dirs(None, include_core=False)
os.chdir(os.path.join(os.getcwd(), "django"))
print("Updating en catalogs for Django and contrib apps...")
call_command("makemessages", locale=["en"])
print("Updating en JS catalogs for Django and contrib apps...")
call_command("makemessages", locale=["en"], domain="djangojs")
# Output changed stats
_check_diff("core", os.path.join(os.getcwd(), "conf", "locale"))
for name, dir_ in contrib_dirs:
_check_diff(name, dir_)
def lang_stats(resources=None, languages=None):
"""
Output language statistics of committed translation files for each
Django catalog.
If resources is provided, it should be a list of translation resource to
limit the output (e.g. ['core', 'gis']).
"""
locale_dirs = _get_locale_dirs(resources)
for name, dir_ in locale_dirs:
print("\nShowing translations stats for '%s':" % name)
langs = sorted(d for d in os.listdir(dir_) if not d.startswith("_"))
for lang in langs:
if languages and lang not in languages:
continue
# TODO: merge first with the latest en catalog
po_path = "{path}/{lang}/LC_MESSAGES/django{ext}.po".format(
path=dir_, lang=lang, ext="js" if name.endswith("-js") else ""
)
p = run(
["msgfmt", "-vc", "-o", "/dev/null", po_path],
capture_output=True,
env={"LANG": "C"},
encoding="utf-8",
)
if p.returncode == 0:
# msgfmt output stats on stderr
print("%s: %s" % (lang, p.stderr.strip()))
else:
print(
"Errors happened when checking %s translation for %s:\n%s"
% (lang, name, p.stderr)
)
def fetch(resources=None, languages=None):
"""
Fetch translations from Transifex, wrap long lines, generate mo files.
"""
locale_dirs = _get_locale_dirs(resources)
errors = []
for name, dir_ in locale_dirs:
# Transifex pull
if languages is None:
run(
[
"tx",
"pull",
"-r",
_tx_resource_for_name(name),
"-a",
"-f",
"--minimum-perc=5",
]
)
target_langs = sorted(
d for d in os.listdir(dir_) if not d.startswith("_") and d != "en"
)
else:
for lang in languages:
run(["tx", "pull", "-r", _tx_resource_for_name(name), "-f", "-l", lang])
target_langs = languages
# msgcat to wrap lines and msgfmt for compilation of .mo file
for lang in target_langs:
po_path = "%(path)s/%(lang)s/LC_MESSAGES/django%(ext)s.po" % {
"path": dir_,
"lang": lang,
"ext": "js" if name.endswith("-js") else "",
}
if not os.path.exists(po_path):
print(
"No %(lang)s translation for resource %(name)s"
% {"lang": lang, "name": name}
)
continue
run(["msgcat", "--no-location", "-o", po_path, po_path])
msgfmt = run(["msgfmt", "-c", "-o", "%s.mo" % po_path[:-3], po_path])
if msgfmt.returncode != 0:
errors.append((name, lang))
if errors:
print("\nWARNING: Errors have occurred in following cases:")
for resource, lang in errors:
print("\tResource %s for language %s" % (resource, lang))
exit(1)
if __name__ == "__main__":
RUNABLE_SCRIPTS = ("update_catalogs", "lang_stats", "fetch")
parser = ArgumentParser()
parser.add_argument("cmd", nargs=1, choices=RUNABLE_SCRIPTS)
parser.add_argument(
"-r",
"--resources",
action="append",
help="limit operation to the specified resources",
)
parser.add_argument(
"-l",
"--languages",
action="append",
help="limit operation to the specified languages",
)
options = parser.parse_args()
eval(options.cmd[0])(options.resources, options.languages)