From aea54324a798ec9c1d9a34a5bd8dedcc8d4c7e4b Mon Sep 17 00:00:00 2001 From: Konstantin Akimov Date: Fri, 30 Oct 2026 02:42:25 +0700 Subject: [PATCH] feat: backport github-merge.py, optimize-pngs.py, update-translations.py to the latest versions --- contrib/devtools/README.md | 9 +- contrib/devtools/github-merge.py | 225 +++++++++++++----------- contrib/devtools/optimize-pngs.py | 45 +++-- contrib/devtools/update-translations.py | 191 ++++++++++++++------ doc/translation_process.md | 9 +- 5 files changed, 301 insertions(+), 178 deletions(-) diff --git a/contrib/devtools/README.md b/contrib/devtools/README.md index 3e5fd56ac6a3..358c238866a3 100644 --- a/contrib/devtools/README.md +++ b/contrib/devtools/README.md @@ -174,7 +174,9 @@ optimize-pngs.py ================ A script to optimize png files in the dash -repository (requires pngcrush). +repository (requires pngcrush and zopflipng). + + ./contrib/devtools/optimize-pngs.py src/qt/res/animation/ src/qt/res/icons/ src/qt/res/images/ share/pixmaps/ update-translations.py ====================== @@ -182,9 +184,12 @@ update-translations.py Run this script from the root of the repository to update all translations from transifex. It will do the following automatically: +- remove current translations - fetch all translations - post-process them into valid and committable format -- add missing translations to the build system (TODO) +- drop untranslated languages +- update git for added translations +- update build system See doc/translation-process.md for more information. diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index 70d91586ea02..cfef618bebae 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -17,18 +17,19 @@ import os from sys import stdin,stdout,stderr import argparse +import re import hashlib -import ssl import subprocess import sys import json import codecs +import unicodedata from urllib.request import Request, urlopen from urllib.error import HTTPError # External tools (can be overridden using environment) GIT = os.getenv('GIT','git') -BASH = os.getenv('BASH','bash') +SHELL = os.getenv('SHELL','bash') GH = os.getenv('GH','gh') # OS specific configuration for terminal attributes @@ -36,14 +37,24 @@ ATTR_PR = '' ATTR_NAME = '' ATTR_WARN = '' +ATTR_HL = '' COMMIT_FORMAT = '%H %s (%an)%d' if os.name == 'posix': # if posix, assume we can use basic terminal escapes ATTR_RESET = '\033[0m' ATTR_PR = '\033[1;36m' ATTR_NAME = '\033[0;36m' ATTR_WARN = '\033[1;31m' + ATTR_HL = '\033[95m' COMMIT_FORMAT = '%C(bold blue)%H%Creset %s %C(cyan)(%an)%Creset%C(green)%d%Creset' +def sanitize(s, newlines=False): + ''' + Strip control characters (optionally except for newlines) from a string. + This prevent text data from doing potentially confusing or harmful things + with ANSI formatting, linefeeds bells etc. + ''' + return ''.join(ch for ch in s if unicodedata.category(ch)[0] != "C" or (ch == '\n' and newlines)) + def git_config_get(option, default=None): ''' Get named configuration option from git repository. @@ -64,74 +75,38 @@ def gh_cli_token(): return None return token or None -_SSL_CONTEXT = None -_SSL_CONTEXT_INITIALIZED = False - -def _ssl_path_has_trust_material(path, is_dir=False): - if path is None: - return False - try: - if is_dir: - if not os.path.isdir(path): - return False - with os.scandir(path) as entries: - for entry in entries: - stem, dot, suffix = entry.name.partition('.') - is_hash_link = (dot and len(stem) == 8 and suffix.isdigit() and - all(c in '0123456789abcdefABCDEF' for c in stem)) - if is_hash_link and entry.is_file(follow_symlinks=True): - return True - return False - return os.path.isfile(path) and os.path.getsize(path) > 0 - except OSError: - return True - -def _ssl_context_with_certifi_fallback(): - ''' - Return an SSL context backed by certifi's CA bundle when the default - trust store is empty (a common cause of urllib HTTPS failures against - api.github.com on machines where Python's OpenSSL cannot find system - certificates). - - Returns None — leaving urllib's default behavior untouched — when: - * SSL_CERT_FILE or SSL_CERT_DIR is set (respect explicit user config), - * the default trust store already has CA certs or CA path material, or - * certifi is not installed. - ''' - if 'SSL_CERT_FILE' in os.environ or 'SSL_CERT_DIR' in os.environ: - return None - try: - default_ctx = ssl.create_default_context() - default_paths = ssl.get_default_verify_paths() - if default_ctx.cert_store_stats().get('x509_ca', 0) > 0: - return None - if _ssl_path_has_trust_material(default_paths.cafile): - return None - if _ssl_path_has_trust_material(default_paths.capath, is_dir=True): - return None - except (OSError, ssl.SSLError): - return None - try: - import certifi - except ImportError: - return None - try: - return ssl.create_default_context(cafile=certifi.where()) - except (OSError, ssl.SSLError): - return None - def get_response(req_url, ghtoken): - global _SSL_CONTEXT, _SSL_CONTEXT_INITIALIZED req = Request(req_url) if ghtoken is not None: req.add_header('Authorization', 'token ' + ghtoken) - if not _SSL_CONTEXT_INITIALIZED: - _SSL_CONTEXT = _ssl_context_with_certifi_fallback() - _SSL_CONTEXT_INITIALIZED = True - if _SSL_CONTEXT is not None: - return urlopen(req, context=_SSL_CONTEXT) return urlopen(req) +def sanitize_ghdata(rec): + ''' + Sanitize comment/review record coming from github API in-place. + This currently sanitizes the following: + - ['title'] PR title (optional, may not have newlines) + - ['body'] Comment body (required, may have newlines) + It also checks rec['user']['login'] (required) to be a valid github username. + + When anything more is used, update this function! + ''' + if 'title' in rec: # only for PRs + rec['title'] = sanitize(rec['title'], newlines=False) + if rec['body'] is None: + rec['body'] = '' + rec['body'] = sanitize(rec['body'], newlines=True) + + if rec['user'] is None: # User deleted account + rec['user'] = {'login': '[deleted]'} + else: + # "Github username may only contain alphanumeric characters or hyphens'. + # Sometimes bot have a "[bot]" suffix in the login, so we also match for that + # Use \Z instead of $ to not match final newline only end of string. + if not re.match(r'[a-zA-Z0-9-]+(\[bot\])?\Z', rec['user']['login'], re.DOTALL): + raise ValueError('Github username contains invalid characters: {}'.format(sanitize(rec['user']['login']))) + return rec + def retrieve_json(req_url, ghtoken, use_pagination=False): ''' Retrieve json from github. @@ -140,7 +115,7 @@ def retrieve_json(req_url, ghtoken, use_pagination=False): try: reader = codecs.getreader('utf-8') if not use_pagination: - return json.load(reader(get_response(req_url, ghtoken))) + return sanitize_ghdata(json.load(reader(get_response(req_url, ghtoken)))) obj = [] page_num = 1 @@ -156,7 +131,7 @@ def retrieve_json(req_url, ghtoken, use_pagination=False): page_num = int(link_next[0][link_next[0].find("page=")+5:link_next[0].find(">")]) continue break - return obj + return [sanitize_ghdata(d) for d in obj] except HTTPError as e: error_message = e.read() print('Warning: unable to retrieve pull information from github: %s' % e) @@ -200,6 +175,8 @@ def tree_sha512sum(commit='HEAD'): for line in subprocess.check_output([GIT, 'ls-tree', '--full-tree', '-r', commit]).splitlines(): name_sep = line.index(b'\t') metadata = line[:name_sep].split() # perms, 'blob', blobid + if metadata[1] == b'commit': + continue assert(metadata[1] == b'blob') name = line[name_sep+1:] files.append(name) @@ -242,50 +219,74 @@ def tree_sha512sum(commit='HEAD'): raise IOError('Non-zero return value executing git cat-file') return overall.hexdigest() -def get_acks_from_comments(head_commit, comments): +def get_acks_from_comments(head_commit, comments) -> dict: # Look for abbreviated commit id, because not everyone wants to type/paste # the whole thing and the chance of collisions within a PR is small enough head_abbrev = head_commit[0:6] - acks = [] + acks = {} for c in comments: - review = [l for l in c['body'].split('\r\n') if 'ACK' in l and head_abbrev in l] + review = [ + l for l in c["body"].splitlines() + if "ACK" in l + and head_abbrev in l + and not l.startswith("> ") # omit if quoted comment + and not l.startswith(" ") # omit if markdown indentation + ] if review and 'coderabbit' not in c['user']['login']: - acks.append((c['user']['login'], review[0])) + acks[c['user']['login']] = review[0] return acks -def make_acks_message(head_commit, acks): +def make_acks_message(head_commit, acks) -> str: if acks: ack_str ='\n\nACKs for top commit:\n'.format(head_commit) - for name, msg in acks: + for name, msg in acks.items(): ack_str += ' {}:\n'.format(name) ack_str += ' {}\n'.format(msg) else: ack_str ='\n\nTop commit has no ACKs.\n' return ack_str -def print_merge_details(pull_reference, title, branch, base_branch, head_branch, acks): +def print_merge_details(pull_reference, title, branch, base_branch, head_branch, acks, message): print('{}{}{} {} {}into {}{}'.format(ATTR_RESET+ATTR_PR,pull_reference,ATTR_RESET,title,ATTR_RESET+ATTR_PR,branch,ATTR_RESET)) - subprocess.check_call([GIT,'log','--graph','--topo-order','--pretty=format:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) + subprocess.check_call([GIT,'--no-pager','log','--graph','--topo-order','--pretty=tformat:'+COMMIT_FORMAT,base_branch+'..'+head_branch]) if acks is not None: if acks: print('{}ACKs:{}'.format(ATTR_PR, ATTR_RESET)) - for (name, message) in acks: - print('* {} {}({}){}'.format(message, ATTR_NAME, name, ATTR_RESET)) + for ack_name, ack_msg in acks.items(): + print('* {} {}({}){}'.format(ack_msg, ATTR_NAME, ack_name, ATTR_RESET)) else: print('{}Top commit has no ACKs!{}'.format(ATTR_WARN, ATTR_RESET)) + show_message = False + if message is not None and '@' in message: + print('{}Merge message contains an @!{}'.format(ATTR_WARN, ATTR_RESET)) + show_message = True + if message is not None and '/), + githubmerge.pushmirrors (default: none, comma-separated list of mirrors to push merges of the master development branch to, e.g. `git@gitlab.com:/.git,git@github.com:/.git`), user.signingkey (mandatory), user.ghtoken (default: token from `gh auth token`, if available). + githubmerge.merge-author-email (default: Email from git config), githubmerge.host (default: git@github.com), githubmerge.branch (no default), githubmerge.testcmd (default: none). ''' parser = argparse.ArgumentParser(description='Utility to merge, sign and push github pull requests', epilog=epilog) + parser.add_argument('--repo-from', '-r', metavar='repo_from', type=str, nargs='?', + help='The repo to fetch the pull request from. Useful for monotree repositories. Can only be specified when branch==master. (default: githubmerge.repository setting)') parser.add_argument('pull', metavar='PULL', type=int, nargs=1, help='Pull request ID to merge') parser.add_argument('branch', metavar='BRANCH', type=str, nargs='?', @@ -297,6 +298,8 @@ def main(): repo = git_config_get('githubmerge.repository') host = git_config_get('githubmerge.host','git@github.com') opt_branch = git_config_get('githubmerge.branch',None) + merge_author_email = git_config_get('githubmerge.merge-author-email',None) + merge_author_name = git_config_get('githubmerge.merge-author-name', 'merge-script') testcmd = git_config_get('githubmerge.testcmd') ghtoken = git_config_get('user.ghtoken') or gh_cli_token() signingkey = git_config_get('user.signingkey') @@ -309,21 +312,26 @@ def main(): print("git config --global user.signingkey ",file=stderr) sys.exit(1) + # Extract settings from command line + args = parse_arguments() + repo_from = args.repo_from or repo + is_other_fetch_repo = repo_from != repo + pull = str(args.pull[0]) + if host.startswith(('https:','http:')): host_repo = host+"/"+repo+".git" + host_repo_from = host+"/"+repo_from+".git" else: host_repo = host+":"+repo - - # Extract settings from command line - args = parse_arguments() - pull = str(args.pull[0]) + host_repo_from = host+":"+repo_from # Receive pull information from github - info = retrieve_pr_info(repo,pull,ghtoken) + info = retrieve_pr_info(repo_from,pull,ghtoken) if info is None: sys.exit(1) title = info['title'].strip() body = info['body'].strip() + pull_reference = repo_from + '#' + pull # Extract forker's repo SSH URL and branch name forker_repo_ssh_url = info['head']['repo']['ssh_url'] target_branch = info['head']['ref'] @@ -335,7 +343,14 @@ def main(): # - 'master' branch = args.branch or opt_branch or info['base']['ref'] or 'master' - pull_reference = repo + '#' + pull + if branch == 'master': + push_mirrors = git_config_get('githubmerge.pushmirrors', default='').split(',') + push_mirrors = [p for p in push_mirrors if p] # Filter empty string + else: + push_mirrors = [] + if is_other_fetch_repo: + print('ERROR: --repo-from is only supported for the master development branch') + sys.exit(1) # Initialize source branches head_branch = 'pull/'+pull+'/head' @@ -347,13 +362,13 @@ def main(): try: subprocess.check_call([GIT,'checkout','-q',branch]) except subprocess.CalledProcessError: - print("ERROR: Cannot check out branch %s." % (branch), file=stderr) + print(f"ERROR: Cannot check out branch {branch}.", file=stderr) sys.exit(3) try: - subprocess.check_call([GIT,'fetch','-q',host_repo,'+refs/pull/'+pull+'/*:refs/heads/pull/'+pull+'/*', + subprocess.check_call([GIT,'fetch','-q',host_repo_from,'+refs/pull/'+pull+'/*:refs/heads/pull/'+pull+'/*', '+refs/heads/'+branch+':refs/heads/'+base_branch]) except subprocess.CalledProcessError: - print("ERROR: Cannot find pull request {} or branch {} on {}.".format(pull_reference,branch,host_repo), file=stderr) + print(f"ERROR: Cannot find pull request {pull_reference} or branch {branch} on {host_repo_from}.", file=stderr) sys.exit(3) # Ask the user if they want to rebase the branch @@ -405,18 +420,17 @@ def main(): subprocess.check_call([GIT,'checkout','-q',base_branch]) subprocess.call([GIT,'branch','-q','-D',local_merge_branch], stderr=devnull) subprocess.check_call([GIT,'checkout','-q','-b',local_merge_branch]) - try: - subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+head_branch], stdout=devnull, stderr=stdout) - head_commit = subprocess.check_output([GIT,'log','-1','--pretty=format:%H',head_branch]).decode('utf-8') + subprocess.check_call([GIT,'--no-pager','log','-q','-1','refs/heads/'+head_branch], stdout=devnull, stderr=stdout) + head_commit = subprocess.check_output([GIT,'--no-pager','log','-1','--pretty=format:%H',head_branch]).decode('utf-8') assert len(head_commit) == 40 except subprocess.CalledProcessError: - print("ERROR: Cannot find head of pull request {} on {}.".format(pull_reference,host_repo), file=stderr) + print(f"ERROR: Cannot find head of pull request {pull_reference} on {host_repo_from}.", file=stderr) sys.exit(3) try: - subprocess.check_call([GIT,'log','-q','-1','refs/heads/'+merge_branch], stdout=devnull, stderr=stdout) + subprocess.check_call([GIT,'--no-pager','log','-q','-1','refs/heads/'+merge_branch], stdout=devnull, stderr=stdout) except subprocess.CalledProcessError: - print("ERROR: Cannot find merge of pull request {} on {}.".format(pull_reference,host_repo), file=stderr) + print(f"ERROR: Cannot find merge of pull request {pull_reference} on {host_repo_from}.", file=stderr) sys.exit(3) subprocess.check_call([GIT,'checkout','-q',base_branch]) subprocess.call([GIT,'branch','-q','-D',local_merge_branch], stderr=devnull) @@ -432,7 +446,7 @@ def main(): else: firstline = 'Merge {}'.format(pull_reference) message = firstline + '\n\n' - message += subprocess.check_output([GIT,'log','--no-merges','--topo-order','--pretty=format:%H %s (%an)',base_branch+'..'+head_branch]).decode('utf-8') + message += subprocess.check_output([GIT,'--no-pager','log','--no-merges','--topo-order','--pretty=format:%H %s (%an)',base_branch+'..'+head_branch]).decode('utf-8') message += '\n\nPull request description:\n\n ' + body.replace('\n', '\n ') + '\n' try: subprocess.check_call([GIT,'merge','-q','--commit','--no-edit','--no-ff','--no-gpg-sign','-m',message.encode('utf-8'),head_branch]) @@ -440,14 +454,14 @@ def main(): print("ERROR: Cannot be merged cleanly.",file=stderr) subprocess.check_call([GIT,'merge','--abort']) sys.exit(4) - logmsg = subprocess.check_output([GIT,'log','--pretty=format:%s','-n','1']).decode('utf-8') + logmsg = subprocess.check_output([GIT,'--no-pager','log','--pretty=format:%s','-n','1']).decode('utf-8') if logmsg.rstrip() != firstline.rstrip(): print("ERROR: Creating merge failed (already merged?).",file=stderr) sys.exit(4) symlink_files = get_symlink_files() for f in symlink_files: - print("ERROR: File %s was a symlink" % f) + print(f"ERROR: File '{f}' was a symlink") if len(symlink_files) > 0: sys.exit(4) @@ -458,13 +472,13 @@ def main(): print("ERROR: Unable to compute tree hash") sys.exit(4) - print_merge_details(pull_reference, title, branch, base_branch, head_branch, None) + print_merge_details(pull_reference, title, branch, base_branch, head_branch, acks=None, message=None) print() # Run test command if configured. if testcmd: if subprocess.call(testcmd,shell=True): - print("ERROR: Running %s failed." % testcmd,file=stderr) + print(f"ERROR: Running '{testcmd}' failed.",file=stderr) sys.exit(5) # Show the created merge. @@ -484,7 +498,7 @@ def main(): print("Type 'exit' when done.",file=stderr) if os.path.isfile('/etc/debian_version'): # Show pull number on Debian default prompt os.putenv('debian_chroot',pull) - subprocess.call([BASH,'-i']) + subprocess.call([SHELL,'-i']) second_sha512 = tree_sha512sum() if first_sha512 != second_sha512: @@ -493,7 +507,7 @@ def main(): # Retrieve PR comments and ACKs and add to commit message, store ACKs to print them with commit # description - comments = retrieve_pr_comments(repo,pull,ghtoken) + retrieve_pr_reviews(repo,pull,ghtoken) + comments = retrieve_pr_comments(repo_from,pull,ghtoken) + retrieve_pr_reviews(repo_from,pull,ghtoken) if comments is None: print("ERROR: Could not fetch PR comments and reviews",file=stderr) sys.exit(1) @@ -508,12 +522,17 @@ def main(): sys.exit(4) # Sign the merge commit. - print_merge_details(pull_reference, title, branch, base_branch, head_branch, acks) + print_merge_details(pull_reference, title, branch, base_branch, head_branch, acks, message) while True: reply = ask_prompt("Type 's' to sign off on the above merge, or 'x' to reject and exit.").lower() if reply == 's': try: - subprocess.check_call([GIT,'commit','-q','--gpg-sign','--amend','--no-edit']) + config = [] + if merge_author_name: + config += ['-c', f'user.name={merge_author_name}'] + if merge_author_email: + config += ['-c', f'user.email={merge_author_email}'] + subprocess.check_call([GIT] + config + ['commit','-q','--gpg-sign','--amend','--no-edit','--reset-author']) break except subprocess.CalledProcessError: print("Error while signing, asking again.",file=stderr) @@ -534,9 +553,11 @@ def main(): # Push the result. while True: - reply = ask_prompt("Type 'push' to push the result to %s, branch %s, or 'x' to exit without pushing." % (host_repo,branch)).lower() + reply = ask_prompt("Type 'push' to push the result to {}, branch {}, or 'x' to exit without pushing.".format(', '.join([host_repo] + push_mirrors), branch)).lower() if reply == 'push': subprocess.check_call([GIT,'push',host_repo,'refs/heads/'+branch]) + for p_mirror in push_mirrors: + subprocess.check_call([GIT,'push',p_mirror,'refs/heads/'+branch]) break elif reply == 'x': sys.exit(1) diff --git a/contrib/devtools/optimize-pngs.py b/contrib/devtools/optimize-pngs.py index 6991dfac69c2..89c5729ac9bd 100755 --- a/contrib/devtools/optimize-pngs.py +++ b/contrib/devtools/optimize-pngs.py @@ -1,16 +1,29 @@ #!/usr/bin/env python3 -# Copyright (c) 2014-2015 The Bitcoin Core developers +# Copyright (c) 2014-2018 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. ''' Run this script every time you change one of the png files. Using pngcrush, it will optimize the png files, remove various color profiles, remove ancillary chunks (alla) and text chunks (text). #pngcrush -brute -ow -rem gAMA -rem cHRM -rem iCCP -rem sRGB -rem alla -rem text + + +Install instructions +==== + +virtualenv ./optimize-pngs-env +source ./optimize-pngs-env/bin/activate +pip install Pillow + +Running +==== + +./optimize-pngs.py ../dash_core/src/qt/res/animation/ ../dash_core/src/qt/res/icons/ ../dash_core/src/qt/res/images/ ../dash_core/share/pixmaps/ ''' +import argparse import os import sys import subprocess import hashlib -# pip3 install Pillow from PIL import Image # type: ignore[import] def file_hash(filename): @@ -25,33 +38,35 @@ def content_hash(filename): data = i.tobytes() return hashlib.sha256(data).hexdigest() +zopflipng = 'zopflipng' pngcrush = 'pngcrush' git = 'git' -folders = [ - "src/qt/res/animation", - "src/qt/res/icons", - "src/qt/res/images", - "share/pixmaps" -] -basePath = subprocess.check_output([git, 'rev-parse', '--show-toplevel'], universal_newlines=True, encoding='utf8').rstrip('\n') + +parser = argparse.ArgumentParser() +parser.add_argument('folder', nargs='+') +folders = parser.parse_args().folder +folders = [os.path.abspath(f) for f in folders] + totalSaveBytes = 0 noHashChange = True outputArray = [] -for folder in folders: - absFolder=os.path.join(basePath, folder) +for absFolder in folders: for file in os.listdir(absFolder): extension = os.path.splitext(file)[1] if extension.lower() == '.png': print("optimizing {}...".format(file), end =' ') file_path = os.path.join(absFolder, file) fileMetaMap = {'file' : file, 'osize': os.path.getsize(file_path), 'sha256Old' : file_hash(file_path)} - fileMetaMap['contentHashPre'] = content_hash(file_path) + contentHashPre = content_hash(file_path) + try: subprocess.call([pngcrush, "-brute", "-ow", "-rem", "gAMA", "-rem", "cHRM", "-rem", "iCCP", "-rem", "sRGB", "-rem", "alla", "-rem", "text", file_path], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.call([zopflipng, '-m', '-y', file_path, file_path], + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) except Exception: - print("pngcrush is not installed, aborting...") + print("pngcrush or zopflipng is not installed, aborting...") sys.exit(0) #verify @@ -60,9 +75,9 @@ def content_hash(filename): sys.exit(1) fileMetaMap['sha256New'] = file_hash(file_path) - fileMetaMap['contentHashPost'] = content_hash(file_path) + contentHashPost = content_hash(file_path) - if fileMetaMap['contentHashPre'] != fileMetaMap['contentHashPost']: + if contentHashPre != contentHashPost: print("Image contents of PNG file {} before and after crushing don't match".format(file)) sys.exit(1) diff --git a/contrib/devtools/update-translations.py b/contrib/devtools/update-translations.py index bc889878e6b2..a20945d3acbd 100755 --- a/contrib/devtools/update-translations.py +++ b/contrib/devtools/update-translations.py @@ -7,13 +7,14 @@ transifex. It will do the following automatically: +- remove current translations - fetch all translations using the tx tool - post-process them into valid and committable format - remove invalid control characters - remove location tags (makes diffs less noisy) - -TODO: -- auto-add new translations to the build system according to the translation process + - drop untranslated languages +- update git for added translations +- update build system ''' import subprocess import re @@ -22,18 +23,28 @@ import io import xml.etree.ElementTree as ET +# Name of Qt lconvert tool +LCONVERT = os.getenv('LCONVERT', 'lconvert') # Name of transifex tool -TX = 'tx' -# Name of source language file -SOURCE_LANG = 'dash_en.ts' +TX = os.getenv('TX', 'tx') +# Name of source language file without extension +SOURCE_LANG = 'dash_en' # Directory with locale files LOCALE_DIR = 'src/qt/locale' -# Minimum number of messages for translation to be considered at all -MIN_NUM_MESSAGES = 10 +# Minimum number of non-numerus messages for translation to be considered at all +MIN_NUM_NONNUMERUS_MESSAGES = 120 # Regexp to check for Bitcoin addresses ADDRESS_REGEXP = re.compile('([13]|bc1)[a-zA-Z0-9]{30,}') # Regexp to check for Dash addresses ADDRESS_REGEXP_DASH = re.compile('[X7][a-zA-Z0-9]{30,}') +# Path to git +GIT = os.getenv("GIT", "git") +# Original content file suffix +ORIGINAL_SUFFIX = '.orig' +# Native Qt translation file (TS) format +FORMAT_TS = '.ts' +# XLIFF file format +FORMAT_XLIFF = '.xlf' def check_at_repository_root(): if not os.path.exists('.git'): @@ -41,11 +52,39 @@ def check_at_repository_root(): print('Execute this script at the root of the repository', file=sys.stderr) sys.exit(1) +def remove_current_translations(): + ''' + Remove current translations. + We only want the active translations that are currently on transifex. + This leaves dash_en.ts untouched. + ''' + for (_,name) in all_ts_files(): + os.remove(name) + +def remove_orig_files(): + ''' + Remove temporary files that might be left behind. + ''' + for (_,name) in all_ts_files(suffix=ORIGINAL_SUFFIX): + os.remove(name + ORIGINAL_SUFFIX) + def fetch_all_translations(): - if subprocess.call([TX, 'pull', '-f', '-a']): + if subprocess.call([TX, 'pull', '--translations', '--force', '--all']): print('Error while fetching translations', file=sys.stderr) sys.exit(1) +def convert_xlf_to_ts(): + xliff_files = all_ts_files(file_format=FORMAT_XLIFF) + if not xliff_files: + return False + + for (_, name) in xliff_files: + outname = name.replace(FORMAT_XLIFF, FORMAT_TS) + print('Converting %s to %s...' % (name, outname)) + subprocess.check_call([LCONVERT, '-o', outname, '-i', name]) + os.remove(name) + return True + def find_format_specifiers(s): '''Find all format specifiers in a string.''' pos = 0 @@ -56,7 +95,7 @@ def find_format_specifiers(s): break try: specifiers.append(s[percent+1]) - except Exception: + except IndexError: print('Failed to get specifier') pos = percent+2 return specifiers @@ -105,10 +144,10 @@ def check_format_specifiers(source, translation, errors, numerus): return False return True -def all_ts_files(suffix=''): +def all_ts_files(file_format=FORMAT_TS, suffix='', include_source=False): for filename in os.listdir(LOCALE_DIR): # process only language files, and do not process source language - if not filename.endswith('.ts'+suffix) or filename == SOURCE_LANG+suffix: + if not filename.endswith(file_format + suffix) or (not include_source and filename == SOURCE_LANG + file_format + suffix): continue if suffix: # remove provided suffix filename = filename[0:-len(suffix)] @@ -141,7 +180,42 @@ def contains_dash_addr(text, errors): return True return False -def postprocess_translations(reduce_diff_hacks=False): +def postprocess_message(filename, message, xliff_compatible_mode): + translation_node = message.find('translation') + if not xliff_compatible_mode and translation_node.get('type') == 'unfinished': + return False + + numerus = message.get('numerus') == 'yes' + source = message.find('source').text + # pick all numerusforms + if numerus: + translations = [i.text for i in translation_node.findall('numerusform')] + else: + if translation_node.text is None or translation_node.text == source: + return False + + translations = [translation_node.text] + + for translation in translations: + if translation is None: + continue + errors = [] + valid = check_format_specifiers(source, translation, errors, numerus) and not contains_bitcoin_addr(translation, errors) + valid = valid and not contains_dash_addr(translation, errors) + + for error in errors: + print('%s: %s' % (filename, error)) + + if not valid: + return False + + # Remove location tags + for location in message.findall('location'): + message.remove(location) + + return True + +def postprocess_translations(xliff_compatible_mode, reduce_diff_hacks=False): print('Checking and postprocessing...') if reduce_diff_hacks: @@ -150,13 +224,12 @@ def postprocess_translations(reduce_diff_hacks=False): ET._escape_cdata = escape_cdata for (filename,filepath) in all_ts_files(): - os.rename(filepath, filepath+'.orig') + os.rename(filepath, filepath + ORIGINAL_SUFFIX) - have_errors = False - for (filename,filepath) in all_ts_files('.orig'): + for (filename,filepath) in all_ts_files(suffix=ORIGINAL_SUFFIX): # pre-fixups to cope with transifex output parser = ET.XMLParser(encoding='utf-8') # need to override encoding because 'utf8' is not understood only 'utf-8' - with open(filepath + '.orig', 'rb') as f: + with open(filepath + ORIGINAL_SUFFIX, 'rb') as f: data = f.read() # remove control characters; this must be done over the entire file otherwise the XML parser will fail data = remove_invalid_characters(data) @@ -166,45 +239,21 @@ def postprocess_translations(reduce_diff_hacks=False): root = tree.getroot() for context in root.findall('context'): for message in context.findall('message'): - numerus = message.get('numerus') == 'yes' - source = message.find('source').text - translation_node = message.find('translation') - # pick all numerusforms - if numerus: - translations = [i.text for i in translation_node.findall('numerusform')] - else: - translations = [translation_node.text] - - for translation in translations: - if translation is None: - continue - errors = [] - valid = check_format_specifiers(source, translation, errors, numerus) and not contains_bitcoin_addr(translation, errors) - valid = valid and not contains_dash_addr(translation, errors) - - for error in errors: - print('%s: %s' % (filename, error)) - - if not valid: # set type to unfinished and clear string if invalid - translation_node.clear() - translation_node.set('type', 'unfinished') - have_errors = True - - # Remove location tags - for location in message.findall('location'): - message.remove(location) - - # Remove entire message if it is an unfinished translation - if translation_node.get('type') == 'unfinished': + if not postprocess_message(filename, message, xliff_compatible_mode): context.remove(message) + if not context.findall('message'): + root.remove(context) + # check if document is (virtually) empty, and remove it if so - num_messages = 0 + num_nonnumerus_messages = 0 for context in root.findall('context'): for message in context.findall('message'): - num_messages += 1 - if num_messages < MIN_NUM_MESSAGES: - print('Removing %s, as it contains only %i messages' % (filepath, num_messages)) + if message.get('numerus') != 'yes': + num_nonnumerus_messages += 1 + + if num_nonnumerus_messages < MIN_NUM_NONNUMERUS_MESSAGES: + print('Removing %s, as it contains only %i non-numerus messages' % (filepath, num_nonnumerus_messages)) continue # write fixed-up tree @@ -218,10 +267,44 @@ def postprocess_translations(reduce_diff_hacks=False): f.write(out) else: tree.write(filepath, encoding='utf-8') - return have_errors + +def update_git(): + ''' + Add new files to git repository. + (Removing files isn't necessary here, as `git commit -a` will take care of removing files that are gone) + ''' + file_paths = [filepath for (filename, filepath) in all_ts_files()] + subprocess.check_call([GIT, 'add'] + file_paths) + +def update_build_systems(): + ''' + Update build system and Qt resource descriptors. + ''' + filename_lang = [re.match(r'((dash_(.*)).ts)$', filename).groups() for (filename, filepath) in all_ts_files(include_source=True)] + filename_lang.sort(key=lambda x: x[0]) + + # update qrc locales + with open('src/qt/dash_locale.qrc', 'w', encoding='utf8') as f: + f.write('\n') + f.write(' \n') + for (filename, basename, lang) in filename_lang: + f.write(f' locale/{basename}.qm\n') + f.write(' \n') + f.write('\n') + + # update Makefile include + with open('src/Makefile.qt_locale.include', 'w', encoding='utf8') as f: + f.write('QT_TS = \\\n') + f.write(' \\\n'.join(f' qt/locale/{filename}' for (filename, basename, lang) in filename_lang)) + f.write('\n') # make sure last line doesn't end with a backslash if __name__ == '__main__': check_at_repository_root() - # fetch_all_translations() - postprocess_translations() + remove_current_translations() + fetch_all_translations() + xliff_compatible_mode = convert_xlf_to_ts() + postprocess_translations(xliff_compatible_mode) + update_git() + update_build_systems() + remove_orig_files() diff --git a/doc/translation_process.md b/doc/translation_process.md index 894bf6c2c61f..a511b7eeb3b1 100644 --- a/doc/translation_process.md +++ b/doc/translation_process.md @@ -57,11 +57,10 @@ The Transifex Dash project config file is included as part of the repo. It can b To assist in updating translations, we have created a script to help. 1. `python contrib/devtools/update-translations.py` -2. Update `src/qt/dash_locale.qrc` manually or via - `ls src/qt/locale/*ts|xargs -n1 basename|sed 's/\(dash_\(.*\)\).ts/locale\/\1.qm<\/file>/'` -3. Update `src/Makefile.qt_locale.include` manually or via - `ls src/qt/locale/*ts|xargs -n1 basename|sed 's/\(dash_\(.*\)\).ts/ qt\/locale\/\1.ts \\/'` -4. `git add` new translations from `src/qt/locale/` + +It fetches the translations, post-processes them, `git add`s new files from +`src/qt/locale/` and regenerates `src/qt/dash_locale.qrc` and +`src/Makefile.qt_locale.include`. **Do not directly download translations** one by one from the Transifex website, as we do a few post-processing steps before committing the translations.