forked from Grasscutters/Grasscutter
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Language linting (Grasscutters#1382)
- Loading branch information
Showing
8 changed files
with
676 additions
and
300 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
name: "Language Lint" | ||
on: | ||
workflow_dispatch: ~ | ||
push: | ||
paths: | ||
- "**.java" | ||
- "**.json" | ||
branches: | ||
- "stable" | ||
- "development" | ||
pull_request: | ||
paths: | ||
- "**.java" | ||
- "**.json" | ||
types: | ||
- opened | ||
- synchronize | ||
- reopened | ||
jobs: | ||
Lint-Language-Keys: | ||
runs-on: ubuntu-latest | ||
steps: | ||
- uses: actions/checkout@v3 | ||
- uses: actions/setup-python@v4 | ||
with: | ||
python-version: '3.10' # Version range or exact version of a Python version to use, using SemVer's version range syntax | ||
- run: python3 manage_languages.py -l |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,321 @@ | ||
# Written for Python 3.6+ | ||
# Older versions don't retain insertion order of regular dicts | ||
import argparse | ||
import cmd | ||
import json | ||
import os | ||
import re | ||
from pprint import pprint | ||
|
||
INDENT = 2 | ||
PRIMARY_LANGUAGE = 'en-US.json' | ||
PRIMARY_FALLBACK_PREFIX = '🇺🇸' # This is invisible in-game, terminal emulators might render it | ||
LANGUAGE_FOLDER = 'src/main/resources/languages/' | ||
LANGUAGE_FILENAMES = sorted(os.listdir(LANGUAGE_FOLDER), key=lambda x: 'AAA' if x == PRIMARY_LANGUAGE else x) | ||
SOURCE_FOLDER = 'src/' | ||
SOURCE_EXTENSIONS = ('java') | ||
|
||
|
||
def ppprint(data): | ||
pprint(data, width=130, sort_dicts=False, compact=True) | ||
|
||
|
||
class JsonHelpers: | ||
@staticmethod | ||
def load(filename: str) -> dict: | ||
with open(filename, 'r') as file: | ||
return json.load(file) | ||
|
||
@staticmethod | ||
def save(filename: str, data: dict) -> None: | ||
with open(filename, 'w', encoding='utf-8', newline='\n') as file: | ||
json.dump(data, file, ensure_ascii=False, indent=INDENT) | ||
|
||
@staticmethod | ||
def flatten(data: dict, prefix='') -> dict: | ||
output = {} | ||
for key, value in data.items(): | ||
if isinstance(value, dict): | ||
for k,v in JsonHelpers.flatten(value, f'{prefix}{key}.').items(): | ||
output[k] = v | ||
else: | ||
output[f'{prefix}{key}'] = value | ||
return output | ||
|
||
@staticmethod | ||
def unflatten(data: dict) -> dict: | ||
output = {} | ||
def add_key(k: list, value, d: dict): | ||
if len(k) == 1: | ||
d[k[0]] = value | ||
else: | ||
d[k[0]] = d.get(k[0], {}) | ||
add_key(k[1:], value, d[k[0]]) | ||
for key, value in data.items(): | ||
add_key(key.split('.'), value, output) | ||
return output | ||
|
||
@staticmethod | ||
def pprint_keys(keys, indent=4) -> str: | ||
# Only strip down to one level | ||
padding = ' ' * indent | ||
roots = {} | ||
for key in keys: | ||
root, _, k = key.rpartition('.') | ||
roots[root] = roots.get(root, []) | ||
roots[root].append(k) | ||
lines = [] | ||
for root, ks in roots.items(): | ||
if len(ks) > 1: | ||
lines.append(f'{padding}{root}.[{", ".join(ks)}]') | ||
else: | ||
lines.append(f'{padding}{root}.{ks[0]}') | ||
return ',\n'.join(lines) | ||
|
||
@staticmethod | ||
def deep_clone_and_fill(d1: dict, d2: dict, fallback_prefix=PRIMARY_FALLBACK_PREFIX) -> dict: | ||
out = {} | ||
for key, value in d1.items(): | ||
if isinstance(value, dict): | ||
out[key] = JsonHelpers.deep_clone_and_fill(value, d2.get(key, {}), fallback_prefix) | ||
else: | ||
v2 = d2.get(key, value) | ||
if type(value) == str and v2 == value: | ||
out[key] = fallback_prefix + value | ||
else: | ||
out[key] = v2 | ||
return out | ||
|
||
|
||
class LanguageManager: | ||
TRANSLATION_KEY = re.compile(r'[Tt]ranslate.*"(\w+\.[\w\.]+)"') | ||
POTENTIAL_KEY = re.compile(r'"(\w+\.[\w\.]+)"') | ||
|
||
def __init__(self): | ||
self.load_jsons() | ||
|
||
def load_jsons(self): | ||
self.language_jsons = [JsonHelpers.load(LANGUAGE_FOLDER + filename) for filename in LANGUAGE_FILENAMES] | ||
self.flattened_jsons = [JsonHelpers.flatten(j) for j in self.language_jsons] | ||
self.update_keys() | ||
|
||
def update_keys(self): | ||
self.key_sets = [set(j.keys()) for j in self.flattened_jsons] | ||
self.common_keys = set.intersection(*self.key_sets) | ||
self.all_keys = set.union(*self.key_sets) | ||
self.used_keys = self.find_all_used_keys(self.all_keys) | ||
self.missing_keys = self.used_keys - self.common_keys | ||
self.unused_keys = self.all_keys - self.used_keys | ||
|
||
def find_all_used_keys(self, expected_keys=[]) -> set: | ||
# Note that this will only find string literals passed to the translate() or sendTranslatedMessage() methods! | ||
# String variables passed to them can be checked against expected_keys | ||
used = set() | ||
potential = set() | ||
for root, dirs, files in os.walk(SOURCE_FOLDER): | ||
for file in files: | ||
if file.rpartition('.')[-1] in SOURCE_EXTENSIONS: | ||
filename = os.path.join(root, file) | ||
with open(filename, 'r') as f: | ||
data = f.read() # Loads in entire file at once | ||
for k in self.TRANSLATION_KEY.findall(data): | ||
used.add(k) | ||
for k in self.POTENTIAL_KEY.findall(data): | ||
potential.add(k) | ||
return used | (potential & expected_keys) | ||
|
||
def _lint_report_language(self, lang: str, keys: set, flattened: dict, primary_language_flattened: dict) -> None: | ||
missing = self.used_keys - keys | ||
unused = keys - self.used_keys | ||
identical_keys = set() if (lang == PRIMARY_LANGUAGE) else {key for key in keys if primary_language_flattened.get(key, None) == flattened.get(key)} | ||
placeholder_keys = {key for key in keys if flattened.get(key).startswith(PRIMARY_FALLBACK_PREFIX)} | ||
p1 = f'Language {lang} has {len(missing)} missing keys and {len(unused)} unused keys.' | ||
p2 = 'This is the primary language.' if (lang == PRIMARY_LANGUAGE) else f'{len(identical_keys)} match {PRIMARY_LANGUAGE}, {len(placeholder_keys)} have the placeholder mark.' | ||
print(f'{p1} {p2}') | ||
|
||
lint_categories = { | ||
'Missing': missing, | ||
'Unused': unused, | ||
f'Matches {PRIMARY_LANGUAGE}': identical_keys, | ||
'Placeholder': placeholder_keys, | ||
} | ||
for name, category in lint_categories.items(): | ||
if len(category) > 0: | ||
print(name + ':') | ||
print(JsonHelpers.pprint_keys(sorted(category))) | ||
|
||
def lint_report(self) -> None: | ||
print(f'There are {len(self.missing_keys)} translation keys in use that are missing from one or more language files.') | ||
print(f'There are {len(self.unused_keys)} translation keys in language files that are not used.') | ||
primary_language_flattened = self.flattened_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)] | ||
for lang, keys, flattened in zip(LANGUAGE_FILENAMES, self.key_sets, self.flattened_jsons): | ||
print('') | ||
self._lint_report_language(lang, keys, flattened, primary_language_flattened) | ||
|
||
def rename_keys(self, key_remappings: dict) -> None: | ||
# Unfortunately we can't rename keys in-place preserving insertion order, so we have to make new dicts | ||
for i in range(len(self.flattened_jsons)): | ||
self.flattened_jsons[i] = {key_remappings.get(k,k):v for k,v in self.flattened_jsons[i].items()} | ||
|
||
def update_secondary_languages(self): | ||
# Push en_US fallback | ||
primary_language_json = self.language_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)] | ||
for filename, lang in zip(LANGUAGE_FILENAMES, self.language_jsons): | ||
if filename != PRIMARY_LANGUAGE: | ||
js = JsonHelpers.deep_clone_and_fill(primary_language_json, lang) | ||
JsonHelpers.save(LANGUAGE_FOLDER + filename, js) | ||
|
||
def update_all_languages_from_flattened(self): | ||
for filename, flat in zip(LANGUAGE_FILENAMES, self.flattened_jsons): | ||
JsonHelpers.save(LANGUAGE_FOLDER + filename, JsonHelpers.unflatten(flat)) | ||
|
||
def save_flattened_languages(self, prefix='flat_'): | ||
for filename, flat in zip(LANGUAGE_FILENAMES, self.flattened_jsons): | ||
JsonHelpers.save(prefix + filename, flat) | ||
|
||
|
||
class InteractiveRename(cmd.Cmd): | ||
intro = 'Welcome to the interactive rename shell. Type help or ? to list commands.\n' | ||
prompt = '(rename) ' | ||
file = None | ||
|
||
def __init__(self, language_manager: LanguageManager) -> None: | ||
super().__init__() | ||
self.language_manager = language_manager | ||
self.flat_keys = [key for key in language_manager.flattened_jsons[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)].keys()] | ||
self.mappings = {} | ||
|
||
def do_add(self, arg): | ||
''' | ||
Prepare to rename an existing translation key. Will not actually rename anything until you confirm all your pending changes with 'rename'. | ||
e.g. a single string: add commands.execution.argument_error commands.generic.invalid.argument | ||
e.g. a group: add commands.enter_dungeon commands.new_enter_dungeon | ||
''' | ||
args = arg.split() | ||
if len(args) < 2: | ||
self.do_help('add') | ||
return | ||
old, new = args[:2] | ||
if old in self.flat_keys: | ||
self.mappings[old] = new | ||
else: | ||
# Check if we are renaming a higher level | ||
if not old.endswith('.'): | ||
old = old + '.' | ||
results = [key for key in self.flat_keys if key.startswith(old)] | ||
if len(results) > 0: | ||
if not new.endswith('.'): | ||
new = new + '.' | ||
new_mappings = {key: key.replace(old, new) for key in results} | ||
# Ask for confirmation | ||
print('Will add the following mappings:') | ||
ppprint(new_mappings) | ||
print('Add these mappings? [y/N]') | ||
if self.prompt_yn(): | ||
for k,v in new_mappings.items(): | ||
self.mappings[k] = v | ||
else: | ||
print('No translation keys matched!') | ||
|
||
def complete_add(self, text: str, line: str, begidx: int, endidx: int) -> list: | ||
if text == '': | ||
return [k for k in {key.partition('.')[0] for key in self.flat_keys}] | ||
results = [key for key in self.flat_keys if key.startswith(text)] | ||
if len(results) > 40: | ||
# Collapse categories | ||
if text[-1] != '.': | ||
text = text + '.' | ||
level = text.count('.') + 1 | ||
new_results = {'.'.join(key.split('.')[:level]) for key in results} | ||
return list(new_results) | ||
return results | ||
|
||
def do_remove(self, arg): | ||
''' | ||
Remove a pending rename mapping. Takes the old name of the key, not the new one. | ||
e.g. a single key: remove commands.execution.argument_error | ||
e.g. a group: remove commands.enter_dungeon | ||
''' | ||
old = arg.split()[0] | ||
if old in self.mappings: | ||
self.mappings.pop(old) | ||
else: | ||
# Check if we are renaming a higher level | ||
if not old.endswith('.'): | ||
old = old + '.' | ||
results = [key for key in self.mappings if key.startswith(old)] | ||
if len(results) > 0: | ||
# Ask for confirmation | ||
print('Will remove the following pending mappings:') | ||
print(JsonHelpers.pprint_keys(results)) | ||
print('Delete these mappings? [y/N]') | ||
if self.prompt_yn(): | ||
for key in results: | ||
self.mappings.pop(key) | ||
else: | ||
print('No pending rename mappings matched!') | ||
|
||
def complete_remove(self, text: str, line: str, begidx: int, endidx: int) -> list: | ||
return [key for key in self.mappings if key.startswith(text)] | ||
|
||
def do_rename(self, _arg): | ||
'Applies pending renames and overwrites language jsons.' | ||
# Ask for confirmation | ||
print('Will perform the following mappings:') | ||
ppprint(self.mappings) | ||
print('Perform and save these rename mappings? [y/N]') | ||
if self.prompt_yn(): | ||
self.language_manager.rename_keys(self.mappings) | ||
self.language_manager.update_all_languages_from_flattened() | ||
print('Renamed keys, closing') | ||
return True | ||
else: | ||
print('Do you instead wish to quit without saving? [yes/N]') | ||
if self.prompt_yn(True): | ||
print('Left rename shell without renaming') | ||
return True | ||
|
||
def prompt_yn(self, strict_yes=False): | ||
if strict_yes: | ||
return input('(yes/N) ').lower() == 'yes' | ||
return input('(y/N) ').lower()[0] == 'y' | ||
|
||
|
||
def main(args: argparse.Namespace): | ||
# print(args) | ||
language_manager = LanguageManager() | ||
errors = None | ||
if args.lint_report: | ||
language_manager.lint_report() | ||
missing = language_manager.used_keys - language_manager.key_sets[LANGUAGE_FILENAMES.index(PRIMARY_LANGUAGE)] | ||
if len(missing) > 0: | ||
errors = f'[ERROR] {len(missing)} keys missing from primary language json!\n{JsonHelpers.pprint_keys(missing)}' | ||
if prefix := args.save_flattened: | ||
language_manager.save_flattened_languages(prefix) | ||
if args.update: | ||
print('Updating secondary languages') | ||
language_manager.update_secondary_languages() | ||
if args.interactive_rename: | ||
language_manager.load_jsons() # Previous actions may have changed them on-disk | ||
try: | ||
InteractiveRename(language_manager).cmdloop() | ||
except KeyboardInterrupt: | ||
print('Left rename shell without renaming') | ||
if errors: | ||
print(errors) | ||
exit(1) | ||
|
||
|
||
|
||
if __name__ == "__main__": | ||
parser = argparse.ArgumentParser(description="Manage Grasscutter's language json files.") | ||
parser.add_argument('-u', '--update', action='store_true', | ||
help=f'Update secondary language files to conform to the layout of the primary language file ({PRIMARY_LANGUAGE}) and contain any new keys from it.') | ||
parser.add_argument('-l', '--lint-report', action='store_true', | ||
help='Prints a lint report, listing unused, missing, and untranslated keys among all language jsons.') | ||
parser.add_argument('-f', '--save-flattened', const='./flat_', metavar='prefix', nargs='?', | ||
help='Save copies of all the language jsons in a flattened key form.') | ||
parser.add_argument('-i', '--interactive-rename', action='store_true', | ||
help='Enter interactive rename mode, in which you can specify keys in flattened form to be renamed.') | ||
args = parser.parse_args() | ||
main(args) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.