crowdin: Remove workarounds for CAF strings
* All CAF strings are moved to cm_* files now
* This makes the translation progress much cleaner and removes the additional noise from the AOSP files
* Allows us to improve strings without having to hunt them down between the AOSP strings we can't touch
Change-Id: Ied9ca33d964d93311dcbb72867bb53509590114b
diff --git a/crowdin_sync.py b/crowdin_sync.py
index 273c302..d5addc1 100755
--- a/crowdin_sync.py
+++ b/crowdin_sync.py
@@ -35,165 +35,11 @@
############################################ FUNCTIONS #############################################
-def get_caf_additions(strings_base, strings_cm):
- # Load AOSP file and resources
- xml_base = minidom.parse(strings_base)
- list_base_string = xml_base.getElementsByTagName('string')
- list_base_string_array = xml_base.getElementsByTagName('string-array')
- list_base_plurals = xml_base.getElementsByTagName('plurals')
- # Load CM file and resources
- xml_cm = minidom.parse(strings_cm)
- list_cm_string = xml_cm.getElementsByTagName('string')
- list_cm_string_array = xml_cm.getElementsByTagName('string-array')
- list_cm_plurals = xml_cm.getElementsByTagName('plurals')
-
- # Load all names from AOSP
- names_base_string = []
- names_base_string_array = []
- names_base_plurals = []
-
- for s in list_base_string :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
- names_base_string.append(s.attributes['name'].value)
- for s in list_base_string_array :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
- names_base_string_array.append(s.attributes['name'].value)
- for s in list_base_plurals :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
- names_base_plurals.append(s.attributes['name'].value)
-
- # Store all differences in this list
- caf_additions = []
-
- # Loop through all CM resources. If an ID cannot be found in AOSP base file,
- # the ID is from CAF and will be added to 'caf_additions'
- for s in list_cm_string :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate') and not s.attributes['name'].value in names_base_string:
- caf_additions.append(' ' + s.toxml())
- for s in list_cm_string_array :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate') and not s.attributes['name'].value in names_base_string_array:
- caf_additions.append(' ' + s.toxml())
- for s in list_cm_plurals :
- if not s.hasAttribute('translatable') and not s.hasAttribute('translate') and not s.attributes['name'].value in names_base_plurals:
- caf_additions.append(' ' + s.toxml())
-
- # Done
- return caf_additions
-
def get_default_branch(xml):
xml_default = xml.getElementsByTagName('default')[0]
xml_default_revision = xml_default.attributes['revision'].value
return re.search('refs/heads/(.*)', xml_default_revision).groups()[0]
-def purge_caf_additions(strings_base, strings_cm):
- # Load AOSP file and resources
- xml_base = minidom.parse(strings_base)
- list_base_string = xml_base.getElementsByTagName('string')
- list_base_string_array = xml_base.getElementsByTagName('string-array')
- list_base_plurals = xml_base.getElementsByTagName('plurals')
- # Load CM file and resources
- xml_cm = minidom.parse(strings_cm)
- list_cm_string = xml_cm.getElementsByTagName('string')
- list_cm_string_array = xml_cm.getElementsByTagName('string-array')
- list_cm_plurals = xml_cm.getElementsByTagName('plurals')
- with codecs.open(strings_cm, 'r', 'utf-8') as f:
- content = [line.rstrip() for line in f]
- shutil.copyfile(strings_cm, strings_cm + '.backup')
- file_this = codecs.open(strings_cm, 'w', 'utf-8')
-
- # All names from AOSP
- names_base_string = []
- names_base_string_array = []
- names_base_plurals = []
-
- # Get all names from AOSP
- for s in list_base_string :
- names_base_string.append(s.attributes['name'].value)
- for s in list_base_string_array :
- names_base_string_array.append(s.attributes['name'].value)
- for s in list_base_plurals :
- names_base_plurals.append(s.attributes['name'].value)
-
- # Get all names from CM
- content2 = []
- for s in list_cm_string :
- name = s.attributes['name'].value
- if name not in names_base_string:
- true = 0
- content2 = []
- for i in content:
- if true == 0:
- test = re.search('(<string name=\"' + name + '\")', i)
- if test is not None:
- test2 = re.search('(</string>)', i)
- if test2:
- true = 2
- else:
- true = 1
- i = ''
- elif true == 1:
- test2 = re.search('(</string>)', i)
- if test2 is not None:
- true = 2
- i = ''
- elif true == 2:
- true = 3
- content2.append(i)
- content = content2
- for s in list_cm_string_array :
- name = s.attributes['name'].value
- if name not in names_base_string_array:
- true = 0
- content2 = []
- for i in content:
- if true == 0:
- test = re.search('(<string-array name=\"' + name + '\")', i)
- if test is not None:
- test2 = re.search('(</string-array>)', i)
- if test2:
- true = 2
- else:
- true = 1
- i = ''
- elif true == 1:
- test2 = re.search('(</string-array>)', i)
- if test2 is not None:
- true = 2
- i = ''
- elif true == 2:
- true = 3
- content2.append(i)
- content = content2
- for s in list_cm_plurals :
- name = s.attributes['name'].value
- if name not in names_base_plurals:
- true = 0
- content2 = []
- for i in content:
- if true == 0:
- test = re.search('(<plurals name=\"' + name + '\")', i)
- if test is not None:
- test2 = re.search('(</plurals>)', i)
- if test2:
- true = 2
- else:
- true = 1
- i = ''
- elif true == 1:
- test2 = re.search('(</plurals>)', i)
- if test2 is not None:
- true = 2
- i = ''
- elif true == 2:
- # The actual purging is done!
- true = 3
- content2.append(i)
- content = content2
-
- for addition in content:
- file_this.write(addition + '\n')
- file_this.close()
-
def push_as_commit(path, name, branch, username):
print('Committing ' + name + ' on branch ' + branch)
@@ -330,12 +176,6 @@
else:
print('Found: android/default.xml')
-# Check for crowdin/caf.xml
-if not os.path.isfile('crowdin/caf.xml'):
- sys.exit('You have no crowdin/caf.xml. Terminating.')
-else:
- print('Found: crowdin/caf.xml')
-
# Check for crowdin/config_aosp.yaml
if not os.path.isfile('crowdin/config_aosp.yaml'):
sys.exit('You have no crowdin/config_aosp.yaml. Terminating.')
@@ -378,11 +218,6 @@
print('Loading: android/default.xml')
xml_android = minidom.parse('android/default.xml')
-# Variables regarding crowdin/caf.xml
-print('Loading: crowdin/caf.xml')
-xml = minidom.parse('crowdin/caf.xml')
-items = xml.getElementsByTagName('item')
-
# Variables regarding crowdin/js.xml
print('Loading: crowdin/js.xml')
xml_js = minidom.parse('crowdin/js.xml')
@@ -392,101 +227,18 @@
default_branch = get_default_branch(xml_android)
print('Default branch: ' + default_branch)
-print('\nSTEP 0C: Download AOSP base files')
-for item in items:
- path_to_values = item.attributes['path'].value
- subprocess.call(['mkdir', '-p', 'tmp/' + path_to_values])
- for aosp_item in item.getElementsByTagName('aosp'):
- url = aosp_item.firstChild.nodeValue
- xml_file = aosp_item.attributes['file'].value
- path_to_base = 'tmp/' + path_to_values + '/' + xml_file
- urlretrieve(url, path_to_base)
- print('Downloaded: ' + path_to_base)
-
############################################## STEP 1 ##############################################
-print('\nSTEP 1: Remove CAF additions (non-AOSP supported languages)')
-# Store all created cm_caf.xml files in here.
-# Easier to remove them afterwards, as they cannot be committed
-cm_caf_add = []
-
-for item in items:
- # Create tmp dir for download of AOSP base file
- path_to_values = item.attributes['path'].value
- for aosp_item in item.getElementsByTagName('aosp'):
- xml_file = aosp_item.attributes['file'].value
- path_to_base = 'tmp/' + path_to_values + '/' + xml_file
- path_to_cm = path_to_values + '/' + xml_file
- purge_caf_additions(path_to_base, path_to_cm)
- cm_caf_add.append(path_to_cm)
- print('Purged ' + path_to_cm + ' from CAF additions')
-
-############################################## STEP 2 ##############################################
-
-print('\nSTEP 2: Upload Crowdin source translations (non-AOSP supported languages)')
+print('\nSTEP 1: Upload Crowdin source translations (non-AOSP supported languages)')
# Execute 'crowdin-cli upload sources' and show output
print(subprocess.check_output(['crowdin-cli', '--config=crowdin/crowdin_aosp.yaml', '--identity=crowdin/config_aosp.yaml', 'upload', 'sources']))
-############################################## STEP 3 ##############################################
-
-print('\nSTEP 3: Revert removal of CAF additions (non-AOSP supported languages)')
-for purged_file in cm_caf_add:
- os.remove(purged_file)
- shutil.move(purged_file + '.backup', purged_file)
- print('Reverted purged file ' + purged_file)
-
-############################################## STEP 4 ##############################################
-
-print('\nSTEP 4: Create source cm_caf.xmls (AOSP supported languages)')
-# Store all created cm_caf.xml files in here.
-# Easier to remove them afterwards, as they cannot be committed
-cm_caf = []
-
-for item in items:
- # Create tmp dir for download of AOSP base file
- path_to_values = item.attributes["path"].value
- # Create cm_caf.xml - header
- f = codecs.open(path_to_values + '/cm_caf.xml', 'w', 'utf-8')
- f.write('<?xml version="1.0" encoding="utf-8"?>\n')
- f.write('<!--\n')
- f.write(' Copyright (C) 2014 The CyanogenMod Project\n')
- f.write('\n')
- f.write(' Licensed under the Apache License, Version 2.0 (the "License");\n')
- f.write(' you may not use this file except in compliance with the License.\n')
- f.write(' You may obtain a copy of the License at\n')
- f.write('\n')
- f.write(' http://www.apache.org/licenses/LICENSE-2.0\n')
- f.write('\n')
- f.write(' Unless required by applicable law or agreed to in writing, software\n')
- f.write(' distributed under the License is distributed on an "AS IS" BASIS,\n')
- f.write(' WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n')
- f.write(' See the License for the specific language governing permissions and\n')
- f.write(' limitations under the License.\n')
- f.write('-->\n')
- f.write('<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">\n')
- # Create cm_caf.xml - contents
- # This means we also support multiple base files (e.g. checking if strings.xml and arrays.xml are changed)
- contents = []
- item_aosp = item.getElementsByTagName('aosp')
- for aosp_item in item_aosp:
- xml_file = aosp_item.attributes["file"].value
- path_to_base = 'tmp/' + path_to_values + '/' + xml_file
- path_to_cm = path_to_values + '/' + xml_file
- contents = contents + get_caf_additions(path_to_base, path_to_cm)
- for addition in contents:
- f.write(addition + '\n')
- # Create cm_caf.xml - the end
- f.write('</resources>')
- f.close()
- cm_caf.append(path_to_values + '/cm_caf.xml')
- print('Created ' + path_to_values + '/cm_caf.xml')
-
-############################################## STEP 5 ##############################################
+############################################## STEP 2 ##############################################
# JS files cannot be translated easily on Crowdin. That's why they are uploaded as Android XML
# files. At this time, the (English) JS source file is converted to an XML file, so Crowdin knows it
# needs to download for it.
-#print('\nSTEP 5: Convert .js source translations to .xml')
+#print('\nSTEP 2: Convert .js source translations to .xml')
#
#js_files = []
#
@@ -496,31 +248,25 @@
# print('Converted: ' + path + 'en.js to en.xml')
# js_files.append(path + 'en.js')
-############################################## STEP 6 ##############################################
+############################################## STEP 3 ##############################################
-print('\nSTEP 6: Upload Crowdin source translations (AOSP supported languages)')
+print('\nSTEP 3: Upload Crowdin source translations (AOSP supported languages)')
# Execute 'crowdin-cli upload sources' and show output
print(subprocess.check_output(['crowdin-cli', '--config=crowdin/crowdin_cm.yaml', '--identity=crowdin/config_cm.yaml', 'upload', 'sources']))
-############################################## STEP 7 ##############################################
+############################################## STEP 4 ##############################################
-print('\nSTEP 7A: Download Crowdin translations (AOSP supported languages)')
+print('\nSTEP 4A: Download Crowdin translations (AOSP supported languages)')
# Execute 'crowdin-cli download' and show output
print(subprocess.check_output(['crowdin-cli', '--config=crowdin/crowdin_cm.yaml', '--identity=crowdin/config_cm.yaml', 'download']))
-print('\nSTEP 7B: Download Crowdin translations (non-AOSP supported languages)')
+print('\nSTEP 4B: Download Crowdin translations (non-AOSP supported languages)')
# Execute 'crowdin-cli download' and show output
print(subprocess.check_output(['crowdin-cli', '--config=crowdin/crowdin_aosp.yaml', '--identity=crowdin/config_aosp.yaml', 'download']))
-############################################## STEP 8 ##############################################
+############################################## STEP 5 ##############################################
-print('\nSTEP 8: Remove temp dir')
-# We are done with cm_caf.xml files, so remove tmp/
-shutil.rmtree(os.getcwd() + '/tmp')
-
-############################################## STEP 9 ##############################################
-
-print('\nSTEP 9: Remove useless empty translations')
+print('\nSTEP 5: Remove useless empty translations')
# Some line of code that I found to find all XML files
result = [os.path.join(dp, f) for dp, dn, filenames in os.walk(os.getcwd()) for f in filenames if os.path.splitext(f)[1] == '.xml']
empty_contents = {'<resources/>', '<resources xmlns:android="http://schemas.android.com/apk/res/android"/>', '<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"/>', '<resources xmlns:android="http://schemas.android.com/apk/res/android" xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"/>'}
@@ -535,24 +281,16 @@
# print('Removing ' + js_file)
# os.remove(js_file)
-############################################## STEP 10 #############################################
+############################################## STEP 6 ##############################################
-print('\nSTEP 10: Create a list of pushable translations')
+print('\nSTEP 6: Create a list of pushable translations')
# Get all files that Crowdin pushed
proc = subprocess.Popen(['crowdin-cli --config=crowdin/crowdin_cm.yaml --identity=crowdin/config_cm.yaml list sources && crowdin-cli --config=crowdin/crowdin_aosp.yaml --identity=crowdin/config_aosp.yaml list sources'], stdout=subprocess.PIPE, shell=True)
proc.wait() # Wait for the above to finish
-############################################## STEP 11 #############################################
+############################################## STEP 7 ##############################################
-print('\nSTEP 11: Remove unwanted source cm_caf.xmls (AOSP supported languages)')
-# Remove all cm_caf.xml files, which you can find in the list 'cm_caf'
-for cm_caf_file in cm_caf:
- print('Removing ' + cm_caf_file)
- os.remove(cm_caf_file)
-
-############################################## STEP 12 #############################################
-
-#print('\nSTEP 12: Convert JS-XML translations to their JS format')
+#print('\nSTEP 7: Convert JS-XML translations to their JS format')
#
#for item in items_js:
# path = item.attributes['path'].value
@@ -564,9 +302,9 @@
# os.remove(path + '/' + item.attributes['source'].value)
#
-############################################## STEP 13 #############################################
+############################################## STEP 8 ##############################################
-print('\nSTEP 13: Commit to Gerrit')
+print('\nSTEP 8: Commit to Gerrit')
xml_extra = minidom.parse('crowdin/extra_packages.xml')
items = xml_android.getElementsByTagName('project')
items += xml_extra.getElementsByTagName('project')