crowdin: Rename script to something simpler
Change-Id: I4910590d85ba8af9b485ff3b48b071c5ab1df73a
diff --git a/crowdin_sync.py b/crowdin_sync.py
new file mode 100755
index 0000000..319855d
--- /dev/null
+++ b/crowdin_sync.py
@@ -0,0 +1,445 @@
+#!/usr/bin/python2
+# -*- coding: utf-8 -*-
+# crowdin_sync.py
+#
+# Updates Crowdin source translations and pushes translations
+# directly to CyanogenMod's Gerrit.
+#
+# Copyright (C) 2014 The CyanogenMod Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import codecs
+import git
+import os
+import os.path
+import re
+import shutil
+import subprocess
+import sys
+from urllib import urlretrieve
+from xml.dom import minidom
+
+def get_caf_additions(strings_base, strings_cm):
+ # Load AOSP file and resources
+ xml_base = minidom.parse(strings_base)
+ list_base_string = xml_base.getElementsByTagName('string')
+ list_base_string_array = xml_base.getElementsByTagName('string-array')
+ list_base_plurals = xml_base.getElementsByTagName('plurals')
+ # Load CM file and resources
+ xml_cm = minidom.parse(strings_cm)
+ list_cm_string = xml_cm.getElementsByTagName('string')
+ list_cm_string_array = xml_cm.getElementsByTagName('string-array')
+ list_cm_plurals = xml_cm.getElementsByTagName('plurals')
+
+ # All names from CM
+ names_cm_string = []
+ names_cm_string_array = []
+ names_cm_plurals = []
+ # All names from AOSP
+ names_base_string = []
+ names_base_string_array = []
+ names_base_plurals = []
+
+ # Get all names from CM
+ for s in list_cm_string :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
+ names_cm_string.append(s.attributes['name'].value)
+ for s in list_cm_string_array :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
+ names_cm_string_array.append(s.attributes['name'].value)
+ for s in list_cm_plurals :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
+ names_cm_plurals.append(s.attributes['name'].value)
+ # Get all names from AOSP
+ for s in list_base_string :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
+ names_base_string.append(s.attributes['name'].value)
+ for s in list_base_string_array :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
+ names_base_string_array.append(s.attributes['name'].value)
+ for s in list_base_plurals :
+ if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
+ names_base_plurals.append(s.attributes['name'].value)
+
+ # Store all differences in this list
+ caf_additions = []
+
+ # Store all found strings/arrays/plurals.
+ # Prevent duplicates with product attribute
+ found_string = []
+ found_string_array = []
+ found_plurals = []
+
+ # Add all CAF additions to the list 'caf_additions'
+ for z in names_cm_string:
+ if z not in names_base_string and z not in found_string:
+ for string_item in list_cm_string:
+ if string_item.attributes['name'].value == z:
+ caf_additions.append(' ' + string_item.toxml())
+ found_string.append(z)
+ for y in names_cm_string_array:
+ if y not in names_base_string_array and y not in found_string_array:
+ for string_array_item in list_cm_string_array:
+ if string_array_item.attributes['name'].value == y:
+ caf_additions.append(' ' + string_array_item.toxml())
+ found_string_array.append(y)
+ for x in names_cm_plurals:
+ if x not in names_base_plurals and x not in found_plurals:
+ for plurals_item in list_cm_plurals:
+ if plurals_item.attributes['name'].value == x:
+ caf_additions.append(' ' + plurals_item.toxml())
+ found_plurals.append(x)
+
+ # Done :-)
+ return caf_additions
+
+def purge_caf_additions(strings_base, strings_cm):
+ # Load AOSP file and resources
+ xml_base = minidom.parse(strings_base)
+ list_base_string = xml_base.getElementsByTagName('string')
+ list_base_string_array = xml_base.getElementsByTagName('string-array')
+ list_base_plurals = xml_base.getElementsByTagName('plurals')
+ # Load CM file and resources
+ xml_cm = minidom.parse(strings_cm)
+ list_cm_string = xml_cm.getElementsByTagName('string')
+ list_cm_string_array = xml_cm.getElementsByTagName('string-array')
+ list_cm_plurals = xml_cm.getElementsByTagName('plurals')
+ with codecs.open(strings_cm, 'r', 'utf-8') as f:
+ content = [line.rstrip() for line in f]
+ shutil.copyfile(strings_cm, strings_cm + '.backup')
+ file_this = codecs.open(strings_cm, 'w', 'utf-8')
+
+ # All names from AOSP
+ names_base_string = []
+ names_base_string_array = []
+ names_base_plurals = []
+
+ # Get all names from AOSP
+ for s in list_base_string :
+ names_base_string.append(s.attributes['name'].value)
+ for s in list_base_string_array :
+ names_base_string_array.append(s.attributes['name'].value)
+ for s in list_base_plurals :
+ names_base_plurals.append(s.attributes['name'].value)
+
+ # Get all names from CM
+ content2 = []
+ for s in list_cm_string :
+ name = s.attributes['name'].value
+ if name not in names_base_string:
+ true = 0
+ content2 = []
+ for i in content:
+ if true == 0:
+ test = re.search('(<string name=\"' + name + '\")', i)
+ if test is not None:
+ test2 = re.search('(</string>)', i)
+ if test2:
+ true = 2
+ else:
+ true = 1
+ i = ''
+ elif true == 1:
+ test2 = re.search('(</string>)', i)
+ if test2 is not None:
+ true = 2
+ i = ''
+ elif true == 2:
+ true = 3
+ content2.append(i)
+ content = content2
+ for s in list_cm_string_array :
+ name = s.attributes['name'].value
+ if name not in names_base_string_array:
+ true = 0
+ content2 = []
+ for i in content:
+ if true == 0:
+ test = re.search('(<string-array name=\"' + name + '\")', i)
+ if test is not None:
+ test2 = re.search('(</string-array>)', i)
+ if test2:
+ true = 2
+ else:
+ true = 1
+ i = ''
+ elif true == 1:
+ test2 = re.search('(</string-array>)', i)
+ if test2 is not None:
+ true = 2
+ i = ''
+ elif true == 2:
+ true = 3
+ content2.append(i)
+ content = content2
+ for s in list_cm_plurals :
+ name = s.attributes['name'].value
+ if name not in names_base_plurals:
+ true = 0
+ content2 = []
+ for i in content:
+ if true == 0:
+ test = re.search('(<plurals name=\"' + name + '\")', i)
+ if test is not None:
+ test2 = re.search('(</plurals>)', i)
+ if test2:
+ true = 2
+ else:
+ true = 1
+ i = ''
+ elif true == 1:
+ test2 = re.search('(</plurals>)', i)
+ if test2 is not None:
+ true = 2
+ i = ''
+ elif true == 2:
+ # The actual purging is done!
+ true = 3
+ content2.append(i)
+ content = content2
+
+ for addition in content:
+ file_this.write(addition + '\n')
+ file_this.close()
+
+def push_as_commit(path, name, branch):
+ # CM gerrit nickname
+ username = 'your_nickname'
+
+ # Get path
+ path = os.getcwd() + '/' + path
+
+ # Create git commit
+ repo = git.Repo(path)
+ removed_files = repo.git.ls_files(d=True).split('\n')
+ try:
+ repo.git.rm(removed_files)
+ except:
+ pass
+ repo.git.add('-A')
+ try:
+ repo.git.commit(m='Automatic translation import')
+ except:
+ print('Failed to create commit for ' + name + ', probably empty: skipping')
+ return
+ repo.git.push('ssh://' + username + '@review.cyanogenmod.org:29418/' + name, 'HEAD:refs/for/' + branch)
+ print('Succesfully pushed commit for ' + name)
+
+print('Welcome to the CM Crowdin sync script!')
+
+print('\nSTEP 0: Checking dependencies')
+# Check for Ruby version of crowdin-cli
+if subprocess.check_output(['rvm', 'all', 'do', 'gem', 'list', 'crowdin-cli', '-i']) == 'true':
+ sys.exit('You have not installed crowdin-cli. Terminating.')
+else:
+ print('Found: crowdin-cli')
+
+# Check for repo
+try:
+ subprocess.check_output(['which', 'repo'])
+except:
+ sys.exit('You have not installed repo. Terminating.')
+
+# Check for android/default.xml
+if not os.path.isfile('android/default.xml'):
+ sys.exit('You have no android/default.xml. Terminating.')
+else:
+ print('Found: android/default.xml')
+
+# Check for crowdin/caf.xml
+if not os.path.isfile('crowdin/caf.xml'):
+ sys.exit('You have no crowdin/caf.xml. Terminating.')
+else:
+ print('Found: crowdin/caf.xml')
+
+# Check for crowdin/crowdin_aosp.yaml
+if not os.path.isfile('crowdin/crowdin_aosp.yaml'):
+ sys.exit('You have no crowdin/crowdin_aosp.yaml. Terminating.')
+else:
+ print('Found: crowdin/crowdin_aosp.yaml')
+
+# Check for crowdin/crowdin_cm.yaml
+if not os.path.isfile('crowdin/crowdin_cm.yaml'):
+ sys.exit('You have no crowdin/crowdin_cm.yaml. Terminating.')
+else:
+ print('Found: crowdin/crowdin_cm.yaml')
+
+# Check for crowdin/extra_packages.xml
+if not os.path.isfile('crowdin/extra_packages.xml'):
+ sys.exit('You have no crowdin/extra_packages.xml. Terminating.')
+else:
+ print('Found: crowdin/extra_packages.xml')
+
+print('\nSTEP 1: Remove CAF additions (non-AOSP supported languages)')
+# Load crowdin/caf.xml
+print('Loading crowdin/caf.xml')
+xml = minidom.parse('crowdin/caf.xml')
+items = xml.getElementsByTagName('item')
+
+# Store all created cm_caf.xml files in here.
+# Easier to remove them afterwards, as they cannot be committed
+cm_caf_add = []
+
+for item in items:
+ # Create tmp dir for download of AOSP base file
+ path_to_values = item.attributes["path"].value
+ subprocess.call(['mkdir', '-p', 'tmp/' + path_to_values])
+ for aosp_item in item.getElementsByTagName('aosp'):
+ url = aosp_item.firstChild.nodeValue
+ xml_file = aosp_item.attributes["file"].value
+ path_to_base = 'tmp/' + path_to_values + '/' + xml_file
+ path_to_cm = path_to_values + '/' + xml_file
+ urlretrieve(url, path_to_base)
+ purge_caf_additions(path_to_base, path_to_cm)
+ cm_caf_add.append(path_to_cm)
+ print('Purged ' + path_to_cm + ' from CAF additions')
+
+print('\nSTEP 2: Upload Crowdin source translations (non-AOSP supported languages)')
+# Execute 'crowdin-cli upload sources' and show output
+print(subprocess.check_output(['crowdin-cli', '-c', 'crowdin/crowdin_aosp.yaml', 'upload', 'sources']))
+
+print('\nSTEP 3: Revert removal of CAF additions (non-AOSP supported languages)')
+for purged_file in cm_caf_add:
+ os.remove(purged_file)
+ shutil.move(purged_file + '.backup', purged_file)
+ print('Reverted purged file ' + purged_file)
+
+print('\nSTEP 4: Create source cm_caf.xmls (AOSP supported languages)')
+# Load crowdin/caf.xml
+print('Loading crowdin/caf.xml')
+xml = minidom.parse('crowdin/caf.xml')
+items = xml.getElementsByTagName('item')
+
+# Store all created cm_caf.xml files in here.
+# Easier to remove them afterwards, as they cannot be committed
+cm_caf = []
+
+for item in items:
+ # Create tmp dir for download of AOSP base file
+ path_to_values = item.attributes["path"].value
+ subprocess.call(['mkdir', '-p', 'tmp/' + path_to_values])
+ # Create cm_caf.xml - header
+ f = codecs.open(path_to_values + '/cm_caf.xml', 'w', 'utf-8')
+ f.write('<?xml version="1.0" encoding="utf-8"?>\n')
+ f.write('<!--\n')
+ f.write(' Copyright (C) 2014 The CyanogenMod Project\n')
+ f.write('\n')
+ f.write(' Licensed under the Apache License, Version 2.0 (the "License");\n')
+ f.write(' you may not use this file except in compliance with the License.\n')
+ f.write(' You may obtain a copy of the License at\n')
+ f.write('\n')
+ f.write(' http://www.apache.org/licenses/LICENSE-2.0\n')
+ f.write('\n')
+ f.write(' Unless required by applicable law or agreed to in writing, software\n')
+ f.write(' distributed under the License is distributed on an "AS IS" BASIS,\n')
+ f.write(' WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n')
+ f.write(' See the License for the specific language governing permissions and\n')
+ f.write(' limitations under the License.\n')
+ f.write('-->\n')
+ f.write('<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">\n')
+ # Create cm_caf.xml - contents
+ # This means we also support multiple base files (e.g. checking if strings.xml and arrays.xml are changed)
+ contents = []
+ item_aosp = item.getElementsByTagName('aosp')
+ for aosp_item in item_aosp:
+ url = aosp_item.firstChild.nodeValue
+ xml_file = aosp_item.attributes["file"].value
+ path_to_base = 'tmp/' + path_to_values + '/' + xml_file
+ path_to_cm = path_to_values + '/' + xml_file
+ urlretrieve(url, path_to_base)
+ contents = contents + get_caf_additions(path_to_base, path_to_cm)
+ for addition in contents:
+ f.write(addition + '\n')
+ # Create cm_caf.xml - the end
+ f.write('</resources>')
+ f.close()
+ cm_caf.append(path_to_values + '/cm_caf.xml')
+ print('Created ' + path_to_values + '/cm_caf.xml')
+
+print('\nSTEP 5: Upload Crowdin source translations (AOSP supported languages)')
+# Execute 'crowdin-cli upload sources' and show output
+print(subprocess.check_output(['crowdin-cli', '-c', 'crowdin/crowdin_cm.yaml', 'upload', 'sources']))
+
+print('\nSTEP 6A: Download Crowdin translations (AOSP supported languages)')
+# Execute 'crowdin-cli download' and show output
+print(subprocess.check_output(['crowdin-cli', '-c', 'crowdin/crowdin_cm.yaml', 'download']))
+
+print('\nSTEP 6B: Download Crowdin translations (non-AOSP supported languages)')
+# Execute 'crowdin-cli download' and show output
+print(subprocess.check_output(['crowdin-cli', '-c', 'crowdin/crowdin_aosp.yaml', 'download']))
+
+print('\nSTEP 7: Remove temp dir')
+# We are done with cm_caf.xml files, so remove tmp/
+shutil.rmtree(os.getcwd() + '/tmp')
+
+print('\nSTEP 8: Remove useless empty translations')
+# Some line of code that I found to find all XML files
+result = [os.path.join(dp, f) for dp, dn, filenames in os.walk(os.getcwd()) for f in filenames if os.path.splitext(f)[1] == '.xml']
+for xml_file in result:
+ # We hate empty, useless files. Crowdin exports them with <resources/> (sometimes with xliff).
+ # That means: easy to find
+ if '<resources/>' in open(xml_file).read():
+ print('Removing ' + xml_file)
+ os.remove(xml_file)
+ elif '<resources xmlns:android="http://schemas.android.com/apk/res/android"/>' in open(xml_file).read():
+ print('Removing ' + xml_file)
+ os.remove(xml_file)
+ elif '<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"/>' in open(xml_file).read():
+ print('Removing ' + xml_file)
+ os.remove(xml_file)
+ elif '<resources xmlns:android="http://schemas.android.com/apk/res/android" xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"/>' in open(xml_file).read():
+ print('Removing ' + xml_file)
+ os.remove(xml_file)
+
+print('\nSTEP 9: Create a list of pushable translations')
+# Get all files that Crowdin pushed
+proc = subprocess.Popen(['crowdin-cli -c crowdin/crowdin_cm.yaml list sources && crowdin-cli -c crowdin/crowdin_aosp.yaml list sources'], stdout=subprocess.PIPE, shell=True)
+proc.wait() # Wait for the above to finish
+
+print('\nSTEP 10: Remove unwanted source cm_caf.xmls (AOSP supported languages)')
+# Remove all cm_caf.xml files, which you can find in the list 'cm_caf'
+for cm_caf_file in cm_caf:
+ print('Removing ' + cm_caf_file)
+ os.remove(cm_caf_file)
+
+print('\nSTEP 11: Commit to Gerrit')
+xml = minidom.parse('android/default.xml')
+xml_extra = minidom.parse('crowdin/extra_packages.xml')
+items = xml.getElementsByTagName('project')
+items += xml_extra.getElementsByTagName('project')
+all_projects = []
+
+for path in iter(proc.stdout.readline,''):
+ # Remove the \n at the end of each line
+ path = path.rstrip()
+ # Get project root dir from Crowdin's output
+ m = re.search('/(.*Superuser)/Superuser.*|/(.*LatinIME).*|/(frameworks/base).*|/(.*CMFileManager).*|/(.*CMHome).*|/(device/.*/.*)/.*/res/values.*|/(hardware/.*/.*)/.*/res/values.*|/(.*)/res/values.*', path)
+ for good_path in m.groups():
+ # When a project has multiple translatable files, Crowdin will give duplicates.
+ # We don't want that (useless empty commits), so we save each project in all_projects
+ # and check if it's already in there.
+ if good_path is not None and not good_path in all_projects:
+ all_projects.append(good_path)
+ for project_item in items:
+ # We need to have the Github repository for the git push url.
+ # Obtain them from android/default.xml or crowdin/extra_packages.xml.
+ if project_item.attributes['path'].value == good_path:
+ if project_item.hasAttribute('revision'):
+ branch = project_item.attributes['revision'].value
+ else:
+ branch = 'cm-11.0'
+ print('Committing ' + project_item.attributes['name'].value + ' on branch ' + branch)
+ push_as_commit(good_path, project_item.attributes['name'].value, branch)
+
+print('\nDone!')