blob: a19d5b976ee97dab5c8dc5fe71cd52220f32b9cf [file] [log] [blame]
Marco Brohet4683bee2014-02-28 01:06:03 +01001#!/usr/bin/python2
Marco Brohetf1742722014-03-04 22:41:18 +01002# -*- coding: utf-8 -*-
Marco Brohet4683bee2014-02-28 01:06:03 +01003# cm_crowdin_sync.py
4#
5# Updates Crowdin source translations and pulls translations
Marco Brohet8b78a1b2014-02-28 21:01:26 +01006# directly to CyanogenMod's Git.
7#
8# Copyright (C) 2014 The CyanogenMod Project
9#
10# Licensed under the Apache License, Version 2.0 (the "License");
11# you may not use this file except in compliance with the License.
12# You may obtain a copy of the License at
13#
14# http://www.apache.org/licenses/LICENSE-2.0
15#
16# Unless required by applicable law or agreed to in writing, software
17# distributed under the License is distributed on an "AS IS" BASIS,
18# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
19# See the License for the specific language governing permissions and
20# limitations under the License.
Marco Brohet4683bee2014-02-28 01:06:03 +010021
Marco Brohetf1742722014-03-04 22:41:18 +010022import codecs
Marco Brohet4683bee2014-02-28 01:06:03 +010023import git
24import mmap
Marco Brohetcf4069b2014-02-28 18:48:17 +010025import os
Marco Brohet4683bee2014-02-28 01:06:03 +010026import os.path
27import re
28import shutil
29import subprocess
30import sys
31from urllib import urlretrieve
32from xml.dom import minidom
33
Marco Brohet8b78a1b2014-02-28 21:01:26 +010034def get_caf_additions(strings_base, strings_cm):
35 # Load AOSP file and resources
36 xml_base = minidom.parse(strings_base)
37 list_base_string = xml_base.getElementsByTagName('string')
38 list_base_string_array = xml_base.getElementsByTagName('string-array')
39 list_base_plurals = xml_base.getElementsByTagName('plurals')
40 # Load CM file and resources
41 xml_cm = minidom.parse(strings_cm)
42 list_cm_string = xml_cm.getElementsByTagName('string')
43 list_cm_string_array = xml_cm.getElementsByTagName('string-array')
44 list_cm_plurals = xml_cm.getElementsByTagName('plurals')
Marco Brohet4683bee2014-02-28 01:06:03 +010045
Marco Brohet8b78a1b2014-02-28 21:01:26 +010046 # All names from CM
47 names_cm_string = []
48 names_cm_string_array = []
49 names_cm_plurals = []
50 # All names from AOSP
51 names_base_string = []
52 names_base_string_array = []
53 names_base_plurals = []
54
55 # Get all names from CM
56 for s in list_cm_string :
57 if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
58 names_cm_string.append(s.attributes['name'].value)
59 for s in list_cm_string_array :
60 if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
61 names_cm_string_array.append(s.attributes['name'].value)
62 for s in list_cm_plurals :
63 if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
64 names_cm_plurals.append(s.attributes['name'].value)
65 # Get all names from AOSP
66 for s in list_base_string :
67 if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
68 names_base_string.append(s.attributes['name'].value)
69 for s in list_base_string_array :
70 if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
71 names_base_string_array.append(s.attributes['name'].value)
72 for s in list_base_plurals :
73 if not s.hasAttribute('translatable') and not s.hasAttribute('translate'):
74 names_base_plurals.append(s.attributes['name'].value)
75
76 # Store all differences in this list
77 caf_additions = []
78
Marco Brohet3cd76422014-03-08 21:12:09 +010079 # Store all found strings/arrays/plurals.
80 # Prevent duplicates with product attribute
81 found_string = []
82 found_string_array = []
83 found_plurals = []
84
Marco Brohet8b78a1b2014-02-28 21:01:26 +010085 # Add all CAF additions to the list 'caf_additions'
86 for z in names_cm_string:
Marco Brohet3cd76422014-03-08 21:12:09 +010087 if z not in names_base_string and z not in found_string:
Marco Brohet25623ce2014-03-08 19:13:07 +010088 for string_item in list_cm_string:
89 if string_item.attributes['name'].value == z:
90 caf_additions.append(' ' + string_item.toxml())
Marco Brohet3cd76422014-03-08 21:12:09 +010091 found_string.append(z)
Marco Brohet25623ce2014-03-08 19:13:07 +010092 for y in names_cm_string_array:
Marco Brohet3cd76422014-03-08 21:12:09 +010093 if y not in names_base_string_array and y not in found_string_array:
Marco Brohet25623ce2014-03-08 19:13:07 +010094 for string_array_item in list_cm_string_array:
95 if string_array_item.attributes['name'].value == y:
96 caf_additions.append(' ' + string_array_item.toxml())
Marco Brohet3cd76422014-03-08 21:12:09 +010097 found_string_array.append(y)
Marco Brohet25623ce2014-03-08 19:13:07 +010098 for x in names_cm_plurals:
Marco Brohet3cd76422014-03-08 21:12:09 +010099 if x not in names_base_plurals and x not in found_plurals:
Marco Brohet25623ce2014-03-08 19:13:07 +0100100 for plurals_item in list_cm_plurals:
101 if plurals_item.attributes['name'].value == x:
102 caf_additions.append(' ' + plurals_item.toxml())
Marco Brohet3cd76422014-03-08 21:12:09 +0100103 found_plurals.append(x)
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100104
105 # Done :-)
106 return caf_additions
107
Marco Brohet7165b4e2014-03-02 17:31:17 +0100108def sync_js_translations(sync_type, path, lang=''):
109 # lang is necessary in download mode
110 if sync_type == 'download' and lang == '':
111 sys.exit('Invalid syntax. Language code is required in download mode.')
112
113 # Read source en.js file. This is necessary for both upload and download modes
Marco Brohet44657ed2014-03-04 22:49:23 +0100114 with codecs.open(path + 'en.js', 'r', 'utf-8') as f:
Marco Brohet7165b4e2014-03-02 17:31:17 +0100115 content = f.readlines()
116
117 if sync_type == 'upload':
118 # Prepare XML file structure
119 doc = xml.dom.minidom.Document()
120 header = doc.createElement('resources')
Marco Brohet44657ed2014-03-04 22:49:23 +0100121 file_write = codecs.open(path + 'en.xml', 'w', 'utf-8')
Marco Brohet7165b4e2014-03-02 17:31:17 +0100122 else:
123 # Open translation files
Marco Brohet44657ed2014-03-04 22:49:23 +0100124 file_write = codecs.open(path + lang + '.js', 'w', 'utf-8')
Marco Brohet7165b4e2014-03-02 17:31:17 +0100125 xml_base = xml.dom.minidom.parse(path + lang + '.xml')
126 tags = xml_base.getElementsByTagName('string')
127
128 # Read each line of en.js
129 for a_line in content:
130 # Regex to determine string id
131 m = re.search(' (.*): [\'|\"]', a_line)
132 if m is not None:
133 for string_id in m.groups():
134 if string_id is not None:
135 # Find string id
136 string_id = string_id.replace(' ', '')
137 m2 = re.search('\'(.*)\'|"(.*)"', a_line)
138 # Find string contents
139 for string_content in m2.groups():
140 if string_content is not None:
141 break
142 if sync_type == 'upload':
143 # In upload mode, create the appropriate string element.
144 contents = doc.createElement('string')
145 contents.attributes['name'] = string_id
146 contents.appendChild(doc.createTextNode(string_content))
147 header.appendChild(contents)
148 else:
149 # In download mode, check if string_id matches a name attribute in the translation XML file.
150 # If it does, replace English text with the translation.
151 # If it does not, English text will remain and will be added to the file to retain the file structure.
152 for string in tags:
153 if string.attributes['name'].value == string_id:
154 a_line = a_line.replace(string_content.rstrip(), string.firstChild.nodeValue)
155 break
156 break
157 # In download mode do not write comments
158 if sync_type == 'download' and not '//' in a_line:
159 # Add language identifier (1)
160 if 'cmaccount.l10n.en' in a_line:
161 a_line = a_line.replace('l10n.en', 'l10n.' + lang)
162 # Add language identifier (2)
163 if 'l10n.add(\'en\'' in a_line:
164 a_line = a_line.replace('l10n.add(\'en\'', 'l10n.add(\'' + lang + '\'')
165 # Now write the line
166 file_write.write(a_line)
Marco Brohet7165b4e2014-03-02 17:31:17 +0100167
168 # Create XML file structure
169 if sync_type == 'upload':
170 header.appendChild(contents)
171 contents = header.toxml().replace('<string', '\n <string').replace('</resources>', '\n</resources>')
172 file_write.write('<?xml version="1.0" encoding="utf-8"?>\n')
173 file_write.write('<!-- .JS CONVERTED TO .XML - DO NOT MERGE THIS FILE -->\n')
174 file_write.write(contents)
175
176 # Close file
177 file_write.close()
178
Michael Bestas10006822014-04-17 15:36:46 +0300179def push_as_commit(path, name, branch):
Michael Bestas3daf4032014-04-17 16:38:20 +0300180 # CM gerrit nickname
181 username = 'your_nickname'
182
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100183 # Get path
184 path = os.getcwd() + '/' + path
185
186 # Create git commit
187 repo = git.Repo(path)
Michael Bestasf2b10902014-06-21 15:15:34 +0300188 removed_files = repo.git.ls_files(d=True).split('\n')
189 try:
190 repo.git.rm(removed_files)
191 except:
192 pass
Marco Brohet98179ea2014-07-09 21:05:53 +0200193 repo.git.add('-A')
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100194 try:
Michael Bestas3daf4032014-04-17 16:38:20 +0300195 repo.git.commit(m='Automatic translation import')
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100196 except:
Marco Brohet98179ea2014-07-09 21:05:53 +0200197 print('Failed to create commit for ' + name + ', probably empty: skipping')
198 return
199 repo.git.push('ssh://' + username + '@review.cyanogenmod.org:29418/' + name, 'HEAD:refs/for/' + branch)
200 print('Succesfully pushed commit for ' + name)
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100201
202print('Welcome to the CM Crowdin sync script!')
203
204print('\nSTEP 0: Checking dependencies')
Marco Brohet7165b4e2014-03-02 17:31:17 +0100205# Check for Ruby version of crowdin-cli
Marco Brohet4683bee2014-02-28 01:06:03 +0100206if subprocess.check_output(['rvm', 'all', 'do', 'gem', 'list', 'crowdin-cli', '-i']) == 'true':
207 sys.exit('You have not installed crowdin-cli. Terminating.')
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100208else:
209 print('Found: crowdin-cli')
Marco Brohet7165b4e2014-03-02 17:31:17 +0100210# Check for caf.xml
Marco Brohet4683bee2014-02-28 01:06:03 +0100211if not os.path.isfile('caf.xml'):
212 sys.exit('You have no caf.xml. Terminating.')
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100213else:
214 print('Found: caf.xml')
Michael Bestas7fd0f072014-03-29 20:29:13 +0200215# Check for android/default.xml
216if not os.path.isfile('android/default.xml'):
217 sys.exit('You have no android/default.xml. Terminating.')
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100218else:
Michael Bestas7fd0f072014-03-29 20:29:13 +0200219 print('Found: android/default.xml')
Michael Bestas6aeac312014-04-19 22:05:58 +0300220# Check for extra_packages.xml
221if not os.path.isfile('extra_packages.xml'):
222 sys.exit('You have no extra_packages.xml. Terminating.')
223else:
224 print('Found: extra_packages.xml')
Marco Brohet7165b4e2014-03-02 17:31:17 +0100225# Check for repo
226try:
227 subprocess.check_output(['which', 'repo'])
228except:
229 sys.exit('You have not installed repo. Terminating.')
Marco Brohet4683bee2014-02-28 01:06:03 +0100230
Marco Brohet25623ce2014-03-08 19:13:07 +0100231print('\nSTEP 1: Create cm_caf.xml')
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100232# Load caf.xml
Marco Brohet25623ce2014-03-08 19:13:07 +0100233print('Loading caf.xml')
234xml = minidom.parse('caf.xml')
235items = xml.getElementsByTagName('item')
Marco Brohet4683bee2014-02-28 01:06:03 +0100236
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100237# Store all created cm_caf.xml files in here.
238# Easier to remove them afterwards, as they cannot be committed
Marco Brohet25623ce2014-03-08 19:13:07 +0100239cm_caf = []
Marco Brohet4683bee2014-02-28 01:06:03 +0100240
Marco Brohet25623ce2014-03-08 19:13:07 +0100241for item in items:
242 # Create tmp dir for download of AOSP base file
243 path_to_values = item.attributes["path"].value
244 subprocess.call(['mkdir', '-p', 'tmp/' + path_to_values])
245 # Create cm_caf.xml - header
246 f = codecs.open(path_to_values + '/cm_caf.xml', 'w', 'utf-8')
247 f.write('<?xml version="1.0" encoding="utf-8"?>\n')
Michael Bestasdb8462d2014-03-25 15:11:50 +0200248 f.write('<!--\n')
249 f.write(' Copyright (C) 2014 The CyanogenMod Project\n')
250 f.write('\n')
251 f.write(' Licensed under the Apache License, Version 2.0 (the "License");\n')
252 f.write(' you may not use this file except in compliance with the License.\n')
253 f.write(' You may obtain a copy of the License at\n')
254 f.write('\n')
255 f.write(' http://www.apache.org/licenses/LICENSE-2.0\n')
256 f.write('\n')
257 f.write(' Unless required by applicable law or agreed to in writing, software\n')
258 f.write(' distributed under the License is distributed on an "AS IS" BASIS,\n')
259 f.write(' WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n')
260 f.write(' See the License for the specific language governing permissions and\n')
261 f.write(' limitations under the License.\n')
262 f.write('-->\n')
Marco Brohet25623ce2014-03-08 19:13:07 +0100263 f.write('<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2">\n')
264 # Create cm_caf.xml - contents
265 # This means we also support multiple base files (e.g. checking if strings.xml and arrays.xml are changed)
266 contents = []
267 item_aosp = item.getElementsByTagName('aosp')
268 for aosp_item in item_aosp:
269 url = aosp_item.firstChild.nodeValue
270 xml_file = aosp_item.attributes["file"].value
271 path_to_base = 'tmp/' + path_to_values + '/' + xml_file
272 path_to_cm = path_to_values + '/' + xml_file
273 urlretrieve(url, path_to_base)
274 contents = contents + get_caf_additions(path_to_base, path_to_cm)
275 for addition in contents:
276 f.write(addition + '\n')
277 # Create cm_caf.xml - the end
278 f.write('</resources>')
279 f.close()
280 cm_caf.append(path_to_values + '/cm_caf.xml')
281 print('Created ' + path_to_values + '/cm_caf.xml')
Marco Brohet4683bee2014-02-28 01:06:03 +0100282
283print('\nSTEP 2: Upload Crowdin source translations')
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100284# Execute 'crowdin-cli upload sources' and show output
Marco Brohet4683bee2014-02-28 01:06:03 +0100285print(subprocess.check_output(['crowdin-cli', 'upload', 'sources']))
286
Michael Bestasb119b2c2014-04-04 18:55:16 +0300287print('\nSTEP 3: Download Crowdin translations')
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100288# Execute 'crowdin-cli download' and show output
Marco Brohetcf4069b2014-02-28 18:48:17 +0100289print(subprocess.check_output(['crowdin-cli', "download"]))
Marco Brohet4683bee2014-02-28 01:06:03 +0100290
Michael Bestasb119b2c2014-04-04 18:55:16 +0300291print('\nSTEP 4A: Clean up of source cm_caf.xmls')
Marco Brohet25623ce2014-03-08 19:13:07 +0100292# Remove all cm_caf.xml files, which you can find in the list 'cm_caf'
293for cm_caf_file in cm_caf:
Michael Bestas35194f32014-07-07 03:11:41 +0300294 print('Removing ' + cm_caf_file)
Marco Brohet25623ce2014-03-08 19:13:07 +0100295 os.remove(cm_caf_file)
Marco Brohetcf4069b2014-02-28 18:48:17 +0100296
Marco Brohet25623ce2014-03-08 19:13:07 +0100297print('\nSTEP 4B: Clean up of temp dir')
298# We are done with cm_caf.xml files, so remove tmp/
299shutil.rmtree(os.getcwd() + '/tmp')
Marco Brohetcf4069b2014-02-28 18:48:17 +0100300
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100301print('\nSTEP 4C: Clean up of empty translations')
302# Some line of code that I found to find all XML files
Marco Brohet4683bee2014-02-28 01:06:03 +0100303result = [os.path.join(dp, f) for dp, dn, filenames in os.walk(os.getcwd()) for f in filenames if os.path.splitext(f)[1] == '.xml']
304for xml_file in result:
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100305 # We hate empty, useless files. Crowdin exports them with <resources/> (sometimes with xliff).
306 # That means: easy to find
Marco Brohet4683bee2014-02-28 01:06:03 +0100307 if '<resources/>' in open(xml_file).read():
Michael Bestas35194f32014-07-07 03:11:41 +0300308 print('Removing ' + xml_file)
Marco Brohet4683bee2014-02-28 01:06:03 +0100309 os.remove(xml_file)
Marco Brohetcf4069b2014-02-28 18:48:17 +0100310 elif '<resources xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"/>' in open(xml_file).read():
Michael Bestas35194f32014-07-07 03:11:41 +0300311 print('Removing ' + xml_file)
Michael Bestas4bfe4522014-05-24 01:22:05 +0300312 os.remove(xml_file)
313 elif '<resources xmlns:android="http://schemas.android.com/apk/res/android" xmlns:xliff="urn:oasis:names:tc:xliff:document:1.2"/>' in open(xml_file).read():
Michael Bestas35194f32014-07-07 03:11:41 +0300314 print('Removing ' + xml_file)
Michael Bestas4bfe4522014-05-24 01:22:05 +0300315 os.remove(xml_file)
Marco Brohet4683bee2014-02-28 01:06:03 +0100316
Marco Brohet4683bee2014-02-28 01:06:03 +0100317print('\nSTEP 5: Push translations to Git')
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100318# Get all files that Crowdin pushed
Marco Brohet4683bee2014-02-28 01:06:03 +0100319proc = subprocess.Popen(['crowdin-cli', 'list', 'sources'],stdout=subprocess.PIPE)
Michael Bestas7fd0f072014-03-29 20:29:13 +0200320xml = minidom.parse('android/default.xml')
Michael Bestas6aeac312014-04-19 22:05:58 +0300321xml_extra = minidom.parse('extra_packages.xml')
Marco Brohet4683bee2014-02-28 01:06:03 +0100322items = xml.getElementsByTagName('project')
Michael Bestas6aeac312014-04-19 22:05:58 +0300323items += xml_extra.getElementsByTagName('project')
Marco Brohetcf4069b2014-02-28 18:48:17 +0100324all_projects = []
Marco Brohet4683bee2014-02-28 01:06:03 +0100325
Marco Brohetcf4069b2014-02-28 18:48:17 +0100326for path in iter(proc.stdout.readline,''):
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100327 # Remove the \n at the end of each line
Marco Brohetcf4069b2014-02-28 18:48:17 +0100328 path = path.rstrip()
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100329 # Get project root dir from Crowdin's output
Marco Brohetcf4069b2014-02-28 18:48:17 +0100330 m = re.search('/(.*Superuser)/Superuser.*|/(.*LatinIME).*|/(frameworks/base).*|/(.*CMFileManager).*|/(device/.*/.*)/.*/res/values.*|/(hardware/.*/.*)/.*/res/values.*|/(.*)/res/values.*', path)
331 for good_path in m.groups():
Marco Brohet8b78a1b2014-02-28 21:01:26 +0100332 # When a project has multiple translatable files, Crowdin will give duplicates.
333 # We don't want that (useless empty commits), so we save each project in all_projects
334 # and check if it's already in there.
Marco Brohetcf4069b2014-02-28 18:48:17 +0100335 if good_path is not None and not good_path in all_projects:
336 all_projects.append(good_path)
Marco Brohetcf4069b2014-02-28 18:48:17 +0100337 for project_item in items:
Michael Bestas6aeac312014-04-19 22:05:58 +0300338 # We need to have the Github repository for the git push url.
339 # Obtain them from android/default.xml or extra_packages.xml.
Marco Brohetcf4069b2014-02-28 18:48:17 +0100340 if project_item.attributes["path"].value == good_path:
Michael Bestas10006822014-04-17 15:36:46 +0300341 if project_item.hasAttribute('revision'):
342 branch = project_item.attributes['revision'].value
343 else:
344 branch = 'cm-11.0'
Michael Bestas35194f32014-07-07 03:11:41 +0300345 print('Committing ' + project_item.attributes['name'].value + ' on branch ' + branch + ' (based on android/default.xml or extra_packages.xml)')
Michael Bestas10006822014-04-17 15:36:46 +0300346 push_as_commit(good_path, project_item.attributes['name'].value, branch)
Marco Brohet4683bee2014-02-28 01:06:03 +0100347
Michael Bestas6aeac312014-04-19 22:05:58 +0300348print('\nSTEP 6: Done!')