Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 1 | #!/usr/bin/env python2 |
| 2 | |
| 3 | # Copyright (C) 2013 Cybojenix <anthonydking@gmail.com> |
| 4 | # Copyright (C) 2013 The OmniROM Project |
| 5 | # Copyright (C) 2015 BlissRoms Project |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 6 | # |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 7 | # This program is free software: you can redistribute it and/or modify |
| 8 | # it under the terms of the GNU General Public License as published by |
| 9 | # the Free Software Foundation, either version 3 of the License, or |
| 10 | # (at your option) any later version. |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 11 | # |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 12 | # This program is distributed in the hope that it will be useful, |
| 13 | # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| 14 | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
| 15 | # GNU General Public License for more details. |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 16 | # |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 17 | # You should have received a copy of the GNU General Public License |
| 18 | # along with this program. If not, see <http://www.gnu.org/licenses/>. |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 19 | |
| 20 | from __future__ import print_function |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 21 | import json |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 22 | import sys |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 23 | import os |
| 24 | import re |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 25 | from xml.etree import ElementTree as ES |
| 26 | # Use the urllib importer from the Cyanogenmod roomservice |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 27 | try: |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 28 | # For python3 |
| 29 | import urllib.request |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 30 | except ImportError: |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 31 | # For python2 |
| 32 | import imp |
| 33 | import urllib2 |
| 34 | urllib = imp.new_module('urllib') |
| 35 | urllib.request = urllib2 |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 36 | |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 37 | # Config |
| 38 | # set this to the default remote to use in repo |
| 39 | default_rem = "github" |
| 40 | # set this to the default revision to use (branch/tag name) |
Amy Grace | ec47df5 | 2022-01-02 13:27:34 +0000 | [diff] [blame] | 41 | default_rev = "arcadia" |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 42 | # set this to the remote that you use for projects from your team repos |
| 43 | # example fetch="https://github.com/BlissRoms-Devices" |
| 44 | default_team_rem = "github" |
| 45 | # this shouldn't change unless google makes changes |
| 46 | local_manifest_dir = ".repo/local_manifests" |
| 47 | # change this to your name on github (or equivalent hosting) |
| 48 | android_team = "BlissRoms-Devices" |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 49 | |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 50 | |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 51 | def check_repo_exists(git_data): |
| 52 | if not int(git_data.get('total_count', 0)): |
| 53 | raise Exception("{} not found in {} Github, exiting " |
| 54 | "roomservice".format(device, android_team)) |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 55 | |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 56 | |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 57 | # Note that this can only be done 5 times per minute |
| 58 | def search_github_for_device(device): |
| 59 | git_device = '+'.join(re.findall('[a-z]+|[\d]+', device)) |
| 60 | git_search_url = "https://api.github.com/search/repositories" \ |
| 61 | "?q=%40{}+android_device+{}+fork:true".format(android_team, git_device) |
| 62 | git_req = urllib.request.Request(git_search_url) |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 63 | try: |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 64 | response = urllib.request.urlopen(git_req) |
| 65 | except urllib.request.HTTPError: |
| 66 | raise Exception("There was an issue connecting to github." |
| 67 | " Please try again in a minute") |
| 68 | git_data = json.load(response) |
| 69 | check_repo_exists(git_data) |
| 70 | print("found the {} device repo".format(device)) |
| 71 | return git_data |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 72 | |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 73 | |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 74 | def get_device_url(git_data): |
| 75 | device_url = "" |
| 76 | for item in git_data['items']: |
| 77 | temp_url = item.get('html_url') |
| 78 | if "{}/android_device".format(android_team) in temp_url: |
| 79 | try: |
| 80 | temp_url = temp_url[temp_url.index("android_device"):] |
| 81 | except ValueError: |
| 82 | pass |
| 83 | else: |
| 84 | if temp_url.endswith(device): |
| 85 | device_url = temp_url |
| 86 | break |
| 87 | |
| 88 | if device_url: |
| 89 | return device_url |
| 90 | raise Exception("{} not found in {} Github, exiting " |
| 91 | "roomservice".format(device, android_team)) |
| 92 | |
| 93 | |
| 94 | def parse_device_directory(device_url,device): |
| 95 | to_strip = "android_device" |
| 96 | repo_name = device_url[device_url.index(to_strip) + len(to_strip):] |
| 97 | repo_name = repo_name[:repo_name.index(device)] |
| 98 | repo_dir = repo_name.replace("_", "/") |
| 99 | repo_dir = repo_dir + device |
| 100 | return "device{}".format(repo_dir) |
| 101 | |
| 102 | |
| 103 | # Thank you RaYmAn |
| 104 | def iterate_manifests(check_all): |
| 105 | files = [] |
| 106 | if check_all: |
| 107 | for file in os.listdir(local_manifest_dir): |
| 108 | if file.endswith('.xml'): |
| 109 | files.append(os.path.join(local_manifest_dir, file)) |
| 110 | files.append('.repo/manifest.xml') |
| 111 | for file in files: |
| 112 | try: |
| 113 | man = ES.parse(file) |
| 114 | man = man.getroot() |
| 115 | except IOError, ES.ParseError: |
| 116 | print("WARNING: error while parsing %s" % file) |
| 117 | else: |
| 118 | for project in man.findall("project"): |
| 119 | yield project |
| 120 | |
| 121 | |
| 122 | def check_project_exists(url): |
| 123 | for project in iterate_manifests(True): |
| 124 | if project.get("name") == url: |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 125 | return True |
| 126 | return False |
| 127 | |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 128 | |
| 129 | def check_dup_path(directory): |
| 130 | for project in iterate_manifests(False): |
| 131 | if project.get("path") == directory: |
| 132 | print ("Duplicate path %s found! Removing" % directory) |
| 133 | return project.get("name") |
| 134 | return None |
| 135 | |
| 136 | # Use the indent function from http://stackoverflow.com/a/4590052 |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 137 | def indent(elem, level=0): |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 138 | i = ''.join(["\n", level*" "]) |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 139 | if len(elem): |
| 140 | if not elem.text or not elem.text.strip(): |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 141 | elem.text = ''.join([i, " "]) |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 142 | if not elem.tail or not elem.tail.strip(): |
| 143 | elem.tail = i |
| 144 | for elem in elem: |
| 145 | indent(elem, level+1) |
| 146 | if not elem.tail or not elem.tail.strip(): |
| 147 | elem.tail = i |
| 148 | else: |
| 149 | if level and (not elem.tail or not elem.tail.strip()): |
| 150 | elem.tail = i |
| 151 | |
Tom Powell | f8adf06 | 2020-02-25 20:45:43 -0800 | [diff] [blame] | 152 | |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 153 | def create_manifest_project(url, directory, |
| 154 | remote=default_rem, |
| 155 | revision=default_rev): |
| 156 | project_exists = check_project_exists(url) |
| 157 | |
| 158 | if project_exists: |
| 159 | return None |
| 160 | |
| 161 | dup_path = check_dup_path(directory) |
| 162 | if not dup_path is None: |
| 163 | write_to_manifest( |
| 164 | append_to_manifest( |
| 165 | create_manifest_remove(dup_path))) |
| 166 | |
| 167 | project = ES.Element("project", |
| 168 | attrib={ |
| 169 | "path": directory, |
| 170 | "name": url, |
| 171 | "remote": remote, |
| 172 | "revision": revision |
| 173 | }) |
| 174 | return project |
| 175 | |
| 176 | |
| 177 | def create_manifest_remove(url): |
| 178 | remove = ES.Element("remove-project", attrib={"name": url}) |
| 179 | return remove |
| 180 | |
| 181 | |
| 182 | def append_to_manifest(project): |
Tom Powell | f8adf06 | 2020-02-25 20:45:43 -0800 | [diff] [blame] | 183 | try: |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 184 | lm = ES.parse('/'.join([local_manifest_dir, "roomservice.xml"])) |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 185 | lm = lm.getroot() |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 186 | except IOError, ES.ParseError: |
| 187 | lm = ES.Element("manifest") |
| 188 | lm.append(project) |
| 189 | return lm |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 190 | |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 191 | |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 192 | def write_to_manifest(manifest): |
| 193 | indent(manifest) |
| 194 | raw_xml = ES.tostring(manifest).decode() |
| 195 | raw_xml = ''.join(['<?xml version="1.0" encoding="UTF-8"?>\n' |
| 196 | '<!--Please do not manually edit this file-->\n', |
| 197 | raw_xml]) |
| 198 | |
| 199 | with open('/'.join([local_manifest_dir, "roomservice.xml"]), 'w') as f: |
| 200 | f.write(raw_xml) |
| 201 | print("wrote the new roomservice manifest") |
| 202 | |
| 203 | def parse_device_from_manifest(device): |
| 204 | for project in iterate_manifests(True): |
| 205 | name = project.get('name') |
| 206 | if name.startswith("android_device_") and name.endswith(device): |
| 207 | return project.get('path') |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 208 | return None |
| 209 | |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 210 | |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 211 | def parse_device_from_folder(device): |
| 212 | search = [] |
| 213 | for sub_folder in os.listdir("device"): |
| 214 | if os.path.isdir("device/%s/%s" % (sub_folder, device)): |
| 215 | search.append("device/%s/%s" % (sub_folder, device)) |
| 216 | if len(search) > 1: |
| 217 | print("multiple devices under the name %s. " |
| 218 | "defaulting to checking the manifest" % device) |
| 219 | location = parse_device_from_manifest(device) |
| 220 | elif len(search) == 1: |
| 221 | location = search[0] |
Luca Stefani | 5c60e4f | 2017-08-17 19:28:48 +0200 | [diff] [blame] | 222 | else: |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 223 | print("your device can't be found in device sources..") |
| 224 | location = parse_device_from_manifest(device) |
| 225 | return location |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 226 | |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 227 | |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 228 | def parse_dependency_file(location): |
| 229 | dep_file = "bliss.dependencies" |
| 230 | dep_location = '/'.join([location, dep_file]) |
| 231 | if not os.path.isfile(dep_location): |
| 232 | print("WARNING: %s file not found" % dep_location) |
| 233 | sys.exit() |
| 234 | try: |
| 235 | with open(dep_location, 'r') as f: |
| 236 | dependencies = json.loads(f.read()) |
| 237 | except ValueError: |
| 238 | raise Exception("ERROR: malformed dependency file") |
| 239 | return dependencies |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 240 | |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 241 | |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 242 | def create_dependency_manifest(dependencies): |
| 243 | projects = [] |
| 244 | for dependency in dependencies: |
| 245 | repository = dependency.get("repository") |
| 246 | target_path = dependency.get("target_path") |
| 247 | revision = dependency.get("revision", default_rev) |
| 248 | remote = dependency.get("remote", default_rem) |
| 249 | |
| 250 | # not adding an organization should default to android_team |
| 251 | # only apply this to github |
| 252 | if remote == "github": |
| 253 | if not "/" in repository: |
| 254 | repository = '/'.join([android_team, repository]) |
| 255 | project = create_manifest_project(repository, |
| 256 | target_path, |
| 257 | remote=remote, |
| 258 | revision=revision) |
| 259 | if not project is None: |
| 260 | manifest = append_to_manifest(project) |
| 261 | write_to_manifest(manifest) |
| 262 | projects.append(target_path) |
| 263 | if len(projects) > 0: |
| 264 | os.system("repo sync --force-sync %s" % " ".join(projects)) |
| 265 | |
| 266 | |
| 267 | def fetch_dependencies(device): |
| 268 | location = parse_device_from_folder(device) |
| 269 | if location is None or not os.path.isdir(location): |
| 270 | raise Exception("ERROR: could not find your device " |
| 271 | "folder location, bailing out") |
| 272 | dependencies = parse_dependency_file(location) |
| 273 | create_dependency_manifest(dependencies) |
| 274 | |
| 275 | |
| 276 | def check_device_exists(device): |
| 277 | location = parse_device_from_folder(device) |
| 278 | if location is None: |
| 279 | return False |
| 280 | return os.path.isdir(location) |
| 281 | |
| 282 | |
| 283 | def fetch_device(device): |
| 284 | if check_device_exists(device): |
| 285 | print("WARNING: Trying to fetch a device that's already there") |
| 286 | return |
| 287 | git_data = search_github_for_device(device) |
| 288 | device_url = android_team+"/"+get_device_url(git_data) |
| 289 | device_dir = parse_device_directory(device_url,device) |
| 290 | project = create_manifest_project(device_url, |
| 291 | device_dir, |
| 292 | remote=default_team_rem) |
| 293 | if not project is None: |
| 294 | manifest = append_to_manifest(project) |
| 295 | write_to_manifest(manifest) |
| 296 | print("syncing the device config") |
| 297 | os.system('repo sync --force-sync %s' % device_dir) |
| 298 | |
| 299 | |
| 300 | if __name__ == '__main__': |
| 301 | if not os.path.isdir(local_manifest_dir): |
| 302 | os.mkdir(local_manifest_dir) |
| 303 | |
| 304 | product = sys.argv[1] |
| 305 | try: |
| 306 | device = product[product.index("_") + 1:] |
| 307 | except ValueError: |
| 308 | device = product |
| 309 | |
| 310 | if len(sys.argv) > 2: |
| 311 | deps_only = sys.argv[2] |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 312 | else: |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 313 | deps_only = False |
Michael Bestas | 3952f6c | 2016-08-26 01:12:08 +0300 | [diff] [blame] | 314 | |
Jackeagle | 881b772 | 2019-10-13 05:33:32 -0400 | [diff] [blame] | 315 | if not deps_only: |
| 316 | fetch_device(device) |
| 317 | fetch_dependencies(device) |