build/tools/repopick.py

#!/usr/bin/env python
#
# Copyright (C) 2013-15 The CyanogenMod Project
#           (C) 2017    The LineageOS Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

#
# Run repopick.py -h for a description of this utility.
#

from __future__ import print_function

import sys
import json
import os
import subprocess
import re
import argparse
import textwrap
from functools import cmp_to_key
from xml.etree import ElementTree

try:
    import requests
except ImportError:
    try:
        # For python3
        import urllib.error
        import urllib.request
    except ImportError:
        # For python2
        import imp
        import urllib2
        urllib = imp.new_module('urllib')
        urllib.error = urllib2
        urllib.request = urllib2


# cmp() is not available in Python 3, define it manually
# See https://docs.python.org/3.0/whatsnew/3.0.html#ordering-comparisons
def cmp(a, b):
    return (a > b) - (a < b)


# Verifies whether pathA is a subdirectory (or the same) as pathB
def is_subdir(a, b):
    a = os.path.realpath(a) + '/'
    b = os.path.realpath(b) + '/'
    return b == a[:len(b)]


def fetch_query_via_ssh(remote_url, query):
    """Given a remote_url and a query, return the list of changes that fit it
       This function is slightly messy - the ssh api does not return data in the same structure as the HTTP REST API
       We have to get the data, then transform it to match what we're expecting from the HTTP RESET API"""
    if remote_url.count(':') == 2:
        (uri, userhost, port) = remote_url.split(':')
        userhost = userhost[2:]
    elif remote_url.count(':') == 1:
        (uri, userhost) = remote_url.split(':')
        userhost = userhost[2:]
        port = 29418
    else:
        raise Exception('Malformed URI: Expecting ssh://[user@]host[:port]')


    out = subprocess.check_output(['ssh', '-x', '-p{0}'.format(port), userhost, 'gerrit', 'query', '--format=JSON --patch-sets --current-patch-set', query])
    if not hasattr(out, 'encode'):
        out = out.decode()
    reviews = []
    for line in out.split('\n'):
        try:
            data = json.loads(line)
            # make our data look like the http rest api data
            review = {
                'branch': data['branch'],
                'change_id': data['id'],
                'current_revision': data['currentPatchSet']['revision'],
                'number': int(data['number']),
                'revisions': {patch_set['revision']: {
                    '_number': int(patch_set['number']),
                    'fetch': {
                        'ssh': {
                            'ref': patch_set['ref'],
                            'url': 'ssh://{0}:{1}/{2}'.format(userhost, port, data['project'])
                        }
                    },
                    'commit': {
                        'parents': [{ 'commit': parent } for parent in patch_set['parents']]
                    },
                } for patch_set in data['patchSets']},
                'subject': data['subject'],
                'project': data['project'],
                'status': data['status']
            }
            reviews.append(review)
        except:
            pass
    args.quiet or print('Found {0} reviews'.format(len(reviews)))
    return reviews


def fetch_query_via_http(remote_url, query):
    if "requests" in sys.modules:
        auth = None
        if os.path.isfile(os.getenv("HOME") + "/.gerritrc"):
            f = open(os.getenv("HOME") + "/.gerritrc", "r")
            for line in f:
                parts = line.rstrip().split("|")
                if parts[0] in remote_url:
                    auth = requests.auth.HTTPBasicAuth(username=parts[1], password=parts[2])
        statusCode = '-1'
        if auth:
            url = '{0}/a/changes/?q={1}&o=CURRENT_REVISION&o=ALL_REVISIONS&o=ALL_COMMITS'.format(remote_url, query)
            data = requests.get(url, auth=auth)
            statusCode = str(data.status_code)
        if statusCode != '200':
            #They didn't get good authorization or data, Let's try the old way
            url = '{0}/changes/?q={1}&o=CURRENT_REVISION&o=ALL_REVISIONS&o=ALL_COMMITS'.format(remote_url, query)
            data = requests.get(url)
        reviews = json.loads(data.text[5:])
    else:
        """Given a query, fetch the change numbers via http"""
        url = '{0}/changes/?q={1}&o=CURRENT_REVISION&o=ALL_REVISIONS&o=ALL_COMMITS'.format(remote_url, query)
        data = urllib.request.urlopen(url).read().decode('utf-8')
        reviews = json.loads(data[5:])

    for review in reviews:
        review['number'] = review.pop('_number')

    return reviews


def fetch_query(remote_url, query):
    """Wrapper for fetch_query_via_proto functions"""
    if remote_url[0:3] == 'ssh':
        return fetch_query_via_ssh(remote_url, query)
    elif remote_url[0:4] == 'http':
        return fetch_query_via_http(remote_url, query.replace(' ', '+'))
    else:
        raise Exception('Gerrit URL should be in the form http[s]://hostname/ or ssh://[user@]host[:port]')

if __name__ == '__main__':
    # Default to LineageOS Gerrit
    default_gerrit = 'https://review.lineageos.org'

    parser = argparse.ArgumentParser(formatter_class=argparse.RawDescriptionHelpFormatter, description=textwrap.dedent('''\
        repopick.py is a utility to simplify the process of cherry picking
        patches from LineageOS's Gerrit instance (or any gerrit instance of your choosing)

        Given a list of change numbers, repopick will cd into the project path
        and cherry pick the latest patch available.

        With the --start-branch argument, the user can specify that a branch
        should be created before cherry picking. This is useful for
        cherry-picking many patches into a common branch which can be easily
        abandoned later (good for testing other's changes.)

        The --abandon-first argument, when used in conjunction with the
        --start-branch option, will cause repopick to abandon the specified
        branch in all repos first before performing any cherry picks.'''))
    parser.add_argument('change_number', nargs='*', help='change number to cherry pick.  Use {change number}/{patchset number} to get a specific revision.')
    parser.add_argument('-i', '--ignore-missing', action='store_true', help='do not error out if a patch applies to a missing directory')
    parser.add_argument('-s', '--start-branch', nargs=1, help='start the specified branch before cherry picking')
    parser.add_argument('-r', '--reset', action='store_true', help='reset to initial state (abort cherry-pick) if there is a conflict')
    parser.add_argument('-a', '--abandon-first', action='store_true', help='before cherry picking, abandon the branch specified in --start-branch')
    parser.add_argument('-b', '--auto-branch', action='store_true', help='shortcut to "--start-branch auto --abandon-first --ignore-missing"')
    parser.add_argument('-q', '--quiet', action='store_true', help='print as little as possible')
    parser.add_argument('-v', '--verbose', action='store_true', help='print extra information to aid in debug')
    parser.add_argument('-f', '--force', action='store_true', help='force cherry pick even if change is closed')
    parser.add_argument('-p', '--pull', action='store_true', help='execute pull instead of cherry-pick')
    parser.add_argument('-P', '--path', help='use the specified path for the change')
    parser.add_argument('-t', '--topic', help='pick all commits from a specified topic')
    parser.add_argument('-Q', '--query', help='pick all commits using the specified query')
    parser.add_argument('-g', '--gerrit', default=default_gerrit, help='Gerrit Instance to use. Form proto://[user@]host[:port]')
    parser.add_argument('-e', '--exclude', nargs=1, help='exclude a list of commit numbers separated by a ,')
    parser.add_argument('-c', '--check-picked', type=int, default=10, help='pass the amount of commits to check for already picked changes')
    args = parser.parse_args()
    if not args.start_branch and args.abandon_first:
        parser.error('if --abandon-first is set, you must also give the branch name with --start-branch')
    if args.auto_branch:
        args.abandon_first = True
        args.ignore_missing = True
        if not args.start_branch:
            args.start_branch = ['auto']
    if args.quiet and args.verbose:
        parser.error('--quiet and --verbose cannot be specified together')

    if (1 << bool(args.change_number) << bool(args.topic) << bool(args.query)) != 2:
        parser.error('One (and only one) of change_number, topic, and query are allowed')

    # Change current directory to the top of the tree
    if 'ANDROID_BUILD_TOP' in os.environ:
        top = os.environ['ANDROID_BUILD_TOP']

        if not is_subdir(os.getcwd(), top):
            sys.stderr.write('ERROR: You must run this tool from within $ANDROID_BUILD_TOP!\n')
            sys.exit(1)
        os.chdir(os.environ['ANDROID_BUILD_TOP'])

    # Sanity check that we are being run from the top level of the tree
    if not os.path.isdir('.repo'):
        sys.stderr.write('ERROR: No .repo directory found. Please run this from the top of your tree.\n')
        sys.exit(1)

    # If --abandon-first is given, abandon the branch before starting
    if args.abandon_first:
        # Determine if the branch already exists; skip the abandon if it does not
        plist = subprocess.check_output(['repo', 'info'])
        if not hasattr(plist, 'encode'):
            plist = plist.decode()
        needs_abandon = False
        for pline in plist.splitlines():
            matchObj = re.match(r'Local Branches.*\[(.*)\]', pline)
            if matchObj:
                local_branches = re.split('\s*,\s*', matchObj.group(1))
                if any(args.start_branch[0] in s for s in local_branches):
                    needs_abandon = True

        if needs_abandon:
            # Perform the abandon only if the branch already exists
            if not args.quiet:
                print('Abandoning branch: %s' % args.start_branch[0])
            subprocess.check_output(['repo', 'abandon', args.start_branch[0]])
            if not args.quiet:
                print('')

    # Get the master manifest from repo
    #   - convert project name and revision to a path
    project_name_to_data = {}
    manifest = subprocess.check_output(['repo', 'manifest'])
    xml_root = ElementTree.fromstring(manifest)
    projects = xml_root.findall('project')
    remotes = xml_root.findall('remote')
    default_revision = xml_root.findall('default')[0].get('revision')

    #dump project data into the a list of dicts with the following data:
    #{project: {path, revision}}

    for project in projects:
        name = project.get('name')
        # when name and path are equal, "repo manifest" doesn't return a path at all, so fall back to name
        path = project.get('path', name)
        revision = project.get('revision')
        if revision is None:
            for remote in remotes:
                if remote.get('name') == project.get('remote'):
                    revision = remote.get('revision')
            if revision is None:
                revision = default_revision

        if not name in project_name_to_data:
            project_name_to_data[name] = {}
        revision = revision.split('refs/heads/')[-1]
        project_name_to_data[name][revision] = path

    # get data on requested changes
    reviews = []
    change_numbers = []

    def cmp_reviews(review_a, review_b):
        current_a = review_a['current_revision']
        parents_a = [r['commit'] for r in review_a['revisions'][current_a]['commit']['parents']]
        current_b = review_b['current_revision']
        parents_b = [r['commit'] for r in review_b['revisions'][current_b]['commit']['parents']]
        if current_a in parents_b:
            return -1
        elif current_b in parents_a:
            return 1
        else:
            return cmp(review_a['number'], review_b['number'])

    if args.topic:
        reviews = fetch_query(args.gerrit, 'topic:{0}'.format(args.topic))
        change_numbers = [str(r['number']) for r in sorted(reviews, key=cmp_to_key(cmp_reviews))]
    if args.query:
        reviews = fetch_query(args.gerrit, args.query)
        change_numbers = [str(r['number']) for r in sorted(reviews, key=cmp_to_key(cmp_reviews))]
    if args.change_number:
        change_url_re = re.compile('https?://.+?/([0-9]+(?:/[0-9]+)?)/?')
        for c in args.change_number:
            change_number = change_url_re.findall(c)
            if change_number:
                change_numbers.extend(change_number)
            elif '-' in c:
                templist = c.split('-')
                for i in range(int(templist[0]), int(templist[1]) + 1):
                    change_numbers.append(str(i))
            else:
                change_numbers.append(c)
        reviews = fetch_query(args.gerrit, ' OR '.join('change:{0}'.format(x.split('/')[0]) for x in change_numbers))

    # make list of things to actually merge
    mergables = []

    # If --exclude is given, create the list of commits to ignore
    exclude = []
    if args.exclude:
        exclude = args.exclude[0].split(',')

    for change in change_numbers:
        patchset = None
        if '/' in change:
            (change, patchset) = change.split('/')

        if change in exclude:
            continue

        change = int(change)

        if patchset:
            patchset = int(patchset)

        review = next((x for x in reviews if x['number'] == change), None)
        if review is None:
            print('Change %d not found, skipping' % change)
            continue

        mergables.append({
            'subject': review['subject'],
            'project': review['project'],
            'branch': review['branch'],
            'change_id': review['change_id'],
            'change_number': review['number'],
            'status': review['status'],
            'fetch': None,
            'patchset': review['revisions'][review['current_revision']]['_number'],
        })

        mergables[-1]['fetch'] = review['revisions'][review['current_revision']]['fetch']
        mergables[-1]['id'] = change
        if patchset:
            try:
                mergables[-1]['fetch'] = [review['revisions'][x]['fetch'] for x in review['revisions'] if review['revisions'][x]['_number'] == patchset][0]
                mergables[-1]['id'] = '{0}/{1}'.format(change, patchset)
                mergables[-1]['patchset'] = patchset
            except (IndexError, ValueError):
                args.quiet or print('ERROR: The patch set {0}/{1} could not be found, using CURRENT_REVISION instead.'.format(change, patchset))

    for item in mergables:
        args.quiet or print('Applying change number {0}...'.format(item['id']))
        # Check if change is open and exit if it's not, unless -f is specified
        if (item['status'] != 'OPEN' and item['status'] != 'NEW' and item['status'] != 'DRAFT') and not args.query:
            if args.force:
                print('!! Force-picking a closed change !!\n')
            else:
                print('Change status is ' + item['status'] + '. Skipping the cherry pick.\nUse -f to force this pick.')
                continue

        # Convert the project name to a project path
        #   - check that the project path exists
        project_path = None

        if item['project'] in project_name_to_data and item['branch'] in project_name_to_data[item['project']]:
            project_path = project_name_to_data[item['project']][item['branch']]
        elif args.path:
            project_path = args.path
        elif item['project'] in project_name_to_data and len(project_name_to_data[item['project']]) == 1:
            local_branch = list(project_name_to_data[item['project']])[0]
            project_path = project_name_to_data[item['project']][local_branch]
            print('WARNING: Project {0} has a different branch ("{1}" != "{2}")'.format(project_path, local_branch, item['branch']))
        elif args.ignore_missing:
            print('WARNING: Skipping {0} since there is no project directory for: {1}\n'.format(item['id'], item['project']))
            continue
        else:
            sys.stderr.write('ERROR: For {0}, could not determine the project path for project {1}\n'.format(item['id'], item['project']))
            sys.exit(1)

        # If --start-branch is given, create the branch (more than once per path is okay; repo ignores gracefully)
        if args.start_branch:
            subprocess.check_output(['repo', 'start', args.start_branch[0], project_path])

        # Determine the maximum commits to check already picked changes
        check_picked_count = args.check_picked
        branch_commits_count = int(subprocess.check_output(['git', 'rev-list', '--count', 'HEAD'], cwd=project_path))
        if branch_commits_count <= check_picked_count:
            check_picked_count = branch_commits_count - 1

        # Check if change is already picked to HEAD...HEAD~check_picked_count
        found_change = False
        for i in range(0, check_picked_count):
            if subprocess.call(['git', 'cat-file', '-e', 'HEAD~{0}'.format(i)], cwd=project_path, stderr=open(os.devnull, 'wb')):
                continue
            output = subprocess.check_output(['git', 'show', '-q', 'HEAD~{0}'.format(i)], cwd=project_path)
            # make sure we have a string on Python 3
            if isinstance(output, bytes):
                output = output.decode('utf-8')
            output = output.split()
            if 'Change-Id:' in output:
                head_change_id = ''
                for j,t in enumerate(reversed(output)):
                    if t == 'Change-Id:':
                        head_change_id = output[len(output) - j]
                        break
                if head_change_id.strip() == item['change_id']:
                    print('Skipping {0} - already picked in {1} as HEAD~{2}'.format(item['id'], project_path, i))
                    found_change = True
                    break
        if found_change:
            continue

        # Print out some useful info
        if not args.quiet:
            print(u'--> Subject:       "{0}"'.format(item['subject']))
            print('--> Project path:  {0}'.format(project_path))
            print('--> Change number: {0} (Patch Set {1})'.format(item['id'], item['patchset']))

        if 'anonymous http' in item['fetch']:
            method = 'anonymous http'
        else:
            method = 'ssh'

        # Try fetching from GitHub first if using default gerrit
        if args.gerrit == default_gerrit:
            if args.verbose:
                print('Trying to fetch the change from GitHub')

            if args.pull:
                cmd = ['git pull --no-edit github', item['fetch'][method]['ref']]
            else:
                cmd = ['git fetch github', item['fetch'][method]['ref']]
            if args.quiet:
                cmd.append('--quiet')
            else:
                print(cmd)
            result = subprocess.call([' '.join(cmd)], cwd=project_path, shell=True)
            FETCH_HEAD = '{0}/.git/FETCH_HEAD'.format(project_path)
            if result != 0 and os.stat(FETCH_HEAD).st_size != 0:
                print('ERROR: git command failed')
                sys.exit(result)
        # Check if it worked
        if args.gerrit != default_gerrit or os.stat(FETCH_HEAD).st_size == 0:
            # If not using the default gerrit or github failed, fetch from gerrit.
            if args.verbose:
                if args.gerrit == default_gerrit:
                    print('Fetching from GitHub didn\'t work, trying to fetch the change from Gerrit')
                else:
                    print('Fetching from {0}'.format(args.gerrit))

            if args.pull:
                cmd = ['git pull --no-edit', item['fetch'][method]['url'], item['fetch'][method]['ref']]
            else:
                cmd = ['git fetch', item['fetch'][method]['url'], item['fetch'][method]['ref']]
            if args.quiet:
                cmd.append('--quiet')
            else:
                print(cmd)
            result = subprocess.call([' '.join(cmd)], cwd=project_path, shell=True)
            if result != 0:
                print('ERROR: git command failed')
                sys.exit(result)
        # Perform the cherry-pick
        if not args.pull:
            cmd = ['git cherry-pick --ff FETCH_HEAD']
            if args.quiet:
                cmd_out = open(os.devnull, 'wb')
            else:
                cmd_out = None
            result = subprocess.call(cmd, cwd=project_path, shell=True, stdout=cmd_out, stderr=cmd_out)
            if result != 0:
                cmd = ['git diff-index --quiet HEAD --']
                result = subprocess.call(cmd, cwd=project_path, shell=True, stdout=cmd_out, stderr=cmd_out)
                if result == 0:
                    print('WARNING: git command resulted with an empty commit, aborting cherry-pick')
                    cmd = ['git cherry-pick --abort']
                    subprocess.call(cmd, cwd=project_path, shell=True, stdout=cmd_out, stderr=cmd_out)
                elif args.reset:
                    print('ERROR: git command failed, aborting cherry-pick')
                    cmd = ['git cherry-pick --abort']
                    subprocess.call(cmd, cwd=project_path, shell=True, stdout=cmd_out, stderr=cmd_out)
                    sys.exit(result)
                else:
                    print('ERROR: git command failed')
                    sys.exit(result)
        if not args.quiet:
            print('')