ail-framework/bin/Update.py

#!/usr/bin/env python3
# -*-coding:UTF-8 -*

"""
Update AIL
============================

Update AIL clone and fork

"""

import configparser
import os
import sys
import argparse

import subprocess

sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
# TODO: move other functions
from packages import git_status


UPDATER_FILENAME = os.path.join(os.environ['AIL_BIN'], 'Update.py')

UPDATER_LAST_MODIFICATION = float(os.stat(UPDATER_FILENAME).st_mtime)

def auto_update_enabled(cfg):
    auto_update = cfg.get('Update', 'auto_update')
    if auto_update == 'True' or auto_update == 'true':
        return True
    else:
        return False

# check if files are modify locally
def check_if_files_modified():
    process = subprocess.run(['git', 'ls-files', '-m'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if process.returncode == 0:
        modified_files = process.stdout
        if modified_files:
            l_modified_files = []
            for modified_file in modified_files.decode().split('\n'):
                if modified_file:
                    if modified_file.split('/')[0] != 'configs':
                        l_modified_files.append(modified_file)
            if l_modified_files:
                print('Modified Files:')
                for modified_file in l_modified_files:
                    print('{}{}{}'.format(TERMINAL_BLUE, modified_file, TERMINAL_DEFAULT))
                print()
                return False
            else:
                return True
        else:
            return True
    else:
        print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
        sys.exit(1)

def repo_is_fork():
    print('Check if this repository is a fork:')
    process = subprocess.run(['git', 'remote', '-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

    if process.returncode == 0:
        res = process.stdout.decode()
        if 'origin	{}'.format(AIL_REPO) in res:
            print('    This repository is a {}clone of {}{}'.format(TERMINAL_BLUE, AIL_REPO, TERMINAL_DEFAULT))
            return False
        elif 'origin	{}'.format(OLD_AIL_REPO) in res:
            print('    old AIL repository, Updating remote origin...')
            res = git_status.set_default_remote(AIL_REPO, verbose=False)
            if res:
                return False
            else:
                return True
        else:
            print('    This repository is a {}fork{}'.format(TERMINAL_BLUE, TERMINAL_DEFAULT))
            print()
            return True
    else:
        print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
        aborting_update()
        sys.exit(0)

def is_upstream_created(upstream):
    process = subprocess.run(['git', 'remote', '-v'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if process.returncode == 0:
        output = process.stdout.decode()
        if upstream in output:
            return True
        else:
            return False
    else:
        print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
        aborting_update()
        sys.exit(0)

def create_fork_upstream(upstream):
    print('{}... Creating upstream ...{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
    print('git remote add {} {}'.format(upstream, AIL_REPO))
    process = subprocess.run(['git', 'remote', 'add', upstream, AIL_REPO],
                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if process.returncode == 0:
        print(process.stdout.decode())
        if is_upstream_created(upstream):
            print('Fork upstream created')
            print('{}...    ...{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
        else:
            print('Fork not created')
            aborting_update()
            sys.exit(0)
    else:
        print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
        aborting_update()
        sys.exit(0)

def update_fork():
    print('{}... Updating fork ...{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
    if cfg.get('Update', 'update-fork') == 'True' or cfg.get('Update', 'update-fork') == 'true':
        upstream = cfg.get('Update', 'upstream')
        if not is_upstream_created(upstream):
            create_fork_upstream(upstream)
        print('{}git fetch {}:{}'.format(TERMINAL_YELLOW, upstream, TERMINAL_DEFAULT))
        process = subprocess.run(['git', 'fetch', upstream], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        if process.returncode == 0:
            print(process.stdout.decode())
            print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
            process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
            if process.returncode == 0:
                print(process.stdout.decode())
                print('{}git merge {}/master:{}'.format(TERMINAL_YELLOW, upstream, TERMINAL_DEFAULT))
                process = subprocess.run(['git', 'merge', '{}/master'.format(upstream)],
                                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                if process.returncode == 0:
                    print(process.stdout.decode())
                    print('{}...    ...{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
                else:
                    print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
                    aborting_update()
                    sys.exit(1)
            else:
                print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
                aborting_update()
                sys.exit(0)
        else:
            print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
            aborting_update()
            sys.exit(0)

    else:
        print('{}Fork Auto-Update disabled in config file{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
        aborting_update()
        sys.exit(0)


def get_git_current_tag(path_current_version):
    try:
        with open(path_current_version, 'r') as version_content:
            version = version_content.read()
    except FileNotFoundError:
        version = 'v5.0' # TODO Replace with VERSION.json
        with open(path_current_version, 'w') as version_content:
            version_content.write(version)

    version = version.replace(" ", "").splitlines()[0]
    if version[0] != 'v':
        version = 'v{}'.format(version)
    return version

def get_git_upper_tags_remote(current_tag, is_fork):
    # keep only first dot
    nb_dot = current_tag.count('.')
    if nb_dot > 0:
        nb_dot = nb_dot - 1
    current_tag_val = current_tag.rsplit('.', nb_dot)
    current_tag_val = ''.join(current_tag_val)

    if is_fork:
        process = subprocess.run(['git', 'tag'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
        if process.returncode == 0:
            list_all_tags = process.stdout.decode().splitlines()

            list_upper_tags = []
            if list_all_tags[-1][1:] == current_tag:
                list_upper_tags.append((list_all_tags[-1], None))
                # force update order
                list_upper_tags.sort()
                return list_upper_tags
            for tag in list_all_tags:
                if float(tag[1:]) >= float(current_tag_val):
                    list_upper_tags.append((tag, None))
            # force update order
            list_upper_tags.sort()
            return list_upper_tags
        else:
            print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
            aborting_update()
            sys.exit(0)
    else:
        process = subprocess.run(['git', 'ls-remote', '--tags'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        if process.returncode == 0:
            list_all_tags = process.stdout.decode().splitlines()
            last_tag = list_all_tags[-1].split('\trefs/tags/')
            last_commit = last_tag[0]
            last_tag = last_tag[1].split('^{}')[0]
            list_upper_tags = []
            if last_tag[1:] == current_tag:
                list_upper_tags.append((last_tag, last_commit))
                # force update order
                list_upper_tags.sort()
                return list_upper_tags
            else:
                dict_tags_commit = {}
                for mess_tag in list_all_tags:
                    commit, tag = mess_tag.split('\trefs/tags/')

                    tag = tag.replace('^{}', '')
                    # remove 'v' version
                    tag = tag.replace('v', '')
                    # keep only first dot
                    nb_dot = tag.count('.')
                    if nb_dot > 0:
                        nb_dot = nb_dot - 1
                    tag_val = tag.rsplit('.', nb_dot)
                    tag_val = ''.join(tag_val)

                    # check if tag is float
                    try:
                        tag_val = float(tag_val)
                    except ValueError:
                        continue

                    if float(current_tag) < 4.2:
                        # add tag with last commit
                        if float(current_tag_val) <= float(tag_val) < float(4.2):
                            dict_tags_commit[tag] = commit
                    else:
                        # add tag with last commit
                        if float(tag_val) >= float(current_tag_val):
                            dict_tags_commit[tag] = commit
                list_upper_tags = [('v{}'.format(key), dict_tags_commit[key]) for key in dict_tags_commit]
                # force update order
                list_upper_tags.sort()
                return list_upper_tags
        else:
            print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
            aborting_update()
            sys.exit(0)

def update_submodules():
    print('{}git submodule update:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
    process = subprocess.run(['git', 'submodule', 'update'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if process.returncode == 0:
        print(process.stdout.decode())
        print()
    else:
        print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))

def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork):
    print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
    process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    # process = subprocess.run(['ls'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if process.returncode == 0:
        print(process.stdout.decode())
        print()

        update_submodules()

        print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
        process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

        if process.returncode == 0:
            output = process.stdout.decode()
            print(output)

            # CHECK IF UPDATER Update
            if float(os.stat(UPDATER_FILENAME).st_mtime) > UPDATER_LAST_MODIFICATION:
                # request updater relaunch
                print(f'{TERMINAL_RED}                  Relaunch Launcher                    {TERMINAL_DEFAULT}')
                sys.exit(3)

            if len(list_upper_tags_remote) == 1:
                # additional update (between 2 commits on the same version)
                additional_update_path = os.path.join(os.environ['AIL_HOME'], 'update', current_tag, 'additional_update.sh')
                if os.path.isfile(additional_update_path):
                    print()
                    print(f'{TERMINAL_YELLOW}------------------------------------------------------------------')
                    print('-                 Launching Additional Update:                   -')
                    print(f'--  --  --  --  --  --  --  --  --  --  --  --  --  --  --  --  --{TERMINAL_DEFAULT}')
                    process = subprocess.run(['bash', additional_update_path],
                                             stdout=subprocess.PIPE, stderr=subprocess.PIPE)
                    if process.returncode == 0:
                        output = process.stdout.decode()
                        print(output)
                    else:
                        print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
                        aborting_update()
                        sys.exit(1)

                print()
                print(f'{TERMINAL_YELLOW}****************  AIL Successfully Updated  *****************{TERMINAL_DEFAULT}')
                print()
                exit(0)

            else:
                # map version with roll back commit
                list_update = []
                previous_commit = list_upper_tags_remote[0][1]
                for row_tuple in list_upper_tags_remote[1:]:
                    tag = row_tuple[0]
                    list_update.append((tag, previous_commit))
                    previous_commit = row_tuple[1]

                for update in list_update:
                    launch_update_version(update[0], update[1], current_version_path, is_fork)
                # Success
                print(f'{TERMINAL_YELLOW}****************  AIL Successfully Updated  *****************{TERMINAL_DEFAULT}')
                print()
                sys.exit(0)
        else:
            print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
            aborting_update()
            sys.exit(1)
    else:
        print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
        aborting_update()
        sys.exit(0)

def launch_update_version(version, roll_back_commit, current_version_path, is_fork):
    update_path = os.path.join(os.environ['AIL_HOME'], 'update', str(version), 'Update.sh')
    print()
    print(f'{TERMINAL_YELLOW}------------------------------------------------------------------')
    print(f'-                 Launching Update: {TERMINAL_BLUE}{version}{TERMINAL_YELLOW}                         -')
    print(f'--  --  --  --  --  --  --  --  --  --  --  --  --  --  --  --  --{TERMINAL_DEFAULT}')
    if not os.path.isfile(update_path):
        update_path = os.path.join(os.environ['AIL_HOME'], 'update', 'default_update', 'Update.sh')
        process = subprocess.Popen(['bash', update_path, version], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    else:
        process = subprocess.Popen(['bash', update_path], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    while True:
        output = process.stdout.readline().decode()
        if output == '' and process.poll() is not None:
            break
        if output:
            print(output.strip())
    if process.returncode == 0:
        # output = process.stdout.decode()
        # print(output)

        with open(current_version_path, 'w') as version_content:
            version_content.write(version)

            print('{}--  --  --  --  --  --  --  --  --  --  --  --  --  --  --  --  --'.format(TERMINAL_YELLOW))
            print('-               Successfully Updated: {}{}{}                        -'.format(TERMINAL_BLUE, version, TERMINAL_YELLOW))
            print('------------------------------------------------------------------{}'.format(TERMINAL_DEFAULT))
            print()
    else:
        # print(process.stdout.read().decode())
        print('{}{}{}'.format(TERMINAL_RED, process.stderr.read().decode(), TERMINAL_DEFAULT))
        print('------------------------------------------------------------------')
        print('                   {}Update Error: {}{}{}'.format(TERMINAL_RED, TERMINAL_BLUE, version, TERMINAL_DEFAULT))
        print('------------------------------------------------------------------')
        if not is_fork:
            roll_back_update(roll_back_commit)
        else:
            aborting_update()
            sys.exit(1)

def roll_back_update(roll_back_commit):
    print('Rolling back to safe commit: {}{}{}'.format(TERMINAL_BLUE, roll_back_commit, TERMINAL_DEFAULT))
    process = subprocess.run(['git', 'checkout', roll_back_commit], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if process.returncode == 0:
        output = process.stdout
        print(output)
        sys.exit(0)
    else:
        print(TERMINAL_RED+process.stderr.decode()+TERMINAL_DEFAULT)
        aborting_update()
        sys.exit(1)

def aborting_update():
    print()
    print('{}Aborting ...{}'.format(TERMINAL_RED, TERMINAL_DEFAULT))
    print('{}******************************************************************'.format(TERMINAL_RED))
    print('*                    AIL Not Updated                             *')
    print('******************************************************************{}'.format(TERMINAL_DEFAULT))
    print()


if __name__ == "__main__":

    TERMINAL_RED = '\033[91m'
    TERMINAL_YELLOW = '\33[93m'
    TERMINAL_BLUE = '\33[94m'
    TERMINAL_BLINK = '\33[6m'
    TERMINAL_DEFAULT = '\033[0m'

    AIL_REPO = 'https://github.com/ail-project/ail-framework'
    OLD_AIL_REPO = 'https://github.com/CIRCL/AIL-framework.git'

    configfile = os.path.join(os.environ['AIL_HOME'], 'configs/update.cfg')
    if not os.path.exists(configfile):
        raise Exception('Unable to find the configuration file. \
                        Did you set environment variables? \
                        Or activate the virtualenv.')
    cfg = configparser.ConfigParser()
    cfg.read(configfile)

    current_version_path = os.path.join(os.environ['AIL_HOME'], 'update/current_version')

    print('{}******************************************************************'.format(TERMINAL_YELLOW))
    print('*                        Updating AIL ...                        *')
    print('******************************************************************{}'.format(TERMINAL_DEFAULT))

    # manual updates
    parser = argparse.ArgumentParser()
    parser.add_argument("--manual", nargs='?', const=True, default=False)
    args = parser.parse_args()
    manual_update = args.manual

    if auto_update_enabled(cfg) or manual_update:
        if check_if_files_modified():
            is_fork = repo_is_fork()
            if is_fork:
                update_fork()

            current_tag = get_git_current_tag(current_version_path)
            print()
            print('Current Version: {}{}{}'.format(TERMINAL_YELLOW, current_tag, TERMINAL_DEFAULT))
            print()
            list_upper_tags_remote = get_git_upper_tags_remote(current_tag.replace('v', ''), is_fork)
            # new release
            if len(list_upper_tags_remote) > 1:
                print('New Releases:')
            if is_fork:
                for upper_tag in list_upper_tags_remote:
                    print('    {}{}{}'.format(TERMINAL_BLUE, upper_tag[0], TERMINAL_DEFAULT))
            else:
                for upper_tag in list_upper_tags_remote:
                    print('    {}{}{}: {}'.format(TERMINAL_BLUE, upper_tag[0], TERMINAL_DEFAULT, upper_tag[1]))
            print()
            update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork)

        else:
            print('Please, commit your changes or stash them before you can update AIL')
            aborting_update()
            sys.exit(0)
    else:
        print('               {}AIL Auto update is disabled{}'.format(TERMINAL_RED, TERMINAL_DEFAULT))
        aborting_update()
        sys.exit(0)


    # r = get_git_upper_tags_remote('4.2.1', False)
    # print(r)