#!/usr/bin/python import os, sys, argparse, re, git, http.client, subprocess import urlgrabber.progress, tarfile, shutil, gitdb, time, fnmatch from datetime import datetime from termcolor import colored from urllib.parse import urlparse galette_dl_repo = 'http://download.tuxfamily.org/galette/' local_dl_repo = os.path.join( os.path.dirname( os.path.dirname(os.path.abspath(__file__)) ), 'dist' ) verbose = False tagrefs = None force = False commit = None extra = None sign = True assume_yes = False nightly = False ssh_key = False nightly_version = None def print_err(msg): """ Display colored error message """ print(colored(msg, 'red', attrs=['bold'])) def get_numeric_version(ver): """ Returns all numeric version """ return re.findall(r'\d+', ver) def valid_version(ver): """ Check if provided version is valid. Takes all digits in passed version, then reassemble them with dots to check if it is the same as original given one. """ return '.'.join(get_numeric_version(ver)) == ver def incr_version(ver): """ Increment version number """ version = get_numeric_version(ver) version[-1] = str(int(version[-1]) + 1) return version def propose_version(): """ Propose new minor and major versions, according to existing git tags """ last_major = '0' last_minor = '0' for tagref in tagrefs: tag = tagref.tag if valid_version(tag.tag): #last minor version is always the last one :) if tag.tag > last_minor: last_minor = tag.tag #last major version if len(tag.tag) == 5 and tag.tag > last_major: last_major = tag.tag if verbose: print('last minor: %s | last major %s' % (last_minor, last_major)) #no version provided. propose one new_minor = None new_major = None if len(last_minor) == 5: #if the latest is a major version new_minor = last_minor + ('.1') else: new_minor = '.'.join(incr_version(last_minor)) new_major = '.'.join(incr_version(last_major)) print("""Proposed versions: minor: %s major: %s """ % (new_minor, new_major)) def get_latest_version(): """ Look for latest version """ last = None for tagref in tagrefs: tag = tagref.tag if tag != None and valid_version(tag.tag): #last minor version is always the last one :) if last == None or tag.tag > last: last = tag.tag return last def is_existing_version(ver): """ Look specified version exists """ for tagref in tagrefs: tag = tagref.tag if valid_version(tag.tag): if tag.tag == ver: return True return False def ask_user_confirm(msg): """ Ask user his confirmation """ if assume_yes: return True else: while True: sys.stdout.write(msg) choice = input().lower() if choice == 'y' or choice == 'yes': return True elif choice == 'n' or choice == 'no': return False else: print_err( "Invalid input. Please enter 'yes' or 'no' (or 'y' or 'n')." ) def get_rel_name(buildver): """ Build archive name from command line parameters That would be used for git archiving prefix and archive name """ archive_name = None if commit and extra: now = datetime.now() archive_name = 'galette-%s-%s-%s-%s' % ( buildver, extra, now.strftime('%Y%m%d'), commit ) elif nightly: archive_name = 'galette-dev' else: archive_name = 'galette-%s' % buildver return archive_name def _do_build(ver): """ Proceed build """ exists = False ascexists = False rel_name = get_rel_name(ver) archive_name = rel_name + '.tar.bz2' galette_archive = os.path.join( local_dl_repo, archive_name ) if not force: #first check if a version local = False ascLocal = False url = galette_dl_repo + '/' + archive_name urlasc = '%s.asc' % url parsed = urlparse(url) ascparsed = urlparse(urlasc) connection = http.client.HTTPConnection(parsed[1], 80) connection.request('HEAD', parsed[2]) response = connection.getresponse() exists = response.status == 200 if not exists: #also check from local repo exists = os.path.exists(galette_archive) if exists: local = True connection = http.client.HTTPConnection(ascparsed[1], 80) connection.request('HEAD', ascparsed[2]) response = connection.getresponse() ascexists = response.status == 200 if not ascexists: #also check from local repo ascexists = os.path.exists( os.path.join( local_dl_repo, archive_name + '.asc' ) ) if ascexists: ascLocal = True if exists or ascexists: msg = None if exists: loctxt = '' if local: loctxt = 'locally ' msg = 'Relase %s already %sexists' % (rel_name, loctxt) if ascexists: loctxt = '' if ascLocal: loctxt = ' locally' if msg is not None: msg += ' and has been %ssigned!' % loctxt else: msg += 'Release has been %ssigned!' % loctxt msg += '\n\nYou will *NOT* build another one :)' print_err(msg) else: print('Building %s...' % rel_name) archive_cmd_pattern = 'git archive --prefix=%s/ %s | bzip2 > %s' if commit and extra or nightly: archive_cmd = archive_cmd_pattern % ( rel_name, commit, galette_archive ) else: archive_cmd = archive_cmd_pattern % ( rel_name, ver, galette_archive ) if verbose: typestr = 'Tag' typever = ver if commit and extra: typestr = 'Commit' typever = commit print('Release name: %s, %s: %s, Dest: %s' % ( rel_name, typestr, typever, galette_archive )) print('Archive command: %s' % (archive_cmd)) if commit and extra: print('Archiving GIT commit %s' % commit) else: print('Archiving GIT tag %s' % ver) p1 = subprocess.Popen(archive_cmd, shell=True) p1.communicate() print('Adding vendor libraries') add_libs(rel_name, galette_archive) if sign: do_sign(galette_archive) upload = ask_user_confirm( 'Do you want to upload archive %s? [yes/No] ' % galette_archive ) if upload: do_scp(galette_archive) def do_sign(archive): sign_cmd = 'gpg --detach-sign --armor %s' % archive p1 = subprocess.Popen(sign_cmd, shell=True) p1.communicate() def do_scp(archive): global ssh_key path = 'galette/galette-repository/' if extra: path += 'dev/' if ssh_key: scp_cmd = 'scp -i %s %s* ssh.tuxfamily.org:%s' % (ssh_key, archive, path) else: scp_cmd = 'scp -r %s* ssh.tuxfamily.org:%s' % (archive, path) print(scp_cmd) p1 = subprocess.Popen(scp_cmd, shell=True) p1.communicate() def add_libs(rel_name, galette_archive): """ Add external libraries to the archive Also write version for GALETTE_NIGHTLY value """ galette = tarfile.open(galette_archive, 'r|bz2', format=tarfile.GNU_FORMAT) src_dir = os.path.join(local_dl_repo, 'src') if not os.path.exists(src_dir): os.makedirs(src_dir) galette.extractall(path=src_dir) galette.close() #set galette nightly version config_dir = os.path.join(src_dir, rel_name, 'galette', 'config') if nightly_version != None: sed_cmd = 'sed -e "s/GALETTE_NIGHTLY\', false/GALETTE_NIGHTLY\', \'%s\'/" -i versions.inc.php' % nightly_version print(sed_cmd) p1 = subprocess.Popen(sed_cmd, shell=True, cwd=config_dir) p1.wait() #install npm modules npm_dir = os.path.join(src_dir, rel_name) npm_cmd = 'npm install --prefix %s' % npm_dir print(npm_cmd) p1 = subprocess.Popen(npm_cmd, shell=True, cwd=npm_dir) p1.wait() #build the UI assets wp_cmd = 'npm run first-build' p1 = subprocess.Popen(wp_cmd, shell=True, cwd=npm_dir) p1.wait() #node modules are no longer needed shutil.rmtree(os.path.join(npm_dir, 'node_modules')) #UI sources and some built assets are no longer needed shutil.rmtree(os.path.join(npm_dir, 'semantic')) shutil.rmtree(os.path.join(npm_dir, 'ui')) shutil.rmtree(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/components')) shutil.rmtree(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/themes/basic')) shutil.rmtree(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/themes/github')) shutil.rmtree(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/themes/material')) os.remove(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/semantic.js')) os.remove(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/semantic.css')) os.remove(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/semantic.rtl.css')) #install php dependencies composer_cmd = 'composer install --ignore-platform-reqs --no-dev' composer_dir = os.path.join(src_dir, rel_name, 'galette') print(composer_dir) p1 = subprocess.Popen(composer_cmd, shell=True, cwd=composer_dir) p1.wait() #cleaunp files not required in releases todrop = [ '.travis.yml', 'apigen.neon', 'gulpfile.js', 'package.json', 'package-lock.json', 'phpcs-rules.xml', 'composer.lock', '.composer-require-checker.config.json', 'semantic.json', os.path.join('galette', 'composer.json'), os.path.join('galette', 'composer.lock'), os.path.join('galette', 'post_contribution_test.php') ] for td in todrop: if os.path.exists(os.path.join(src_dir, rel_name, td)): os.remove(os.path.join(src_dir, rel_name, td)) if os.path.exists(os.path.join(src_dir, rel_name, 'patches')): shutil.rmtree(os.path.join(src_dir, rel_name, 'patches')) if os.path.exists(os.path.join(src_dir, rel_name, 'tests')): shutil.rmtree(os.path.join(src_dir, rel_name, 'tests')) if os.path.exists(os.path.join(src_dir, rel_name, '.github')): shutil.rmtree(os.path.join(src_dir, rel_name, '.github')) #cleanup vendors for root, dirnames, filenames in os.walk(os.path.join(composer_dir, 'vendor')): #remove git directories for dirname in fnmatch.filter(dirnames, '.git*'): remove_dir = os.path.join(composer_dir, root, dirname) shutil.rmtree(remove_dir) #remove test directories for dirname in fnmatch.filter(dirnames, 'test?'): remove_dir = os.path.join(composer_dir, root, dirname) shutil.rmtree(remove_dir) #remove examples directories for dirname in fnmatch.filter(dirnames, 'example?'): remove_dir = os.path.join(composer_dir, root, dirname) shutil.rmtree(remove_dir) #remove doc directories for dirname in fnmatch.filter(dirnames, 'doc?'): remove_dir = os.path.join(composer_dir, root, dirname) shutil.rmtree(remove_dir) #remove composer stuff for filename in fnmatch.filter(filenames, 'composer*'): remove_file = os.path.join(composer_dir, root, filename) os.remove(remove_file) for dirname in dirnames: #remove Faker useless languages if root.endswith('src/Faker/Provider'): if dirname not in ['en_US', 'fr_FR', 'de_DE']: shutil.rmtree(os.path.join(root, dirname)) #begin to remove tcpdf not used fonts if root.endswith('tcpdf/fonts'): if dirname != 'dejavu-fonts-ttf-2.34': shutil.rmtree(os.path.join(root, dirname)) if root.endswith('vendor'): if dirname == 'bin': shutil.rmtree(os.path.join(root, dirname)) for filename in filenames: if os.path.islink(os.path.join(root, filename)): os.remove(os.path.join(root, filename)) #remove tcpdf not used fonts if root.endswith('tcpdf/fonts'): if filename not in [ 'dejavusansbi.ctg.z', 'dejavusansbi.z', 'dejavusansb.z', 'dejavusansi.ctg.z', 'dejavusansi.z', 'dejavusans.z', 'zapfdingbats.php', 'dejavusansb.ctg.z', 'dejavusansbi.php', 'dejavusansb.php', 'dejavusans.ctg.z', 'dejavusansi.php', 'dejavusans.php', 'helvetica.php' ]: os.remove(os.path.join(root, filename)) galette = tarfile.open(galette_archive, 'w|bz2', format=tarfile.GNU_FORMAT) for i in os.listdir(src_dir): galette.add( os.path.join(src_dir, i), arcname=rel_name ) galette.close() shutil.rmtree(src_dir) def valid_commit(repo, c): """ Validate commit existance in repository Also prepare version for GALETTE_NIGHTLY value """ global commit, nightly_version try: dformat = '%a, %d %b %Y %H:%M' repo_commit = repo.commit(c) commit = repo_commit.hexsha[:10] nightly_version = '%s (%s)' % ( commit, time.strftime('%Y-%m-%d %H:%M:%S GMT%z', time.localtime(repo_commit.committed_date)), ) print(colored("""Commit information: Hash: %s Author: %s Authored date: %s Commiter: %s Commit date: %s Message: %s""" % ( commit, repo_commit.author, time.strftime(dformat, time.gmtime(repo_commit.authored_date)), repo_commit.committer, time.strftime(dformat, time.gmtime(repo_commit.committed_date)), repo_commit.message ), None, 'on_grey', attrs=['bold'])) return True except gitdb.exc.BadObject: return False def main(): """ Main method """ global verbose, tagrefs, force, extra, assume_yes, nightly, sign, ssh_key parser = argparse.ArgumentParser(description='Release Galette') group = parser.add_mutually_exclusive_group() group.add_argument( '-v', '--version', help='Version to release' ) group.add_argument( '-p', '--propose', help='Calculate and propose next possible versions', action='store_true' ) parser.add_argument( '-c', '--commit', help='Specify commit to archive (-v required)' ) parser.add_argument( '-e', '--extra', help='Extra version information (-c required)' ) parser.add_argument( '-Y', '--assume-yes', help='Assume YES to all questions. Be sure to understand what you are doing!', action='store_true' ) parser.add_argument( '-V', '--verbose', help='Be more verbose', action="store_true" ) parser.add_argument( '-n', '--nightly', help='Build nightly', action="store_true" ) parser.add_argument( '-k', '--ssh-key', help='SSH key to be used for uploading', ) parser.add_argument('-f', action='store_true') args = parser.parse_args() verbose=args.verbose if verbose: print(args) galette_repo = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) repo = git.Repo(galette_repo) tagrefs = repo.tags if args.f == True: force = ask_user_confirm( 'Are you *REALLY* sure you mean -f when you typed -f? [yes/No] ' ) assume_yes=args.assume_yes if args.ssh_key: ssh_key = args.ssh_key build = False buildver = None if args.nightly: nightly = True buildver = 'dev' args.commit = repo.commit('develop') if valid_commit(repo, args.commit): force = True build = True sign = False assume_yes = True else: print_err('Invalid commit ref %s' % args.commit) elif (args.extra or args.commit) and (not args.extra or not args.commit or not args.version): print_err('You have to specify --version --commit and --extra all together') sys.exit(1) elif args.commit and args.version and args.extra: if valid_commit(repo, args.commit): if verbose: print('Commit is valid') build = True buildver = args.version extra = args.extra else: print_err('Invalid commit ref %s' % args.commit) elif args.version: if not valid_version(args.version): print_err('%s is not a valid version number!' % args.version) sys.exit(1) else: #check if specified version exists if not is_existing_version(args.version): print_err('%s does not exist!' % args.version) else: build = True buildver = args.version elif args.propose: propose_version() else: buildver = get_latest_version() if force: build = True else: build = ask_user_confirm( 'Do you want to build Galette version %s? [Yes/no] ' % buildver ) if build: _do_build(buildver) if __name__ == "__main__": main()