bin/release

   1 #!/usr/bin/python
   2
   3 import os, sys, argparse, re, git, http.client, subprocess
   4 import urlgrabber.progress, tarfile, shutil, gitdb, time, fnmatch
   5 from datetime import datetime
   6 from termcolor import colored
   7 from urllib.parse import urlparse
   8
   9 galette_dl_repo = 'http://download.tuxfamily.org/galette/'
  10 local_dl_repo = os.path.join(
  11     os.path.dirname(
  12         os.path.dirname(os.path.abspath(__file__))
  13     ),
  14     'dist'
  15 )
  16 verbose = False
  17 tagrefs = None
  18 force = False
  19 commit = None
  20 extra = None
  21 sign = True
  22 assume_yes = False
  23 nightly = False
  24 ssh_key = False
  25 nightly_version = None
  26
  27 def print_err(msg):
  28     """
  29     Display colored error message
  30     """
  31     print(colored(msg, 'red', attrs=['bold']))
  32
  33 def get_numeric_version(ver):
  34     """
  35     Returns all numeric version
  36     """
  37     return re.findall(r'\d+', ver)
  38
  39 def valid_version(ver):
  40     """
  41     Check if provided version is valid.
  42
  43     Takes all digits in passed version, then reassemble them with dots
  44     to check if it is the same as original given one.
  45     """
  46     return '.'.join(get_numeric_version(ver)) == ver
  47
  48 def incr_version(ver):
  49     """
  50     Increment version number
  51     """
  52     version = get_numeric_version(ver)
  53     version[-1] = str(int(version[-1]) + 1)
  54     return version
  55
  56 def propose_version():
  57     """
  58     Propose new minor and major versions,
  59     according to existing git tags
  60     """
  61     last_major = '0'
  62     last_minor = '0'
  63
  64     for tagref in tagrefs:
  65         tag = tagref.tag
  66         if valid_version(tag.tag):
  67             #last minor version is always the last one :)
  68             if tag.tag > last_minor:
  69                 last_minor = tag.tag
  70
  71             #last major version
  72             if len(tag.tag) == 5 and tag.tag > last_major:
  73                 last_major = tag.tag
  74
  75     if verbose:
  76         print('last minor: %s | last major %s' % (last_minor, last_major))
  77
  78     #no version provided. propose one
  79     new_minor = None
  80     new_major = None
  81
  82     if len(last_minor) == 5:
  83         #if the latest is a major version
  84         new_minor = last_minor + ('.1')
  85     else:
  86         new_minor = '.'.join(incr_version(last_minor))
  87
  88     new_major = '.'.join(incr_version(last_major))
  89
  90     print("""Proposed versions:
  91     minor: %s
  92     major: %s
  93     """ % (new_minor, new_major))
  94
  95 def get_latest_version():
  96     """
  97     Look for latest version
  98     """
  99     last = None
 100     for tagref in tagrefs:
 101         tag = tagref.tag
 102         if tag != None and valid_version(tag.tag):
 103             #last minor version is always the last one :)
 104             if last == None or tag.tag > last:
 105                 last = tag.tag
 106
 107     return last
 108
 109 def is_existing_version(ver):
 110     """
 111     Look specified version exists
 112     """
 113     for tagref in tagrefs:
 114         tag = tagref.tag
 115         if valid_version(tag.tag):
 116             if tag.tag == ver:
 117                 return True
 118     return False
 119
 120 def ask_user_confirm(msg):
 121     """
 122     Ask user his confirmation
 123     """
 124     if assume_yes:
 125         return True
 126     else:
 127         while True:
 128             sys.stdout.write(msg)
 129             choice = input().lower()
 130             if choice == 'y' or choice == 'yes':
 131                 return True
 132             elif choice == 'n' or choice == 'no':
 133                 return False
 134             else:
 135                 print_err(
 136                     "Invalid input. Please enter 'yes' or 'no' (or 'y' or 'n')."
 137                 )
 138
 139 def get_rel_name(buildver):
 140     """
 141     Build archive name from command line parameters
 142     That would be used for git archiving prefix and archive name
 143     """
 144     archive_name = None
 145
 146     if commit and extra:
 147         now = datetime.now()
 148         archive_name = 'galette-%s-%s-%s-%s' % (
 149             buildver,
 150             extra,
 151             now.strftime('%Y%m%d'),
 152             commit
 153         )
 154     elif nightly:
 155         archive_name = 'galette-dev'
 156     else:
 157         archive_name = 'galette-%s' % buildver
 158
 159     return archive_name
 160
 161 def _do_build(ver):
 162     """
 163     Proceed build
 164     """
 165     exists = False
 166     ascexists = False
 167     rel_name = get_rel_name(ver)
 168     archive_name = rel_name + '.tar.bz2'
 169     galette_archive = os.path.join(
 170         local_dl_repo,
 171         archive_name
 172     )
 173
 174     if not force:
 175         #first check if a version
 176         local = False
 177         ascLocal = False
 178
 179         url = galette_dl_repo + '/' + archive_name
 180         urlasc = '%s.asc' % url
 181         parsed = urlparse(url)
 182         ascparsed = urlparse(urlasc)
 183
 184         connection = http.client.HTTPConnection(parsed[1], 80)
 185         connection.request('HEAD', parsed[2])
 186         response = connection.getresponse()
 187         exists = response.status == 200
 188
 189         if not exists:
 190             #also check from local repo
 191             exists = os.path.exists(galette_archive)
 192             if exists:
 193                 local = True
 194
 195         connection = http.client.HTTPConnection(ascparsed[1], 80)
 196         connection.request('HEAD', ascparsed[2])
 197         response = connection.getresponse()
 198         ascexists = response.status == 200
 199
 200         if not ascexists:
 201             #also check from local repo
 202             ascexists = os.path.exists(
 203                 os.path.join(
 204                     local_dl_repo,
 205                     archive_name + '.asc'
 206                 )
 207             )
 208             if ascexists:
 209                 ascLocal = True
 210
 211     if exists or ascexists:
 212         msg = None
 213         if exists:
 214             loctxt = ''
 215             if local:
 216                 loctxt = 'locally '
 217             msg = 'Relase %s already %sexists' % (rel_name, loctxt)
 218
 219         if ascexists:
 220             loctxt = ''
 221             if ascLocal:
 222                 loctxt = ' locally'
 223             if msg is not None:
 224                 msg += ' and has been %ssigned!' % loctxt
 225             else:
 226                 msg += 'Release has been %ssigned!' % loctxt
 227
 228         msg += '\n\nYou will *NOT* build another one :)'
 229         print_err(msg)
 230     else:
 231         print('Building %s...' % rel_name)
 232
 233         archive_cmd_pattern = 'git archive --prefix=%s/ %s | bzip2 > %s'
 234         if commit and extra or nightly:
 235             archive_cmd = archive_cmd_pattern % (
 236                 rel_name,
 237                 commit,
 238                 galette_archive
 239             )
 240         else:
 241             archive_cmd = archive_cmd_pattern % (
 242                 rel_name,
 243                 ver,
 244                 galette_archive
 245             )
 246
 247         if verbose:
 248             typestr = 'Tag'
 249             typever = ver
 250
 251             if commit and extra:
 252                 typestr = 'Commit'
 253                 typever = commit
 254
 255             print('Release name: %s, %s: %s, Dest: %s' % (
 256                 rel_name,
 257                 typestr,
 258                 typever,
 259                 galette_archive
 260             ))
 261             print('Archive command: %s' % (archive_cmd))
 262
 263         if commit and extra:
 264             print('Archiving GIT commit %s' % commit)
 265         else:
 266             print('Archiving GIT tag %s' % ver)
 267
 268         p1 = subprocess.Popen(archive_cmd, shell=True)
 269         p1.communicate()
 270
 271         print('Adding vendor libraries')
 272         add_libs(rel_name, galette_archive)
 273
 274         if sign:
 275             do_sign(galette_archive)
 276
 277         upload = ask_user_confirm(
 278             'Do you want to upload archive %s? [yes/No] ' % galette_archive
 279         )
 280
 281         if upload:
 282             do_scp(galette_archive)
 283
 284 def do_sign(archive):
 285     sign_cmd = 'gpg --detach-sign --armor %s' % archive
 286     p1 = subprocess.Popen(sign_cmd, shell=True)
 287     p1.communicate()
 288
 289 def do_scp(archive):
 290     global ssh_key
 291
 292     path = 'galette/galette-repository/'
 293     if extra:
 294         path += 'dev/'
 295
 296     if ssh_key:
 297         scp_cmd = 'scp -i %s %s* ssh.tuxfamily.org:%s' % (ssh_key, archive, path)
 298     else:
 299         scp_cmd = 'scp -r %s* ssh.tuxfamily.org:%s' % (archive, path)
 300     print(scp_cmd)
 301     p1 = subprocess.Popen(scp_cmd, shell=True)
 302     p1.communicate()
 303
 304 def add_libs(rel_name, galette_archive):
 305     """
 306     Add external libraries to the archive
 307     Also write version for GALETTE_NIGHTLY value
 308     """
 309     galette = tarfile.open(galette_archive, 'r|bz2', format=tarfile.GNU_FORMAT)
 310     src_dir = os.path.join(local_dl_repo, 'src')
 311     if not os.path.exists(src_dir):
 312         os.makedirs(src_dir)
 313     galette.extractall(path=src_dir)
 314     galette.close()
 315
 316     #set galette nightly version
 317     config_dir = os.path.join(src_dir, rel_name, 'galette', 'config')
 318     if nightly_version != None:
 319         sed_cmd = 'sed -e "s/GALETTE_NIGHTLY\', false/GALETTE_NIGHTLY\', \'%s\'/" -i versions.inc.php' % nightly_version
 320         print(sed_cmd)
 321         p1 = subprocess.Popen(sed_cmd, shell=True, cwd=config_dir)
 322         p1.wait()
 323
 324     #install npm modules
 325     npm_dir = os.path.join(src_dir, rel_name)
 326     npm_cmd = 'npm install --prefix %s' % npm_dir
 327     print(npm_cmd)
 328     p1 = subprocess.Popen(npm_cmd, shell=True, cwd=npm_dir)
 329     p1.wait()
 330
 331     #build the UI assets
 332     wp_cmd = 'npm run first-build'
 333     p1 = subprocess.Popen(wp_cmd, shell=True, cwd=npm_dir)
 334     p1.wait()
 335
 336     #node modules are no longer needed
 337     shutil.rmtree(os.path.join(npm_dir, 'node_modules'))
 338
 339     #UI sources and some built assets are no longer needed
 340     shutil.rmtree(os.path.join(npm_dir, 'semantic'))
 341     shutil.rmtree(os.path.join(npm_dir, 'ui'))
 342     shutil.rmtree(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/components'))
 343     shutil.rmtree(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/themes/basic'))
 344     shutil.rmtree(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/themes/github'))
 345     shutil.rmtree(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/themes/material'))
 346     os.remove(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/semantic.js'))
 347     os.remove(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/semantic.css'))
 348     os.remove(os.path.join(src_dir, rel_name, 'galette/webroot/themes/default/ui/semantic.rtl.css'))
 349
 350     #install php dependencies
 351     composer_cmd = 'composer install --ignore-platform-reqs --no-dev'
 352     composer_dir = os.path.join(src_dir, rel_name, 'galette')
 353     print(composer_dir)
 354     p1 = subprocess.Popen(composer_cmd, shell=True, cwd=composer_dir)
 355     p1.wait()
 356
 357     #cleaunp files not required in releases
 358     todrop = [
 359         '.travis.yml',
 360         'apigen.neon',
 361         'gulpfile.js',
 362         'package.json',
 363         'package-lock.json',
 364         'phpcs-rules.xml',
 365         'composer.lock',
 366         '.composer-require-checker.config.json',
 367         'semantic.json',
 368         os.path.join('galette', 'composer.json'),
 369         os.path.join('galette', 'composer.lock'),
 370         os.path.join('galette', 'post_contribution_test.php')
 371     ]
 372     for td in todrop:
 373         if os.path.exists(os.path.join(src_dir, rel_name, td)):
 374             os.remove(os.path.join(src_dir, rel_name, td))
 375
 376     if os.path.exists(os.path.join(src_dir, rel_name, 'patches')):
 377         shutil.rmtree(os.path.join(src_dir, rel_name, 'patches'))
 378
 379     if os.path.exists(os.path.join(src_dir, rel_name, 'tests')):
 380         shutil.rmtree(os.path.join(src_dir, rel_name, 'tests'))
 381
 382     if os.path.exists(os.path.join(src_dir, rel_name, '.github')):
 383         shutil.rmtree(os.path.join(src_dir, rel_name, '.github'))
 384
 385     #cleanup vendors
 386     for root, dirnames, filenames in os.walk(os.path.join(composer_dir, 'vendor')):
 387         #remove git directories
 388         for dirname in fnmatch.filter(dirnames, '.git*'):
 389             remove_dir = os.path.join(composer_dir, root, dirname)
 390             shutil.rmtree(remove_dir)
 391         #remove test directories
 392         for dirname in fnmatch.filter(dirnames, 'test?'):
 393             remove_dir = os.path.join(composer_dir, root, dirname)
 394             shutil.rmtree(remove_dir)
 395         #remove examples directories
 396         for dirname in fnmatch.filter(dirnames, 'example?'):
 397             remove_dir = os.path.join(composer_dir, root, dirname)
 398             shutil.rmtree(remove_dir)
 399         #remove doc directories
 400         for dirname in fnmatch.filter(dirnames, 'doc?'):
 401             remove_dir = os.path.join(composer_dir, root, dirname)
 402             shutil.rmtree(remove_dir)
 403         #remove composer stuff
 404         for filename in fnmatch.filter(filenames, 'composer*'):
 405             remove_file = os.path.join(composer_dir, root, filename)
 406             os.remove(remove_file)
 407
 408         for dirname in dirnames:
 409             #remove Faker useless languages
 410             if root.endswith('src/Faker/Provider'):
 411                 if dirname not in ['en_US', 'fr_FR', 'de_DE']:
 412                     shutil.rmtree(os.path.join(root, dirname))
 413             #begin to remove tcpdf not used fonts
 414             if root.endswith('tcpdf/fonts'):
 415                 if dirname != 'dejavu-fonts-ttf-2.34':
 416                     shutil.rmtree(os.path.join(root, dirname))
 417             if root.endswith('vendor'):
 418                 if dirname == 'bin':
 419                     shutil.rmtree(os.path.join(root, dirname))
 420
 421         for filename in filenames:
 422             if os.path.islink(os.path.join(root, filename)):
 423                 os.remove(os.path.join(root, filename))
 424             #remove tcpdf not used fonts
 425             if root.endswith('tcpdf/fonts'):
 426                 if filename not in [
 427                         'dejavusansbi.ctg.z',
 428                         'dejavusansbi.z',
 429                         'dejavusansb.z',
 430                         'dejavusansi.ctg.z',
 431                         'dejavusansi.z',
 432                         'dejavusans.z',
 433                         'zapfdingbats.php',
 434                         'dejavusansb.ctg.z',
 435                         'dejavusansbi.php',
 436                         'dejavusansb.php',
 437                         'dejavusans.ctg.z',
 438                         'dejavusansi.php',
 439                         'dejavusans.php',
 440                         'helvetica.php'
 441                 ]:
 442                     os.remove(os.path.join(root, filename))
 443
 444     galette = tarfile.open(galette_archive, 'w|bz2', format=tarfile.GNU_FORMAT)
 445
 446     for i in os.listdir(src_dir):
 447         galette.add(
 448             os.path.join(src_dir, i),
 449             arcname=rel_name
 450         )
 451
 452     galette.close()
 453     shutil.rmtree(src_dir)
 454
 455 def valid_commit(repo, c):
 456     """
 457     Validate commit existance in repository
 458     Also prepare version for GALETTE_NIGHTLY value
 459     """
 460     global commit, nightly_version
 461
 462     try:
 463         dformat = '%a, %d %b %Y %H:%M'
 464         repo_commit = repo.commit(c)
 465
 466         commit = repo_commit.hexsha[:10]
 467
 468         nightly_version = '%s (%s)' % (
 469             commit,
 470             time.strftime('%Y-%m-%d %H:%M:%S GMT%z', time.localtime(repo_commit.committed_date)),
 471         )
 472
 473         print(colored("""Commit information:
 474         Hash:          %s
 475         Author:        %s
 476         Authored date: %s
 477         Commiter:      %s
 478         Commit date:   %s
 479         Message:       %s""" % (
 480             commit,
 481             repo_commit.author,
 482             time.strftime(dformat, time.gmtime(repo_commit.authored_date)),
 483             repo_commit.committer,
 484             time.strftime(dformat, time.gmtime(repo_commit.committed_date)),
 485             repo_commit.message
 486         ), None, 'on_grey', attrs=['bold']))
 487         return True
 488     except gitdb.exc.BadObject:
 489         return False
 490
 491 def main():
 492     """
 493     Main method
 494     """
 495     global verbose, tagrefs, force, extra, assume_yes, nightly, sign, ssh_key
 496
 497     parser = argparse.ArgumentParser(description='Release Galette')
 498     group = parser.add_mutually_exclusive_group()
 499     group.add_argument(
 500         '-v',
 501         '--version',
 502         help='Version to release'
 503     )
 504     group.add_argument(
 505         '-p',
 506         '--propose',
 507         help='Calculate and propose next possible versions',
 508         action='store_true'
 509     )
 510     parser.add_argument(
 511         '-c',
 512         '--commit',
 513         help='Specify commit to archive (-v required)'
 514     )
 515     parser.add_argument(
 516         '-e',
 517         '--extra',
 518         help='Extra version information (-c required)'
 519     )
 520     parser.add_argument(
 521         '-Y',
 522         '--assume-yes',
 523         help='Assume YES to all questions. Be sure to understand what you are doing!',
 524         action='store_true'
 525     )
 526     parser.add_argument(
 527         '-V',
 528         '--verbose',
 529         help='Be more verbose',
 530         action="store_true"
 531     )
 532     parser.add_argument(
 533         '-n',
 534         '--nightly',
 535         help='Build nightly',
 536         action="store_true"
 537     )
 538     parser.add_argument(
 539         '-k',
 540         '--ssh-key',
 541         help='SSH key to be used for uploading',
 542     )
 543     parser.add_argument('-f', action='store_true')
 544     args = parser.parse_args()
 545
 546     verbose=args.verbose
 547
 548     if verbose:
 549         print(args)
 550
 551     galette_repo = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 552     repo = git.Repo(galette_repo)
 553     tagrefs = repo.tags
 554
 555     if args.f == True:
 556         force = ask_user_confirm(
 557             'Are you *REALLY* sure you mean -f when you typed -f? [yes/No] '
 558         )
 559     assume_yes=args.assume_yes
 560
 561     if args.ssh_key:
 562         ssh_key = args.ssh_key
 563
 564     build = False
 565     buildver = None
 566     if args.nightly:
 567         nightly = True
 568         buildver = 'dev'
 569         args.commit = repo.commit('develop')
 570         if valid_commit(repo, args.commit):
 571             force = True
 572             build = True
 573             sign = False
 574             assume_yes = True
 575         else:
 576             print_err('Invalid commit ref %s' % args.commit)
 577     elif (args.extra or args.commit) and (not args.extra or not args.commit or not args.version):
 578         print_err('You have to specify --version --commit and --extra all together')
 579         sys.exit(1)
 580     elif args.commit and args.version and args.extra:
 581         if valid_commit(repo, args.commit):
 582             if verbose:
 583                 print('Commit is valid')
 584             build = True
 585             buildver = args.version
 586             extra = args.extra
 587         else:
 588             print_err('Invalid commit ref %s' % args.commit)
 589     elif args.version:
 590         if not valid_version(args.version):
 591             print_err('%s is not a valid version number!' % args.version)
 592             sys.exit(1)
 593         else:
 594             #check if specified version exists
 595             if not is_existing_version(args.version):
 596                 print_err('%s does not exist!' % args.version)
 597             else:
 598                 build = True
 599                 buildver = args.version
 600     elif args.propose:
 601         propose_version()
 602     else:
 603         buildver = get_latest_version()
 604         if force:
 605             build = True
 606         else:
 607             build = ask_user_confirm(
 608                 'Do you want to build Galette version %s? [Yes/no] ' % buildver
 609             )
 610
 611     if build:
 612         _do_build(buildver)
 613
 614 if __name__ == "__main__":
 615     main()