Almost done overhauling step3 script

release_scripts/README.txt

@@ -217,7 +217,7 @@ def getMasterPackageList():
    m[pkg]['OSArchLink']   = '32'
    m[pkg]['HasBundle']    = True
    m[pkg]['BundleDeps']   = 'offline_deps_ubuntu32'
-   m[pkg]['BundleSuffix'] = 'offline_ubuntu_12.0432.tar.gz'
+   m[pkg]['BundleSuffix'] = 'offline_ubuntu_12.04-32.tar.gz'
    m[pkg]['BundleOSVar']  = '12.04'
 
 
@@ -233,7 +233,7 @@ def getMasterPackageList():
    m[pkg]['OSArchLink']   = '64'
    m[pkg]['HasBundle']    = True
    m[pkg]['BundleDeps']   = 'offline_deps_ubuntu64'
-   m[pkg]['BundleSuffix'] = 'offline_ubuntu64.tar.gz'
+   m[pkg]['BundleSuffix'] = 'offline_ubuntu_12.04-64.tar.gz'
    m[pkg]['BundleOSVar']  = '12.04'
 
    pkg = 'RaspberryPi'

release_scripts/Step1_Online_PrepareForSigning.py

@@ -4,7 +4,6 @@
 import time
 import shutil
 import ast
-import textwrap
 from sys import argv
 from release_utils import *
 
@@ -17,6 +16,7 @@
 CLONE_URL = 'https://github.com/etotheipi/BitcoinArmory.git'
 
 if len(argv)<3:
+   import textwrap
    print textwrap.dedent("""
       Script Arguments (* is optional)
             argv[0]   "python %s"
@@ -51,7 +51,7 @@
 # Throw in the Bitcoin Core hashes file if supplied
 if shaCore is not None:
    checkExists(shaCore)
-   shutil.copy(shaCore, outDir)
+   shutil.copy(shaCore, os.path.join(outDir,'SHA256SUMS.asc'))
 
 # Grab the list of scp and cp commands from fetchlist
 for pkgName,pkgInfo in masterPkgList.iteritems():

release_scripts/Step2_Offline_PackageSigning.py

@@ -14,10 +14,9 @@
 from release_settings import getReleaseParams, getMasterPackageList
 #####
 
-masterPkgList = getMasterPackageList()
-
 
 if len(argv)<6:
+   import textwrap
    print textwrap.dedent("""
       Script Arguments (* is optional)
          argv[0]   "python %s"
@@ -35,20 +34,22 @@
 bundleDir  = argv[3]
 gitBranch  = 'master' if len(argv)<4 else argv[4]
 testParams = (len(argv)>5 and not argv[5]=="0")
-
 outDir = makeOutputDir(outDir, wipe=False)
 
 
+masterPkgList = getMasterPackageList()
 RELEASE = getReleaseParams(testParams)
 
+
 # Other defaults -- same for all Armory releases
 builder        = RELEASE['Builder']
 gituser        = RELEASE['GitUser']
 gitemail       = RELEASE['GitEmail']
 signAddress    = RELEASE['SignAddr']
 announceName   = RELEASE['AnnounceFile']
-bucketAnnounce = RELEASE['BucketAnnounce']
-bucketReleases = RELEASE['BucketReleases']
+bucketPrefix   = RELEASE['BucketPrefix']
+bucketAnnounce = bucketPrefix + RELEASE['BucketAnnounce']
+bucketReleases = bucketPrefix + RELEASE['BucketReleases']
 gpgKeyID       = RELEASE['GPGKeyID']
 btcWltID       = RELEASE['BTCWltID']
 
@@ -64,7 +65,7 @@ def logprint(s):
 srcGitRepo  = checkExists(os.path.join(inDir, 'BitcoinArmory'))
 srcInstalls = checkExists(os.path.join(inDir, 'installers'))
 srcAnnounce = checkExists(os.path.join(inDir, 'unsignedannounce'))
-srcCoreSHA  = checkExists(os.path.join(inDir, 'SHASUMS256.asc'), 'skip')
+srcCoreSHA  = checkExists(os.path.join(inDir, 'SHA256SUMS.asc'), 'skip')
 
 # Check that all the paths expected from step 1 actually exist
 dstGitRepo  =               os.path.join(outDir, 'BitcoinArmory')
@@ -73,7 +74,7 @@ def logprint(s):
 
 # Scan the list of files in installers dir to get latest 
 instList = [fn for fn in os.listdir(srcInstalls)]
-topVerInt,topVerStr,topVerType = getLatestVerFromList2(instList)
+topVerInt,topVerStr,topVerType = getLatestVerFromList(instList)
 
 # A shortcut to get the full path of the installer filename for a given pkg
 def getSrcPath(pkgName, suffixStr='FileSuffix'):
@@ -82,7 +83,13 @@ def getSrcPath(pkgName, suffixStr='FileSuffix'):
    return os.path.join(srcInstalls, fname)
 
 def getDstPath(pkgName, suffixStr='FileSuffix'):
-   pkgSuffix = masterPkgList[pkgName][suffixStr]
+   if pkgName=='SHAFILE_TXT':
+      pkgSuffix = 'sha256sum.txt'
+   elif pkgName=='SHAFILE_ASC':
+      pkgSuffix = 'sha256sum.txt.asc'
+   else:
+      pkgSuffix = masterPkgList[pkgName][suffixStr]
+
    fname = 'armory_%s%s_%s' % (topVerStr, topVerType, pkgSuffix)
    return os.path.join(dstInstalls, fname)
 
@@ -118,7 +125,7 @@ def getDstPath(pkgName, suffixStr='FileSuffix'):
    logprint('\n'*2)
    logprint('*'*80)
    logprint('Output of gpg-verify on SHA256SUMS')
-   logprint(execAndWait('gpg -v %s'%srcCoreSHA))
+   logprint(execAndWait('gpg -v %s' % srcCoreSHA)[0])
    logprint('*'*80)
    logprint('Contents of SHA256SUM.asc:')
    logprint(open(srcCoreSHA,'r').read())
@@ -190,7 +197,8 @@ def getDstPath(pkgName, suffixStr='FileSuffix'):
 
 
 newHashes = getAllHashes(filesToSign)
-hashname = 'armory_%s%s_sha256sum.txt' % (topVerStr, topVerType)
+#hashname = 'armory_%s%s_sha256sum.txt' % (topVerStr, topVerType)
+hashname = getDstPath('SHAFILE_TXT')
 hashpath = os.path.join(dstInstalls, hashname)
 with open(hashpath, 'w') as hashfile:
    for fn,sha2 in newHashes:

release_scripts/Step3_Online_VerifyAndUpload.py

@@ -5,76 +5,82 @@
 #   dpkg-sig --verify *.deb
 #   gpg -v *.asc
 #   cd BitcoinArmory; git tag -v v0.90-beta  (or whatever the tag is)
-import sys
+from sys import argv
 import os
 import time
 import shutil
 from release_utils import *
 
+#####
+from release_settings import getReleaseParams, getMasterPackageList
+#####
+
+if len(argv)<2:
+   import textwrap
+   print textwrap.dedent("""
+      Script Arguments (* is optional)
+            argv[0]   "python %s"
+            argv[1]   inputDir       (from Step2)
+            argv[2]*  isDryRun       (default ~ 0)
+            argv[3]*  useTestParams  (default ~ 0)
+      """) % argv[0]
+   exit(1)
+
+
+# Parse CLI args
+inDir      = checkExists(argv[1])
+isDryRun   = (len(argv)>2 and not argv[2]=='0')
+testParams = (len(argv)>3 and not argv[3]=='0')
+
+
+masterPkgList = getMasterPackageList()
+RELEASE = getReleaseParams(testParams)
+
+signAddress    = RELEASE['SignAddr']
+announceName   = RELEASE['AnnounceFile']
+bucketPrefix   = RELEASE['BucketPrefix']
+htmlRelease    = bucketPrefix + RELEASE['BucketAnnounce']
+htmlAnnounce   = bucketPrefix + RELEASE['BucketReleases']
+s3Release      = 's3://%s' % RELEASE['BucketAnnounce']
+s3Announce     = 's3://%s' % RELEASE['BucketReleases']
+gpgKeyID       = RELEASE['GPGKeyID']
+btcWltID       = RELEASE['BTCWltID']
 
-startDir = os.getcwd()
-latestVerInt, latestVerStr, latestRelease = 0,'',''
-unpackDir = 'signed_release_unpack'
-bucket = 'bitcoinarmory-releases'
-bucketS3 = 's3://%s' % bucket
-bucketdl = 'https://s3.amazonaws.com/%s' % bucket
 
 #uploadlog = open('step3_log_%d.txt' % long(time.time()), 'w')
 uploadlog = open('step3_log.txt', 'w')
 def logprint(s):
    print s
    uploadlog.write(s + '\n')
 
-# Find the latest signed_release archive 
-for fn in os.listdir('.'):
-   if not fn.startswith('signed_release'):
-      continue 
+srcGitRepo  = checkExists(os.path.join(inDir, 'BitcoinArmory'))
+srcInstalls = checkExists(os.path.join(inDir, 'installers'))
+srcAnnounce = checkExists(os.path.join(inDir, 'signedannounce'))
 
-   fivevals = parseInstallerName2(fn, ignoreExt=True)
+# Scan the list of files in installers dir to get latest 
+instList = [fn for fn in os.listdir(srcInstalls)]
+topVerInt,topVerStr,topVerType = getLatestVerFromList(instList)
 
-   if fivevals==None:
-      continue
 
-   vs,vi,vt = fivevals[1:4]
-   if vi>latestVerInt:
-      latestVerInt = vi
-      latestVerStr = vs
-      latestVerType = vt
-      latestRelease = fn
-
-
-# Get the version type "testing", "beta", etc
-locver  = latestRelease.index(latestVerStr)
-lenver  = len(latestVerStr)
-locdot  = latestRelease.index('.', locver+lenver+1)
-verFullStr = latestVerStr + latestVerType
-
-
-logprint('')
-logprint('*'*80)
-logprint('Detected Release Parameters:')
-logprint('   Version type: ' + latestVerType)
-logprint('   Full version: ' + verFullStr)
-logprint('   Release file: ' + latestRelease)
-logprint('   S3 Bucket   : ' + bucketS3)
-logprint('   DL Links    : ' + bucketdl)
-logprint('')
+def getPkgFilename(pkgName, offline=False):
+   if pkgName=='SHAFILE_TXT':
+      suffix = 'sha256sum.txt'
+   elif pkgName=='SHAFILE_ASC':
+      suffix = 'sha256sum.txt.asc'
+   else:
+      suffixType = 'BundleSuffix' if offline else 'FileSuffix'
+      suffix = masterPkgList[pkgName][suffixType]
 
-################################################################################
-# Now unpack and start doing our thing
-if os.path.exists(unpackDir):
-   logprint('Removing previous unpack-directory: %s' % unpackDir)
-   shutil.rmtree(unpackDir)
+   return 'armory_%s%s_%s' % (topVerStr, topVerType, suffix)
 
-os.mkdir(unpackDir)
-execAndWait('tar -zxf %s -C %s' % (latestRelease, unpackDir))
 
 # Create [relpath, filename, isBundle, isHashes, [pentuple]] 
+"""
 uploads = []
 ascfile = ''
 for fn in os.listdir(unpackDir):
    fullfn = os.path.join(unpackDir, fn)
-   fivevals = parseInstallerName2(fullfn, ignoreExt=True)
+   fivevals = parseInstallerName(fullfn, ignoreExt=True)
    if fivevals==None or os.path.isdir(fullfn):
       continue
 
@@ -144,50 +150,100 @@ def logprint(s):
 pkgMap['ubuntu32'] = ['Ubuntu', '12.04+', '(32bit)' ]
 pkgMap['ubuntu64'] = ['Ubuntu', '12.04+', '(64bit)' ]
 
+"""
 
-uploads.sort(key=lambda x: x[1])
+#uploads.sort(key=lambda x: x[1])
 
 # Now actually execute the uploads and make them public
 forumTextList = []
 htmlTextList  = []
 rawUrlList    = []
 s3cmdList     = []
-for fullfn, fn, isbundle, ishash, fivevals in uploads:
+
+
+uploads = []
+for pkgName,pkgInfo in masterPkgList.iteritems():
+   pkgDict = {}
+   pkgDict['SrcFile']   = getPkgFilename(pkgName)
+   pkgDict['SrcPath']   = os.path.join(inDir, pkgDict['SrcFile'])
+   pkgDict['OSName']    = pkgInfo['OSNameDisp']
+   pkgDict['OSVar']     = pkgInfo['OSVarDisp']
+   pkgDict['OSArch']    = pkgInfo['OSArchDisp']
+   pkgDict['IsHash']    = False
+   pkgDict['IsBundle']  = False
+   pkgDict['DstUpload'] = '%s/%s' % (s3Release,   pkgDict['SrcFile'])
+   pkgDict['DstHtml']   = '%s/%s' % (htmlRelease, pkgDict['SrcFile'])
+   uploads.append(pkgDict)
+
+   if pkgInfo['HasBundle']:
+      pkgDict = {}
+      pkgDict['SrcFile']   = getPkgFilename(pkgName, offline=True)
+      pkgDict['SrcPath']   = os.path.join(inDir, pkgDict['SrcFile'])
+      pkgDict['OSName']    = pkgInfo['OSNameDisp']
+      pkgDict['OSVar']     = pkgInfo['BundleOSVar']
+      pkgDict['OSArch']    = pkgInfo['OSArchDisp']
+      pkgDict['IsHash']    = False
+      pkgDict['IsBundle']  = True
+      pkgDict['DstUpload'] = '%s/%s' % (s3Release,   pkgDict['SrcFile'])
+      pkgDict['DstHtml']   = '%s/%s' % (htmlRelease, pkgDict['SrcFile'])
+      uploads.append(pkgDict)
+
+
+ascDict = {}
+ascDict['SrcFile']   = getPkgFilename('SHAFILE_ASC')
+ascDict['SrcPath']   = os.path.join(inDir, ascDict['SrcFile'])
+ascDict['IsHash']    = True
+pkgDict['DstUpload'] = '%s%s' % (s3Release,   pkgDict['SrcFile'])
+pkgDict['DstHtml']   = '%s%s' % (htmlRelease, pkgDict['SrcFile'])
+uploads.append(ascDict)
+
+
+for upl in uploads:
+   print 'Going to upload:'
+   for key,val in upl.iteritems():
+      print '   ', key.ljust(10), ':', val
+   print ''
+
+
+exit(0)
+
+for pkgDict in uploads:
 
    #osStr, subOS, bits, vi, vs = fivevals
    #print fullfn, fn, isbundle, ishash, osStr, subOS, bits, vi, vs
-
-   osName,verStr,verInt,verType,ext = fivevals[:]
-   humanOS,humanVer,humanBits = pkgMap[ext]
+   verFullStr = topVerStr + topVerType
 
    humanText = 'Armory %s' % verFullStr
-   if isbundle:
+   if pkgDict['IsBundle']:
       humanText += ' Offline Bundle'
 
-   if ishash: 
+   if pkgDict['IsHash']: 
       humanText += ': Signed hashes of all installers '
    else:
-      humanText += ' for %s %s %s' % tuple(pkgMap[ext])
+      osParams = [pkgDict[a] for a in ['OSName', 'OSVar', 'OSArch']]
+      humanText += ' for %s %s %s' % tuple(osParams)
 
-   uploadurl = '%s/%s' % (bucketS3, fn)
-   linkurl = '%s/%s' % (bucketdl, fn)
+   uploadurl = pkgDict['DstUpload']
+   linkurl   = pkgDict['DstHtml']
 
-   s3cmd = 's3cmd put --acl-public %s %s' % (fullfn, uploadurl)
-   #s3cmd = 's3cmd put %s %s' % (fullfn, uploadurl)
+   s3cmd = 's3cmd put --acl-public %s %s' % (pkgDict['SrcPath'], uploadurl)
    forumText = '[url=%s]%s[/url]' % (linkurl, humanText)
    htmlText  = '<a href="%s">%s</a>' % (linkurl, humanText)
 
    forumTextList.append(forumText)
    htmlTextList.append(htmlText)
    rawUrlList.append(linkurl)
    s3cmdList.append(s3cmd)
-
 
-for ann in announceFiles:
-   uploadurl = '%s/%s' % (bucketS3, fn)
-   s3cmd = 's3cmd put --acl-public %s %s' % (ann, uploadurl)
-   s3cmdList.append(s3cmd)
 
+announceDir = os.path.join(inDir, 'signedannounce')
+for fn in os.listdir(announceDir):
+   fpath = os.path.join(announceDir, fn)
+   uploadurl = '%s/%s' % (s3Announce, fn)
+   s3cmd = 's3cmd put --acl-public %s %s' % (fpath, uploadurl)
+   s3cmdList.append(s3cmd)
+
+
 logprint('\nRAW URL LIST')
 for txt in rawUrlList:
    logprint('  '+txt)