Merge pull request openedx-unsupported#1050 from edx/jarv/add-clone-db

A script to clone a database into a vpc using stack-name

util/vpc-tools/db-clone.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python
+import boto
+import boto.route53
+import boto.route53.record
+import boto.ec2.elb
+import boto.rds2
+import time
+from argparse import ArgumentParser, RawTextHelpFormatter
+import datetime
+import sys
+from vpcutil import rds_subnet_group_name_for_stack_name, all_stack_names
+
+description = """
+
+   Creates a new RDS instance in a VPC using restore
+   from point in time using the latest available backup.
+   The new db will be the same size as the original.
+   The name of the db will remain the same, the master db password
+   will be changed and is set on the command line.
+
+   New db name defaults to "from-<source db name>-<human date>-<ts>"
+   A new DNS entry will be created for the RDS when provided
+   on the command line
+
+"""
+
+RDS_SIZES = [
+    'db.m1.small',
+    'db.m1.large',
+    'db.m1.xlarge',
+    'db.m2.xlarge',
+    'db.m2.2xlarge',
+    'db.m2.4xlarg',
+]
+
+
+def parse_args(args=sys.argv[1:]):
+
+    stack_names = all_stack_names()
+    rds = boto.rds2.connect_to_region('us-east-1')
+    dbs = [db['DBInstanceIdentifier']
+           for db in rds.describe_db_instances()['DescribeDBInstancesResponse']['DescribeDBInstancesResult']['DBInstances']]
+
+    parser = ArgumentParser(description=description, formatter_class=RawTextHelpFormatter)
+    parser.add_argument('-s', '--stack-name', choices=stack_names,
+                        default='stage-edx',
+                        help='Stack name for where you want this RDS instance launched')
+    parser.add_argument('-t', '--type', choices=RDS_SIZES,
+                        default='db.m1.small', help='RDS size to create instances of')
+    parser.add_argument('-d', '--db-source', choices=dbs,
+                        default=u'stage-edx', help="source db to clone")
+    parser.add_argument('-p', '--password', required=True,
+                        help="password for the new database", metavar="NEW PASSWORD")
+    parser.add_argument('-r', '--region', default='us-east-1',
+                        help="region to connect to")
+    return parser.parse_args(args)
+
+
+def wait_on_db_status(db_name, region='us-east-1', wait_on='available', aws_id=None, aws_secret=None):
+    rds = boto.rds2.connect_to_region(region)
+    while True:
+        statuses = rds.describe_db_instances(db_name)['DescribeDBInstancesResponse']['DescribeDBInstancesResult']['DBInstances']
+        if len(statuses) > 1:
+            raise Exception("More than one instance returned for {0}".format(db_name))
+        if statuses[0]['DBInstanceStatus'] == wait_on:
+            break
+        sys.stdout.write(".")
+        sys.stdout.flush()
+        time.sleep(2)
+    return
+
+if __name__ == '__main__':
+    args = parse_args()
+
+    rds = boto.rds2.connect_to_region(args.region)
+    subnet_name = rds_subnet_group_name_for_stack_name(args.stack_name)
+    restore_dbid = 'from-{0}-{1}-{2}'.format(args.db_source, datetime.date.today(), int(time.time()))
+    rds.restore_db_instance_to_point_in_time(
+        source_db_instance_identifier=args.db_source,
+        target_db_instance_identifier=restore_dbid,
+        use_latest_restorable_time=True,
+        db_instance_class=args.type,
+        db_subnet_group_name=subnet_name)
+    wait_on_db_status(restore_dbid)

util/vpc-tools/vpcutil.py

@@ -1,21 +1,38 @@
 import boto
+import boto.rds2
+import boto.rds
+
+CFN_TAG_KEY = 'aws:cloudformation:stack-name'
 
 def vpc_for_stack_name(stack_name, aws_id=None, aws_secret=None):
     cfn = boto.connect_cloudformation(aws_id, aws_secret)
     resources = cfn.list_stack_resources(stack_name)
     for resource in resources:
-      if resource.resource_type == 'AWS::EC2::VPC':
-        return resource.physical_resource_id
+        if resource.resource_type == 'AWS::EC2::VPC':
+            return resource.physical_resource_id
+
 
-def stack_name_for_vpc(vpc_name):
-    cfn_tag_key = 'aws:cloudformation:stack-name'
+def stack_name_for_vpc(vpc_name, aws_id, aws_secret):
     vpc = boto.connect_vpc(aws_id, aws_secret)
     resource = vpc.get_all_vpcs(vpc_ids=[vpc_name])[0]
-    if cfn_tag_key in resource.tags:
-        return resource.tags[cfn_tag_key]
+    if CFN_TAG_KEY in resource.tags:
+        return resource.tags[CFN_TAG_KEY]
     else:
         msg = "VPC({}) is not part of a cloudformation stack.".format(vpc_name)
         raise Exception(msg)
 
 
+def rds_subnet_group_name_for_stack_name(stack_name, region='us-east-1', aws_id=None, aws_secret=None):
+    # Helper function to look up a subnet group name by stack name
+    rds = boto.rds2.connect_to_region(region)
+    vpc = vpc_for_stack_name(stack_name)
+    for group in rds.describe_db_subnet_groups()['DescribeDBSubnetGroupsResponse']['DescribeDBSubnetGroupsResult']['DBSubnetGroups']:
+        if group['VpcId'] == vpc:
+            return group['DBSubnetGroupName']
+    return None
+
 
+def all_stack_names(region='us-east-1', aws_id=None, aws_secret=None):
+    vpc_conn = boto.connect_vpc(aws_id, aws_secret)
+    return [vpc.tags[CFN_TAG_KEY] for vpc in vpc_conn.get_all_vpcs()
+            if CFN_TAG_KEY in vpc.tags.keys()]