diff --git a/README.md b/README.md index 60ef7ea..3dcf9e7 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,16 @@ your security groups that are properly tagged will be updated accordingly. For more information on ip-ranges.json, read the documentation on [AWS IP Address Ranges](http://docs.aws.amazon.com/general/latest/gr/aws-ip-ranges.html). +## amazon-cloudfront-staging-to-production + +This is a python command line script that replicates staging distribution to production. + +You can define environment variables separately and pipeline your changes. + +## cloudfront-traffic-csv + +This script pulls out CloudWatch metric of Amazon CloudFront distributions into comma separated format. + *** Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved. diff --git a/amazon-cloudfront-staging-to-production/README.md b/amazon-cloudfront-staging-to-production/README.md index 8f3df57..b4b9f11 100644 --- a/amazon-cloudfront-staging-to-production/README.md +++ b/amazon-cloudfront-staging-to-production/README.md @@ -1,10 +1,44 @@ # cloudfront-staging-to-production -Replicate one Amazon CloudFront Distribution config to another. -Major use case for this is creating a staging CF Distribution, test/experiment features and push that to production Distribution. -based on python3 and boto3. +This is a command line tool for replicating one CloudFront distribution to new or existing one. +It is recommended to have a production and staging environment separated as a best practice, and you can use this script to apply changes of your staging CF distribution to production. -Usage: python3 cloudfront-staging-to-production.py [destination(production) CF Distribution ID] +## Setup +You need to have a JSON file that defines environment variables, all of them are optional but highly recommended to have: + - Aliases: Host name for production domain. *Please note that CloudFront you cannot use same Alias to more than one CloudFront distribution and API call will fail.* + - Comment: This is shown when you go to CloudFront console + - Logging: Define S3 bucket and prefix + - ViewerCertificate: ACM certificate arn -Regarding the settings.json: -example1.json shows changing only Alt name and access log setting. -example2.json shows different TLS cert between staging and production distribution. +*example:* +```yaml +{ + "Aliases": {"Quantity": 1, "Items": ["www.example.com"]}, + "Comment": "production distribution for www.example.com", + "Logging": { + "Enabled": true, + "IncludeCookies": false, + "Bucket": "loggingbucket.s3.amazonaws.com", + "Prefix": "examplecom-log" }, + "ViewerCertificate": { + "ACMCertificateArn": "", + "SSLSupportMethod": "sni-only", + "MinimumProtocolVersion": "TLSv1.2_2018", + "Certificate": "", + "CertificateSource": "acm" + } +} +``` + +#### Usage +``` +python3 cloudfront-staging-to-production.py [destination(production) CF distribution ID] +``` +*example: to create new CF distribution from E300AABBCCDDEE* +``` +python3 cloudfront-staging-to-production.py www.json E300AABBCCDDEE +``` +*example: update existing CF distribution E499ZZXXYYDDEE with E300AABBCCDDEE config* + +``` +python3 cloudfront-staging-to-production.py www.json E300AABBCCDDEE E499ZZXXYYDDEE +``` diff --git a/cloudfront-traffic-csv/README.md b/cloudfront-traffic-csv/README.md new file mode 100644 index 0000000..81dcc3f --- /dev/null +++ b/cloudfront-traffic-csv/README.md @@ -0,0 +1,17 @@ +# cloudfront-traffic-csv +Ever had to create regular traffic report for 100+ Amazon CloudFront distributions? +CloudFront Usage Reports under CloudFront console provides great one but it requires user to select each distribution and generate report one by one. +This script pulls Amazon CloudFront metric from CloudWatch, which can be modified to your requirement. + +### Usage: +``` +python cloudfront-traffic-csv.py [--metric ] [--tag tag_key] +``` +Startdate, Enddate defines period of data. +metric is the metric name for CloudFront, default is BytesDownloaded. +Use tag to insert tag value into result if needed. + +You can use command line to pull bytedownload and store to csv +``` +python cloudfront-traffic-csv.py 2020-01-01 2020-05-31 > result.csv +``` \ No newline at end of file diff --git a/cloudfront-traffic-csv/cloudfront-traffic-csv.py b/cloudfront-traffic-csv/cloudfront-traffic-csv.py new file mode 100644 index 0000000..5c32922 --- /dev/null +++ b/cloudfront-traffic-csv/cloudfront-traffic-csv.py @@ -0,0 +1,89 @@ +import boto3 +import argparse +import datetime, dateutil + +#GLOBAL_SETTINGS +timezone = datetime.timezone(datetime.timedelta(hours=0)) #UTC +0, this should be set accordingly. + +#global resource to connect API +cf_client = boto3.client('cloudfront') + +def get_cname(distribution): + ''' + returns 1st CNAME alias or '' if not found + ''' + return distribution['Aliases']['Items'][0] if distribution['Aliases']['Quantity'] > 0 else '' + +def get_tag(arn,tag_key): + ''' + returns given tag value of resource, or '' if not found + ''' + tags = cf_client.list_tags_for_resource(Resource=arn) + result = '' + if len(tags['Tags']['Items']) != 0: + for tag in tags['Tags']['Items']: + if tag['Key'] == tag_key: + result = tag['Value'] + + return result + +def get_traffic_csv_list(start_date, end_date, metric_name, reporting_tag=''): + ''' + returns all CF distribution's Cloudwatch metric, as list of comma separated values. + ''' + + #first load all distributions + distributions = cf_client.list_distributions()['DistributionList']['Items'] + + #prepare metric query + metric_data_queries = [] + + for dist in distributions: + metric_data_queries.append({ + 'Id': 'm_'+dist['Id'], + 'Label': '%s,%s' % (dist['Id'], get_cname(dist), ) + (',%s' % (get_tag(dist['ARN'],reporting_tag), ) if reporting_tag else ''), + 'MetricStat': { + 'Metric': { + 'MetricName': metric_name, + 'Namespace': 'AWS/CloudFront', + 'Dimensions': [ + {'Name': 'DistributionId', 'Value': dist['Id']}, + {'Name': 'Region', 'Value': 'Global'} + ] + }, + 'Period': 86400, + 'Stat': 'Sum', + 'Unit': 'None' + } + }) + + #call Cloudwatch get_metric_data + cw_client = boto3.client('cloudwatch', region_name='us-east-1') + result = cw_client.get_metric_data(MetricDataQueries=metric_data_queries, StartTime=start_date, EndTime=end_date) + + #result csv + csv=['Distribution Id, CNAME, ' + ('Tag, ' if reporting_tag else '') + 'Date, '+metric_name] + for r in result['MetricDataResults']: + for i in range(len(r['Timestamps'])): + csv.append('%s,%s,%f' % (r['Label'],r['Timestamps'][i].astimezone().strftime('%Y-%m-%d'), r['Values'][i],)) + + return csv + +if __name__ == '__main__': + #define command arguments + parser = argparse.ArgumentParser(description='Read CloudWatch Metric of all CloudFront distribution') + parser.add_argument('startdate', action='store', type=lambda x: datetime.datetime.strptime(x, '%Y-%m-%d').replace(tzinfo=timezone), + help='Start date of data period, YYYY-MM-DD.') + parser.add_argument('enddate', action='store', type=lambda x: datetime.datetime.strptime(x+' 23:59:59', '%Y-%m-%d %H:%M:%S').replace(tzinfo=timezone), + help='End date of data period, YYYY-MM-DD.') + parser.add_argument('-m','--metric', default='BytesDownloaded', choices=['BytesDownloaded','Requests', 'BytesUploaded'], + help='Reporting metric, default is BytesDownloaded') + parser.add_argument('-t','--tag', help='Reporting Tag key') + + args = parser.parse_args() + + #call functions + csv_list = get_traffic_csv_list(args.startdate, args.enddate, args.metric, args.tag) + + for line in csv_list: + print(line) \ No newline at end of file