diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 9c0e05c..41f6ebc 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -1,13 +1,10 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - name: Python package on: push: - branches: [ develop ] + branches: [ master ] pull_request: - branches: [ develop ] + branches: [ master ] jobs: build: @@ -15,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 @@ -26,8 +23,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi - name: Test with pytest run: | pytest diff --git a/.github/workflows/python-validate.yml b/.github/workflows/python-validate.yml index 58c3bc2..7731bdd 100644 --- a/.github/workflows/python-validate.yml +++ b/.github/workflows/python-validate.yml @@ -2,9 +2,9 @@ name: Python validate releasability on: push: - branches: [ develop ] + branches: [ master ] pull_request: - branches: [ develop ] + branches: [ master ] jobs: build: @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 @@ -23,8 +23,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install pytest if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi - name: Validate packaging / publishability run: | pip install setuptools wheel twine readme_renderer[md] diff --git a/.gitignore b/.gitignore index f69388d..5d1a686 100644 --- a/.gitignore +++ b/.gitignore @@ -54,3 +54,6 @@ docs/_build/ # Emacs backup files *~ + +datalake_auditor +data \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index ca64b81..c95575d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,13 @@ language: python dist: xenial python: - - "2.7" - - "3.4" - - "3.5" - "3.6" - "3.7" + - "3.8" + - "3.9" install: - pip install -r requirements.txt + - pip install -r requirements-dev.txt - pip install coverage python-coveralls script: nosetests tests/unit --cover-erase --with-coverage --cover-package skew after_success: coveralls diff --git a/README.md b/README.md index bd9fa21..6f3cada 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,12 @@ This pattern identifies a specific EC2 instance running in the `us-west-2` region under the account ID `123456789012`. The account ID is the 12-digit unique identifier for a specific AWS account as described [here](http://docs.aws.amazon.com/general/latest/gr/acct-identifiers.html). + +## Configuration + +Without any configuration file, `skew` use the account ID of the caller, +and credentials defined on your system (aws environment variable, iam role of your instance, ...) + To allow `skew` to find your account number, you need to create a `skew` YAML config file. By default, `skew` will look for your config file in `~/.skew` but you can use the `SKEW_CONFIG` environment variable to tell `skew` @@ -46,15 +52,19 @@ file. The main purpose of skew is to identify resources or sets of resources across services, regions, and accounts and to quickly and easily return the -data associated with those resources. For example, if you wanted to return -the data associated with the example ARN above: +data associated with those resources. + +## Usage + +For example, if you wanted to return the data associated with the example ARN above: ```python from skew import scan arn = scan('arn:aws:ec2:us-west-2:123456789012:instance/i-12345678') for resource in arn: - print(resource.data) + print(resource.data) # return a dict + print(resource.json_dump()) # dump resource in json ``` The call to `scan` returns an ARN object which implements the @@ -81,8 +91,28 @@ you would use: arn = scan('arn:aws:dynamodb:us-.*:234567890123:table/*') ``` -CloudWatch Metrics ------------------- +## Command line Usage + +```bash +python -m "skew" --uri "arn:aws:events:eu-west-1:*:rule/*" --output-path "./data" +``` + +In order to retreive all options: + +```bash +python -m "skew" -h +usage: __main__.py [-h] --uri URI --output-path OUTPUT_PATH [--normalize] + +SKEW alias Stock Keeping Unit + +optional arguments: + -h, --help show this help message and exit + --uri URI scan uri (arn:aws:*:*:1235678910:*/*) + --output-path OUTPUT_PATH + output directory +``` + +## CloudWatch Metrics In addition to making the metadata about a particular AWS resource available to you, `skew` also tries to make it easy to access the available CloudWatch @@ -154,8 +184,7 @@ You can also customize the data returned rather than using the default settings: >>> ``` -Filtering Data --------------- +## Filtering Data Each resource that is retrieved is a Python dictionary. Some of these (e.g. an EC2 Instance) can be quite large and complex. Skew allows you to filter @@ -182,8 +211,7 @@ filtered data is available as the `filtered_data` attribute of the Resource object. The full, unfiltered data is still available as the `data` attribute. -Multithreaded Usage -------------------- +## Multithreaded Usage Skew is single-threaded by default, like most Python libraries. In order to speed up the enumeration of matching resources, you can use multiple threads: @@ -211,11 +239,121 @@ for service in arn.service.choices(): (thanks to @alFReD-NSH for the snippet) -More Examples -------------- +## More Examples [Find Unattached Volumes](https://gist.github.com/garnaat/73804a6b0bd506ee6075) [Audit Security Groups](https://gist.github.com/garnaat/4123f1aefe7d65df9b48) [Find Untagged Instances](https://gist.github.com/garnaat/11004f5661b4798d27c7) + +## Supported Service + +| Name | +| ---------------- | +| route53 | +| cloudfront | +| elasticbeanstalk | +| ecs | +| kms | +| redshift | +| efs | +| sns | +| cloudwatch | +| cloudtrail | +| acm | +| sqs | +| elasticache | +| ecr | +| lambda | +| elb | +| stepfunctions | +| events | +| iam | +| rds | +| cloudsearch | +| logs | +| firehose | +| autoscaling | +| s3 | +| support | +| ec2 | +| cloudformation | +| opsworks | +| es | +| elbv2 | +| kinesis | +| ses | +| dynamodb | +| apigateway | + +## Supported Resource + +| Name | +| --------------------------------------- | +| aws.acm.certificate | +| aws.apigateway.restapis | +| aws.autoscaling.autoScalingGroup | +| aws.autoscaling.launchConfigurationName | +| aws.cloudfront.distribution | +| aws.cloudformation.stack | +| aws.cloudsearch.domain | +| aws.cloudwatch.alarm | +| aws.logs.log-group | +| aws.cloudtrail.trail | +| aws.dynamodb.table | +| aws.ec2.address | +| aws.ec2.customer-gateway | +| aws.ec2.key-pair | +| aws.ec2.image | +| aws.ec2.instance | +| aws.ec2.natgateway | +| aws.ec2.network-acl | +| aws.ec2.route-table | +| aws.ec2.internet-gateway | +| aws.ec2.security-group | +| aws.ec2.snapshot | +| aws.ec2.volume | +| aws.ec2.vpc | +| aws.ec2.flow-log | +| aws.ec2.vpc-peering-connection | +| aws.ec2.subnet | +| aws.ec2.launch-template | +| aws.ecs.cluster | +| aws.ecs.task-definition | +| aws.ecr.registery | +| aws.ecr.repository | +| aws.efs.filesystem | +| aws.elasticache.cluster | +| aws.elasticache.subnet-group | +| aws.elasticache.snapshot | +| aws.elasticbeanstalk.application | +| aws.elasticbeanstalk.environment | +| aws.elb.loadbalancer | +| aws.elbv2.loadbalancer | +| aws.elbv2.targetgroup | +| aws.es.domain | +| aws.events.rule | +| aws.firehose.deliverystream | +| aws.iam.group | +| aws.iam.instance-profile | +| aws.iam.role | +| aws.iam.policy | +| aws.iam.user | +| aws.iam.server-certificate | +| aws.kinesis.stream | +| aws.kms.key | +| aws.lambda.function | +| aws.opsworks.stack | +| aws.rds.db | +| aws.rds.secgrp | +| aws.redshift.cluster | +| aws.route53.hostedzone | +| aws.route53.healthcheck | +| aws.s3.bucket | +| aws.stepfunctions.statemachine | +| aws.sqs.queue | +| aws.ses.identity | +| aws.sns.subscription | +| aws.sns.topic | +| aws.support.check | diff --git a/changelog.md b/changelog.md new file mode 100644 index 0000000..ffd02ea --- /dev/null +++ b/changelog.md @@ -0,0 +1,49 @@ +# Change log + +## 1.0.0 (coming soon) + +- Python 3 and dependencies: + - Fix yam constructor DeprecationWarning and update PyYaml + - Remove python 2, python 3.4 and 3.5 support, add 3.8, 3.9 test unit configuration + - Update and separate dev dependencies from module dependencies + - Align python syntax on version 3 +- Integrate Change from Christophe Morio (https://github.com/lbncmorio/skew/tree/more-resources) +- Configuration: + - Remove mandatory needs for skew.yaml (using iam metadata associated and default boto3 credentials initialization) +- aws client: + - Fix Error and termination BUG with awsclient + - Add boto3 config default with retries={"max_attempts": 20, "mode": "adaptive"} +- Resource Enumeration: + - Fix resource enumeration when no resource type is define + - Rewrote filtering resource and add a warning if filter operation is missing when needed + - Fix bad component matchs operation if similar component share a common prefix (like elb and elbv2) +- Resource Loading + - Change enumerate to avoir loading all resources loaded in memory + - Add lazy loading of full data with method _load_extra_attribute on Resource + - Add lazy load per item on Log group for log_streams, metric_filters, queries, subscriptions +- Additional Ressource and details: + - Group users, policy inline and attached + - kinesis description + - S3 bucket properties (acl, encryption, logging, cors, policy, notifications, ...) + - elbV2 and target group + - Cloud front Domain + - cloud search and region list update + - opsworks availaible on 9 regions + - api gateway + - Name EC2 with Instane Id or Tag Name value if exists + - Cloudtrail: fix enumeration and tags, add trail detail and trail status + - Add json_dump with optional normalisation + - Add ECR Registery + - Add ECR Repository + - Add Kms Key + - Add service definition on ecs cluster + - Add StepFunction (alias states) + - Add Event rule +- Github: + - set master branch as base branch + - update workflow +- Add Command line utility + +## 0.19.0 + +- no change log diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..904c0f2 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,7 @@ +pytest +mock==4.0.3 +nose==1.3.7 +tox==3.20.1 +placebo==0.9.0 +coverage +python-coveralls diff --git a/requirements.txt b/requirements.txt index 7962bec..0f18cfe 100755 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,2 @@ -boto3>=1.2.3 -placebo==0.4.3 -six>=1.9.0 -PyYAML==3.13 -python-dateutil>=2.1,<3.0.0 -mock==1.0.1 -nose==1.3.4 -tox==1.8.1 +boto3==1.16.35 +PyYAML==5.3.1 \ No newline at end of file diff --git a/setup.py b/setup.py index ca00d65..b45f935 100755 --- a/setup.py +++ b/setup.py @@ -36,7 +36,6 @@ 'Intended Audience :: System Administrators', 'Natural Language :: English', 'License :: OSI Approved :: Apache Software License', - 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3' ], ) diff --git a/skew/__init__.py b/skew/__init__.py index 54302df..e267179 100644 --- a/skew/__init__.py +++ b/skew/__init__.py @@ -14,8 +14,16 @@ import os from skew.arn import ARN +from skew.awsdefaults import get_all_activated_regions, get_caller_identity_account_id -__version__ = open(os.path.join(os.path.dirname(__file__), '_version')).read() +__all__ = [ + "__version__", + "get_all_activated_regions", + "get_caller_identity_account_id", + "scan", +] + +__version__ = open(os.path.join(os.path.dirname(__file__), "_version")).read() def scan(sku, **kwargs): diff --git a/skew/__main__.py b/skew/__main__.py new file mode 100644 index 0000000..7f23df9 --- /dev/null +++ b/skew/__main__.py @@ -0,0 +1,96 @@ +# Copyright (c) 2020 Jerome Guibert +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import argparse +import skew + +def _make_directory(path): + try: + os.makedirs(path) + except OSError: + # Already exists + pass + + +def _call_back(resource): + resource.tags + if resource.Meta.service == "s3": + resource.location + resource.acl + resource.cors + resource.encryption + resource.lifecycle + resource.logging + resource.policy + resource.policy_status + resource.notifications + resource.versioning + resource.website + + +def _create_parser(): + + parser = argparse.ArgumentParser(description="SKEW alias Stock Keeping Unit") + + parser.add_argument( + "--uri", + action="store", + type=str, + nargs=1, + help="scan uri (arn:aws:*:*:1235678910:*/*)", + required=True, + ) + + parser.add_argument( + "--output-path", + action="store", + type=str, + nargs=1, + help="output directory", + required=True, + ) + + parser.add_argument( + "--normalize", + action="store_true", + help="normalize json", + dest="normalize", + ) + return parser + + +args = _create_parser().parse_args() + + +_uri = str(args.uri[0]) +_output_path = args.output_path[0] +for resource in skew.scan(_uri): + _call_back(resource) + directory = None + identifier = None + if "/" in resource.arn: + data = resource.arn.split("/") + identifier = data.pop(-1) + data = ":".join(data).split(":") + directory = os.path.join(_output_path, *data) + else: + data = resource.arn.split(":") + identifier = data.pop(-1) + directory = os.path.join(_output_path, *data) + + _make_directory(directory) + + with open(os.path.join(directory, f"{identifier}.json"), "w") as f: + f.write((resource.json_dump(normalize=args.normalize))) diff --git a/skew/arn/__init__.py b/skew/arn/__init__.py index 06b332c..3fc5556 100644 --- a/skew/arn/__init__.py +++ b/skew/arn/__init__.py @@ -1,5 +1,6 @@ # Copyright 2014 Scopely, Inc. # Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -23,13 +24,13 @@ from skew.config import get_config LOG = logging.getLogger(__name__) -DebugFmtString = '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +DebugFmtString = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" class ARNComponent(object): - def __init__(self, pattern, arn): self.pattern = pattern + # arn is Arn parent instance self._arn = arn def __repr__(self): @@ -62,8 +63,11 @@ def match(self, pattern, context=None): """ matches = [] regex = pattern - if regex == '*': - regex = '.*' + if regex == "*": + regex = ".*" + elif regex: + # avoid match of elb and elbv2 + regex += "$" regex = re.compile(regex) for choice in self.choices(context): if regex.search(choice): @@ -77,25 +81,26 @@ def matches(self, context=None): """ return self.match(self.pattern, context) - def complete(self, prefix='', context=None): + def complete(self, prefix="", context=None): return [c for c in self.choices(context) if c.startswith(prefix)] class Resource(ARNComponent): - def _split_resource(self, resource): - LOG.debug('split_resource: %s', resource) - if '/' in resource: - resource_type, resource_id = resource.split('/', 1) - elif ':' in resource: - resource_type, resource_id = resource.split(':', 1) + LOG.debug("split_resource: %s", resource) + if "/" in resource: + resource_type, resource_id = resource.split("/", 1) + elif ":" in resource: + resource_type, resource_id = resource.split(":", 1) else: # TODO: Some services use ARN's that include only a resource # identifier (i.e. no resource type). SNS is one example but # there are others. We need to refactor this code to allow # the splitting of the resource part of the ARN to be handled # by the individual resource classes rather than here. - resource_type = None + + # Fix resource enumeration when no resource type is define + resource_type = "*" # one resource type in this case per service resource_id = resource return (resource_type, resource_id) @@ -111,76 +116,111 @@ def choices(self, context=None): provider = self._arn.provider.pattern all_resources = skew.resources.all_types(provider, service) if not all_resources: - all_resources = ['*'] + all_resources = ["*"] return all_resources def enumerate(self, context, **kwargs): - LOG.debug('Resource.enumerate %s', context) + LOG.debug("Resource.enumerate %s", context) _, provider, service_name, region, account = context resource_type, resource_id = self._split_resource(self.pattern) - LOG.debug('resource_type=%s, resource_id=%s', - resource_type, resource_id) + LOG.debug("resource_type=%s, resource_id=%s", resource_type, resource_id) resources = [] for resource_type in self.matches(context): - resource_path = '.'.join([provider, service_name, resource_type]) + resource_path = ".".join([provider, service_name, resource_type]) resource_cls = skew.resources.find_resource_class(resource_path) - resources.extend(resource_cls.enumerate( - self._arn, region, account, resource_id, **kwargs)) + resources.extend( + resource_cls.enumerate( + self._arn, region, account, resource_id, **kwargs + ) + ) return resources class Account(ARNComponent): - def __init__(self, pattern, arn): - self._accounts = get_config()['accounts'] + self._accounts = get_config()["accounts"] super(Account, self).__init__(pattern, arn) def choices(self, context=None): return list(self._accounts.keys()) def enumerate(self, context, **kwargs): - LOG.debug('Account.enumerate %s', context) + LOG.debug("Account.enumerate %s", context) for match in self.matches(context): context.append(match) - for resource in self._arn.resource.enumerate( - context, **kwargs): + for resource in self._arn.resource.enumerate(context, **kwargs): yield resource context.pop() class Region(ARNComponent): - _all_region_names = ['us-east-1', - 'us-east-2', - 'us-west-1', - 'us-west-2', - 'eu-west-1', - 'eu-west-2', - 'eu-west-3', - 'eu-central-1', - 'eu-north-1', - 'eu-south-1', - 'ap-southeast-1', - 'ap-southeast-2', - 'ap-northeast-1', - 'ap-northeast-2', - 'ap-south-1', - 'ap-east-1', - 'af-south-1' - 'ca-central-1', - 'sa-east-1', - 'me-south-1', - 'cn-north-1', - 'cn-northwest-1'] - - _no_region_required = [''] + _all_region_names = [ + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + "eu-west-1", + "eu-west-2", + "eu-west-3", + "eu-central-1", + "eu-north-1", + "eu-south-1", + "ap-southeast-1", + "ap-southeast-2", + "ap-northeast-1", + "ap-northeast-2", + "ap-south-1", + "ap-east-1", + "af-south-1", + "ca-central-1", + "sa-east-1", + "me-south-1", + "cn-north-1", + "cn-northwest-1", + ] + + _no_region_required = [""] _service_region_map = { - 'redshift': _all_region_names, - 'glacier': ['ap-northeast-1', 'ap-northeast-2', 'ap-south-1', 'ap-southeast-2', 'ca-central-1', 'eu-central-1', - 'eu-west-1', 'eu-west-2', 'us-east-1', 'us-east-2', 'us-west-1', 'us-west-2'], - 'cloudfront': _no_region_required, - 'iam': _no_region_required, - 'route53': _no_region_required + "glacier": [ + "ap-northeast-1", + "ap-northeast-2", + "ap-south-1", + "ap-southeast-2", + "ca-central-1", + "eu-central-1", + "eu-west-1", + "eu-west-2", + "us-east-1", + "us-east-2", + "us-west-1", + "us-west-2", + ], + "cloudfront": _no_region_required, + "iam": _no_region_required, + "route53": _no_region_required, + "cloudsearch": [ + "us-east-1", + "us-west-2", + "us-west-1", + "eu-west-1", + "eu-central-1", + "sa-east-1", + "ap-southeast-1", + "ap-northeast-1", + "ap-southeast-2", + ], + "opsworks": [ + "us-east-2", + "us-west-1", + "us-east-1", + "us-west-2", + "eu-central-1", + "eu-west-1", + "ap-southeast-1", + "ap-northeast-1", + "ap-southeast-2", + ], } def choices(self, context=None): @@ -188,21 +228,18 @@ def choices(self, context=None): service = context[2] else: service = self._arn.service - return self._service_region_map.get( - service, self._all_region_names) + return self._service_region_map.get(service, self._all_region_names) def enumerate(self, context, **kwargs): - LOG.debug('Region.enumerate %s', context) + LOG.debug("Region.enumerate %s", context) for match in self.matches(context): context.append(match) - for account in self._arn.account.enumerate( - context, **kwargs): + for account in self._arn.account.enumerate(context, **kwargs): yield account context.pop() class Service(ARNComponent): - def choices(self, context=None): if context: provider = context[1] @@ -211,41 +248,36 @@ def choices(self, context=None): return skew.resources.all_services(provider) def enumerate(self, context, **kwargs): - LOG.debug('Service.enumerate %s', context) + LOG.debug("Service.enumerate %s", context) for match in self.matches(context): context.append(match) - for region in self._arn.region.enumerate( - context, **kwargs): + for region in self._arn.region.enumerate(context, **kwargs): yield region context.pop() class Provider(ARNComponent): - def choices(self, context=None): - return ['aws'] + return ["aws"] def enumerate(self, context, **kwargs): - LOG.debug('Provider.enumerate %s', context) + LOG.debug("Provider.enumerate %s", context) for match in self.matches(context): context.append(match) - for service in self._arn.service.enumerate( - context, **kwargs): + for service in self._arn.service.enumerate(context, **kwargs): yield service context.pop() class Scheme(ARNComponent): - def choices(self, context=None): - return ['arn'] + return ["arn"] def enumerate(self, context, **kwargs): - LOG.debug('Scheme.enumerate %s', context) + LOG.debug("Scheme.enumerate %s", context) for match in self.matches(context): context.append(match) - for provider in self._arn.provider.enumerate( - context, **kwargs): + for provider in self._arn.provider.enumerate(context, **kwargs): yield provider context.pop() @@ -254,17 +286,17 @@ class ARN(object): ComponentClasses = [Scheme, Provider, Service, Region, Account, Resource] - def __init__(self, arn_string='arn:aws:*:*:*:*', **kwargs): + def __init__(self, arn_string="arn:aws:*:*:*:*", **kwargs): self.query = None self._components = None self._build_components_from_string(arn_string) self.kwargs = kwargs def __repr__(self): - return ':'.join([str(c) for c in self._components]) + return ":".join([str(c) for c in self._components]) def debug(self): - self.set_logger('skew', logging.DEBUG) + self.set_logger("skew", logging.DEBUG) def set_logger(self, logger_name, level=logging.DEBUG): """ @@ -287,11 +319,12 @@ def set_logger(self, logger_name, level=logging.DEBUG): log.addHandler(ch) def _build_components_from_string(self, arn_string): - if '|' in arn_string: - arn_string, query = arn_string.split('|') + if "|" in arn_string: + arn_string, query = arn_string.split("|") self.query = jmespath.compile(query) pairs = zip_longest( - self.ComponentClasses, arn_string.split(':', 5), fillvalue='*') + self.ComponentClasses, arn_string.split(":", 5), fillvalue="*" + ) self._components = [c(n, self) for c, n in pairs] @property diff --git a/skew/awsclient.py b/skew/awsclient.py index 85820ef..ae44106 100644 --- a/skew/awsclient.py +++ b/skew/awsclient.py @@ -1,4 +1,5 @@ # Copyright 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -19,6 +20,8 @@ import jmespath import boto3 from botocore.exceptions import ClientError +from botocore.config import Config +import botocore from skew.config import get_config @@ -35,23 +38,31 @@ def json_encoder(obj): class AWSClient(object): + boto3_retry_config = {"max_attempts": 20, "mode": "adaptive"} + def __init__(self, service_name, region_name, account_id, **kwargs): - self._config = get_config() + _config = get_config() self._service_name = service_name self._region_name = region_name self._account_id = account_id self._has_credentials = False - self.aws_creds = kwargs.get('aws_creds') - if self.aws_creds is None: - self.aws_creds = self._config['accounts'][account_id].get( - 'credentials') - if self.aws_creds is None: - # no aws_creds, need profile to get creds from ~/.aws/credentials - self._profile = self._config['accounts'][account_id]['profile'] - self.placebo = kwargs.get('placebo') - self.placebo_dir = kwargs.get('placebo_dir') - self.placebo_mode = kwargs.get('placebo_mode', 'record') - self._client = self._create_client() + self.aws_creds = kwargs.get("aws_creds") + self._profile = None + if self.aws_creds is None and account_id in _config["accounts"]: + self.aws_creds = _config["accounts"][account_id].get("credentials") + if self.aws_creds: + # no aws_creds, need profile to get creds from ~/.aws/credentials or iam metadata role instance + self._profile = _config["accounts"][account_id].get("profile") + + self._client = self._create_client( + placebo=kwargs.get("placebo"), + placebo_dir=kwargs.get("placebo_dir"), + placebo_mode=kwargs.get("placebo_mode", "record"), + ) + # remove test key from client call + kwargs.pop("placebo", None) + kwargs.pop("placebo_dir", None) + kwargs.pop("placebo_mode", None) @property def service_name(self): @@ -69,20 +80,27 @@ def account_id(self): def profile(self): return self._profile - def _create_client(self): + def _create_client(self, placebo=None, placebo_dir=None, placebo_mode=None): + # Initialize Session if self.aws_creds: session = boto3.Session(**self.aws_creds) + elif self.profile: + session = boto3.Session(profile_name=self.profile) else: - session = boto3.Session( - profile_name=self.profile) - if self.placebo and self.placebo_dir: - pill = self.placebo.attach(session, self.placebo_dir) - if self.placebo_mode == 'record': + session = boto3.Session() + # Placebo Condiguration + if placebo and placebo_dir: + pill = placebo.attach(session, data_path=placebo_dir) + if placebo_mode == "record": pill.record() - elif self.placebo_mode == 'playback': + elif placebo_mode == "playback": pill.playback() - return session.client(self.service_name, - region_name=self.region_name if self.region_name else None) + # create boto3 client + return session.client( + self.service_name, + region_name=self.region_name if self.region_name else None, + config=Config(retries=self.boto3_retry_config), + ) def call(self, op_name, query=None, **kwargs): """ @@ -112,8 +130,8 @@ def call(self, op_name, query=None, **kwargs): to the method when making the request. """ LOG.debug(kwargs) - if query: - query = jmespath.compile(query) + + data = {} if self._client.can_paginate(op_name): paginator = self._client.get_paginator(op_name) results = paginator.paginate(**kwargs) @@ -121,27 +139,35 @@ def call(self, op_name, query=None, **kwargs): else: op = getattr(self._client, op_name) done = False - data = {} while not done: try: data = op(**kwargs) done = True except ClientError as e: LOG.debug(e, kwargs) - if 'Throttling' in str(e): + if "Throttling" in str(e): time.sleep(1) - elif 'AccessDenied' in str(e): + elif "AccessDenied" in str(e): + done = True + elif "UnrecognizedClientException" in str(e): + LOG.error(e) + self._client = self._create_client() + elif "NoSuchTagSet" in str(e): done = True - elif 'NoSuchTagSet' in str(e): + else: + # Avoid infinite loop done = True except Exception: done = True if query: - data = query.search(data) + return jmespath.compile(query).search(data) return data def get_awsclient(service_name, region_name, account_id, **kwargs): - if region_name == '': - region_name = None - return AWSClient(service_name, region_name, account_id, **kwargs) + return AWSClient( + service_name=service_name, + region_name=None if region_name == "" else region_name, + account_id=account_id, + **kwargs + ) diff --git a/skew/awsdefaults.py b/skew/awsdefaults.py new file mode 100644 index 0000000..d3b3d49 --- /dev/null +++ b/skew/awsdefaults.py @@ -0,0 +1,69 @@ +# Copyright (c) 2020 Jerome Guibert +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from typing import List +import os +import boto3 +from functools import lru_cache +from botocore.config import Config + +__all__ = [ + "get_default_region", + "get_all_activated_regions", + "get_caller_identity_account_id", + "get_default_session", + "get_client", +] + + +@lru_cache(maxsize=10) +def get_default_region() -> str: + default_region = os.environ.get("DEFAULT_AWS_REGION", "us-east-1") + if "gov-" in default_region: + default_region = "us-gov-west-1" + elif "cn-" in default_region: + default_region = "cn-north-1" + else: + default_region = "us-east-1" + return default_region + + +@lru_cache(maxsize=10) +def get_all_activated_regions() -> List[str]: + """Return a list of enabled region of caller account.""" + return list( + map( + lambda r: r["RegionName"], + get_client(session=get_default_session(), service="ec2").describe_regions()[ + "Regions" + ], + ) + ) + + +def get_default_session(): + return boto3.Session(region_name=get_default_region()) + + +def get_client(session, service, region=None, max_attempts=20): + # see https://botocore.amazonaws.com/v1/documentation/api/latest/reference/config.html + return session.client( + service, + region_name=region, + config=Config(retries={"max_attempts": max_attempts, "mode": "adaptive"}), + ) + + +def get_caller_identity_account_id(): + return get_client(get_default_session(), "sts").get_caller_identity()["Account"] diff --git a/skew/config.py b/skew/config.py index 28aec81..55785c9 100644 --- a/skew/config.py +++ b/skew/config.py @@ -1,4 +1,5 @@ # Copyright 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -18,9 +19,11 @@ import yaml from skew.exception import ConfigNotFoundError +from skew.awsdefaults import get_caller_identity_account_id LOG = logging.getLogger(__name__) +__all__ = ["get_config"] _config = None @@ -28,11 +31,15 @@ def get_config(): global _config if _config is None: - path = os.environ.get('SKEW_CONFIG', os.path.join('~', '.skew')) + path = os.environ.get("SKEW_CONFIG", os.path.join("~", ".skew")) path = os.path.expanduser(path) path = os.path.expandvars(path) - if not os.path.exists(path): - raise ConfigNotFoundError('Unable to find skew config file') - with open(path) as config_file: - _config = yaml.safe_load(config_file) + if os.path.exists(path): + with open(path) as config_file: + _config = yaml.load(config_file, Loader=yaml.FullLoader) + else: + LOG.warning("Unable to find skew config file") + _config = {"accounts": {}} + _config["accounts"][get_caller_identity_account_id()] = {} + LOG.warning(f"Default skew Configuration: {_config}") return _config diff --git a/skew/resources/__init__.py b/skew/resources/__init__.py index e586d1d..120fff0 100644 --- a/skew/resources/__init__.py +++ b/skew/resources/__init__.py @@ -1,5 +1,6 @@ # Copyright (c) 2014 Scopely, Inc. # Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -17,72 +18,86 @@ # Maps resources names as they appear in ARN's to the path name # of the Python class representing that resource. ResourceTypes = { - 'aws.acm.certificate': 'aws.acm.Certificate', - 'aws.apigateway.restapis': 'aws.apigateway.RestAPI', - 'aws.autoscaling.autoScalingGroup': 'aws.autoscaling.AutoScalingGroup', - 'aws.autoscaling.launchConfigurationName': 'aws.autoscaling.LaunchConfiguration', - 'aws.cloudfront.distribution': 'aws.cloudfront.Distribution', - 'aws.cloudformation.stack': 'aws.cloudformation.Stack', - 'aws.cloudwatch.alarm': 'aws.cloudwatch.Alarm', - 'aws.logs.log-group': 'aws.cloudwatch.LogGroup', - 'aws.cloudtrail.trail': 'aws.cloudtrail.CloudTrail', - 'aws.dynamodb.table': 'aws.dynamodb.Table', - 'aws.ec2.address': 'aws.ec2.Address', - 'aws.ec2.customer-gateway': 'aws.ec2.CustomerGateway', - 'aws.ec2.key-pair': 'aws.ec2.KeyPair', - 'aws.ec2.image': 'aws.ec2.Image', - 'aws.ec2.instance': 'aws.ec2.Instance', - 'aws.ec2.natgateway': 'aws.ec2.NatGateway', - 'aws.ec2.network-acl': 'aws.ec2.NetworkAcl', - 'aws.ec2.route-table': 'aws.ec2.RouteTable', - 'aws.ec2.internet-gateway': 'aws.ec2.InternetGateway', - 'aws.ec2.security-group': 'aws.ec2.SecurityGroup', - 'aws.ec2.snapshot': 'aws.ec2.Snapshot', - 'aws.ec2.volume': 'aws.ec2.Volume', - 'aws.ec2.vpc': 'aws.ec2.Vpc', - 'aws.ec2.flow-log': 'aws.ec2.FlowLog', - 'aws.ec2.vpc-peering-connection': 'aws.ec2.VpcPeeringConnection', - 'aws.ec2.subnet': 'aws.ec2.Subnet', - 'aws.ec2.launch-template': 'aws.ec2.LaunchTemplate', - 'aws.elasticache.cluster': 'aws.elasticache.Cluster', - 'aws.elasticache.subnet-group': 'aws.elasticache.SubnetGroup', - 'aws.elasticache.snapshot': 'aws.elasticache.Snapshot', - 'aws.elasticbeanstalk.application': 'aws.elasticbeanstalk.Application', - 'aws.elasticbeanstalk.environment': 'aws.elasticbeanstalk.Environment', - 'aws.elb.loadbalancer': 'aws.elb.LoadBalancer', - 'aws.es.domain': 'aws.es.ElasticsearchDomain', - 'aws.firehose.deliverystream': 'aws.firehose.DeliveryStream', - 'aws.iam.group': 'aws.iam.Group', - 'aws.iam.instance-profile': 'aws.iam.InstanceProfile', - 'aws.iam.role': 'aws.iam.Role', - 'aws.iam.policy': 'aws.iam.Policy', - 'aws.iam.user': 'aws.iam.User', - 'aws.iam.server-certificate': 'aws.iam.ServerCertificate', - 'aws.kinesis.stream': 'aws.kinesis.Stream', - 'aws.lambda.function': 'aws.lambda.Function', - 'aws.rds.db': 'aws.rds.DBInstance', - 'aws.rds.secgrp': 'aws.rds.DBSecurityGroup', - 'aws.redshift.cluster': 'aws.redshift.Cluster', - 'aws.route53.hostedzone': 'aws.route53.HostedZone', - 'aws.route53.healthcheck': 'aws.route53.HealthCheck', - 'aws.s3.bucket': 'aws.s3.Bucket', - 'aws.sqs.queue': 'aws.sqs.Queue', - 'aws.sns.subscription': 'aws.sns.Subscription', - 'aws.sns.topic': 'aws.sns.Topic' + "aws.acm.certificate": "aws.acm.Certificate", + "aws.apigateway.restapis": "aws.apigateway.RestAPI", + "aws.autoscaling.autoScalingGroup": "aws.autoscaling.AutoScalingGroup", + "aws.autoscaling.launchConfigurationName": "aws.autoscaling.LaunchConfiguration", + "aws.cloudfront.distribution": "aws.cloudfront.Distribution", + "aws.cloudformation.stack": "aws.cloudformation.Stack", + "aws.cloudsearch.domain": "aws.cloudsearch.Domain", + "aws.cloudwatch.alarm": "aws.cloudwatch.Alarm", + "aws.logs.log-group": "aws.cloudwatch.LogGroup", + "aws.cloudtrail.trail": "aws.cloudtrail.CloudTrail", + "aws.dynamodb.table": "aws.dynamodb.Table", + "aws.ec2.address": "aws.ec2.Address", + "aws.ec2.customer-gateway": "aws.ec2.CustomerGateway", + "aws.ec2.key-pair": "aws.ec2.KeyPair", + "aws.ec2.image": "aws.ec2.Image", + "aws.ec2.instance": "aws.ec2.Instance", + "aws.ec2.natgateway": "aws.ec2.NatGateway", + "aws.ec2.network-acl": "aws.ec2.NetworkAcl", + "aws.ec2.route-table": "aws.ec2.RouteTable", + "aws.ec2.internet-gateway": "aws.ec2.InternetGateway", + "aws.ec2.security-group": "aws.ec2.SecurityGroup", + "aws.ec2.snapshot": "aws.ec2.Snapshot", + "aws.ec2.volume": "aws.ec2.Volume", + "aws.ec2.vpc": "aws.ec2.Vpc", + "aws.ec2.flow-log": "aws.ec2.FlowLog", + "aws.ec2.vpc-peering-connection": "aws.ec2.VpcPeeringConnection", + "aws.ec2.subnet": "aws.ec2.Subnet", + "aws.ec2.launch-template": "aws.ec2.LaunchTemplate", + "aws.ecs.cluster": "aws.ecs.Cluster", + "aws.ecs.task-definition": "aws.ecs.TaskDefinition", + "aws.ecr.registery": "aws.ecr.Registery", + "aws.ecr.repository": "aws.ecr.Repository", + "aws.efs.filesystem": "aws.efs.Filesystem", + "aws.elasticache.cluster": "aws.elasticache.Cluster", + "aws.elasticache.subnet-group": "aws.elasticache.SubnetGroup", + "aws.elasticache.snapshot": "aws.elasticache.Snapshot", + "aws.elasticbeanstalk.application": "aws.elasticbeanstalk.Application", + "aws.elasticbeanstalk.environment": "aws.elasticbeanstalk.Environment", + "aws.elb.loadbalancer": "aws.elb.LoadBalancer", + "aws.elbv2.loadbalancer": "aws.elbv2.LoadBalancer", + "aws.elbv2.targetgroup": "aws.elbv2.TargetGroup", + "aws.es.domain": "aws.es.ElasticsearchDomain", + "aws.events.rule": "aws.cloudwatch.CloudWatchEventRule", + "aws.firehose.deliverystream": "aws.firehose.DeliveryStream", + "aws.iam.group": "aws.iam.Group", + "aws.iam.instance-profile": "aws.iam.InstanceProfile", + "aws.iam.role": "aws.iam.Role", + "aws.iam.policy": "aws.iam.Policy", + "aws.iam.user": "aws.iam.User", + "aws.iam.server-certificate": "aws.iam.ServerCertificate", + "aws.kinesis.stream": "aws.kinesis.Stream", + "aws.kms.key": "aws.kms.Key", + "aws.lambda.function": "aws.lambda.Function", + "aws.opsworks.stack": "aws.opsworks.Stack", + "aws.rds.db": "aws.rds.DBInstance", + "aws.rds.secgrp": "aws.rds.DBSecurityGroup", + "aws.redshift.cluster": "aws.redshift.Cluster", + "aws.route53.hostedzone": "aws.route53.HostedZone", + "aws.route53.healthcheck": "aws.route53.HealthCheck", + "aws.s3.bucket": "aws.s3.Bucket", + "aws.stepfunctions.statemachine": "aws.stepfunctions.StateMachines", + "aws.sqs.queue": "aws.sqs.Queue", + "aws.ses.identity": "aws.ses.Identity", + "aws.sns.subscription": "aws.sns.Subscription", + "aws.sns.topic": "aws.sns.Topic", + "aws.support.check": "aws.support.Check", } def all_providers(): providers = set() for resource_type in ResourceTypes: - providers.add(resource_type.split('.')[0]) + providers.add(resource_type.split(".")[0]) return list(providers) def all_services(provider_name): services = set() for resource_type in ResourceTypes: - t = resource_type.split('.') + t = resource_type.split(".") if t[0] == provider_name: services.add(t[1]) return list(services) @@ -91,7 +106,7 @@ def all_services(provider_name): def all_types(provider_name, service_name): types = set() for resource_type in ResourceTypes: - t = resource_type.split('.') + t = resource_type.split(".") if t[0] == provider_name and t[1] == service_name: types.add(t[2]) return list(types) @@ -103,7 +118,7 @@ def find_resource_class(resource_path): """ class_path = ResourceTypes[resource_path] # First prepend our __name__ to the resource string passed in. - full_path = '.'.join([__name__, class_path]) + full_path = ".".join([__name__, class_path]) class_data = full_path.split(".") module_path = ".".join(class_data[:-1]) class_str = class_data[-1] diff --git a/skew/resources/aws/__init__.py b/skew/resources/aws/__init__.py index ff0ae41..a006eed 100644 --- a/skew/resources/aws/__init__.py +++ b/skew/resources/aws/__init__.py @@ -1,5 +1,6 @@ # Copyright (c) 2014 Scopely, Inc. # Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -83,58 +84,65 @@ class AWSResource(Resource): given type. But you can also tell it to filter the results by passing in a list of id's. This parameter tells it the name of the parameter to use to specify this list of id's. + + """ class Meta(object): - type = 'awsresource' + type = "awsresource" @classmethod def filter(cls, arn, resource_id, data): + """ + If the API does not support filtering, the resource + return True if the returned data matches the + resource ID we are looking for. + """ + LOG.warning("filter classmethod must be implemented for %s", cls) pass def __init__(self, client, data, query=None): - self._client = client + super(AWSResource, self).__init__(client=client, data=data) self._query = query - if data is None: - data = {} - self.data = data - if self._query: - self.filtered_data = self._query.search(self.data) - else: - self.filtered_data = None - if hasattr(self.Meta, 'id') and isinstance(self.data, dict): - self._id = self.data.get(self.Meta.id, '') - else: - self._id = '' + self.filtered_data = self._query.search(self._data) if self._query else None self._cloudwatch = None - if hasattr(self.Meta, 'dimension') and self.Meta.dimension: + if hasattr(self.Meta, "dimension") and self.Meta.dimension: self._cloudwatch = skew.awsclient.get_awsclient( - 'cloudwatch', self._client.region_name, - self._client.account_id) - self._metrics = None - self._name = None - self._date = None + "cloudwatch", self._client.region_name, self._client.account_id + ) self._tags = None + self._extra_attribute_loaded = False def __repr__(self): return self.arn + @property + def data(self): + if not self._extra_attribute_loaded: + if hasattr(self, "_load_extra_attribute"): + self._load_extra_attribute() + self._extra_attribute_loaded = True + return super(AWSResource, self).data + @property def arn(self): - return 'arn:aws:%s:%s:%s:%s/%s' % ( + return "arn:aws:%s:%s:%s:%s/%s" % ( self._client.service_name, self._client.region_name, - self._client.account_id, self.resourcetype, self.id) + self._client.account_id, + self.resourcetype, + self.id, + ) @property def metrics(self): if self._metrics is None: if self._cloudwatch: data = self._cloudwatch.call( - 'list_metrics', - Dimensions=[{'Name': self.Meta.dimension, - 'Value': self._id}]) - self._metrics = jmespath.search('Metrics', data) + "list_metrics", + Dimensions=[{"Name": self.Meta.dimension, "Value": self._id}], + ) + self._metrics = jmespath.search("Metrics", data) else: self._metrics = [] return self._metrics @@ -142,48 +150,38 @@ def metrics(self): @property def tags(self): """ - Convert the ugly Tags JSON into a real dictionary and + Load and Convert the ugly Tags JSON into a real dictionary and memorize the result. """ if self._tags is None: - LOG.debug('need to build tags') + LOG.debug("need to build tags") self._tags = {} - if hasattr(self.Meta, 'tags_spec') and (self.Meta.tags_spec is not None): - LOG.debug('have a tags_spec') + if hasattr(self.Meta, "tags_spec") and (self.Meta.tags_spec is not None): + LOG.debug("have a tags_spec") method, path, param_name, param_value = self.Meta.tags_spec[:4] kwargs = {} - filter_type = getattr(self.Meta, 'filter_type', None) - if filter_type == 'arn': + filter_type = getattr(self.Meta, "filter_type", None) + if filter_type == "arn": kwargs = {param_name: [getattr(self, param_value)]} - elif filter_type == 'list': + elif filter_type == "list": kwargs = {param_name: [getattr(self, param_value)]} else: kwargs = {param_name: getattr(self, param_value)} if len(self.Meta.tags_spec) > 4: kwargs.update(self.Meta.tags_spec[4]) - LOG.debug('fetching tags') - self.data['Tags'] = self._client.call( - method, query=path, **kwargs) - LOG.debug(self.data['Tags']) - - if 'Tags' in self.data: - _tags = self.data['Tags'] - if isinstance(_tags, list): - for kvpair in _tags: - if kvpair['Key'] in self._tags: - if not isinstance(self._tags[kvpair['Key']], list): - self._tags[kvpair['Key']] = [self._tags[kvpair['Key']]] - self._tags[kvpair['Key']].append(kvpair['Value']) - else: - self._tags[kvpair['Key']] = kvpair['Value'] - elif isinstance(_tags, dict): - self._tags = _tags + LOG.debug("fetching tags") + self._data["Tags"] = self._client.call(method, query=path, **kwargs) + LOG.debug(self._data["Tags"]) + + if "Tags" in self._data: + self._tags = self._normalize_tags(self._data["Tags"]) + return self._tags def find_metric(self, metric_name): for m in self.metrics: - if m['MetricName'] == metric_name: + if m["MetricName"] == metric_name: return m return None @@ -191,12 +189,20 @@ def _total_seconds(self, delta): # python2.6 does not have timedelta.total_seconds() so we have # to calculate this ourselves. This is straight from the # datetime docs. - return ((delta.microseconds + (delta.seconds + delta.days * 24 * 3600) - * 10 ** 6) / 10 ** 6) + return ( + delta.microseconds + (delta.seconds + delta.days * 24 * 3600) * 10 ** 6 + ) / 10 ** 6 - def get_metric_data(self, metric_name=None, metric=None, - days=None, hours=1, minutes=None, - statistics=None, period=None): + def get_metric_data( + self, + metric_name=None, + metric=None, + days=None, + hours=1, + minutes=None, + statistics=None, + period=None, + ): """ Get metric data for this resource. You can specify the time frame for the data as either the number of days or number of @@ -234,7 +240,7 @@ def get_metric_data(self, metric_name=None, metric=None, been calculated by skew. """ if not statistics: - statistics = ['Average'] + statistics = ["Average"] if days: delta = datetime.timedelta(days=days) elif hours: @@ -249,18 +255,47 @@ def get_metric_data(self, metric_name=None, metric=None, end = datetime.datetime.utcnow() start = end - delta data = self._cloudwatch.call( - 'get_metric_statistics', - Dimensions=metric['Dimensions'], - Namespace=metric['Namespace'], - MetricName=metric['MetricName'], - StartTime=start.isoformat(), EndTime=end.isoformat(), - Statistics=statistics, Period=period) - return MetricData(jmespath.search('Datapoints', data), - period) + "get_metric_statistics", + Dimensions=metric["Dimensions"], + Namespace=metric["Namespace"], + MetricName=metric["MetricName"], + StartTime=start.isoformat(), + EndTime=end.isoformat(), + Statistics=statistics, + Period=period, + ) + return MetricData(jmespath.search("Datapoints", data), period) else: - raise ValueError('Metric (%s) not available' % metric_name) + raise ValueError("Metric (%s) not available" % metric_name) + + def _normalize_tags(self, tags): + """Convert the ugly Tags JSON into a real dictionary. """ + result = {} + if isinstance(tags, list): + for kvpair in tags: + # Compatibility fix for ECS, that use lowercase 'key' and 'value' as dict keys + # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/ecs.html#ECS.Client.list_tags_for_resource + tags_key = kvpair.get("Key", kvpair.get("key")) + tags_value = kvpair.get("Value", kvpair.get("value")) + if tags_key in tags: + if not isinstance(tags[tags_key], list): + result[tags_key] = [tags[tags_key]] + result[tags_key].append(tags_value) + else: + result[tags_key] = tags_value + elif isinstance(tags, dict): + result = tags + return result + + def _feed_from_spec(self, attr_spec): + """Utilty to call boto3 on demand.""" + method, path, param_name, param_value = attr_spec[:4] + kwargs = {param_name: getattr(self, param_value)} + if path: + kwargs["query"] = path + return self._client.call(method, **kwargs) -ArnComponents = namedtuple('ArnComponents', - ['scheme', 'provider', 'service', 'region', - 'account', 'resource']) +ArnComponents = namedtuple( + "ArnComponents", ["scheme", "provider", "service", "region", "account", "resource"] +) diff --git a/skew/resources/aws/acm.py b/skew/resources/aws/acm.py index 484abee..0e1d5aa 100644 --- a/skew/resources/aws/acm.py +++ b/skew/resources/aws/acm.py @@ -1,5 +1,6 @@ # Copyright (c) 2014 Scopely, Inc. # Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -23,37 +24,34 @@ class Certificate(AWSResource): - class Meta(object): - service = 'acm' - type = 'certificate' - enum_spec = ('list_certificates', 'CertificateSummaryList', None) - detail_spec = ('describe_certificate', 'CertificateArn', 'Certificate') - id = 'CertificateArn' - tags_spec = ('list_tags_for_certificate', 'Tags[]', - 'CertificateArn', 'id') + service = "acm" + type = "certificate" + enum_spec = ("list_certificates", "CertificateSummaryList", None) + detail_spec = ("describe_certificate", "CertificateArn", "Certificate") + id = "CertificateArn" + tags_spec = ("list_tags_for_certificate", "Tags[]", "CertificateArn", "id") filter_name = None - name = 'DomainName' - date = 'CreatedAt' + name = "DomainName" + date = "CreatedAt" dimension = None @classmethod def filter(cls, arn, resource_id, data): - certificate_id = data.get(cls.Meta.id).split('/')[-1] - LOG.debug('%s == %s', resource_id, certificate_id) + certificate_id = data.get(cls.Meta.id).split("/")[-1] + LOG.debug("%s == %s", resource_id, certificate_id) return resource_id == certificate_id @property def arn(self): - return self.data['CertificateArn'] + return self._data["CertificateArn"] def __init__(self, client, data, query=None): super(Certificate, self).__init__(client, data, query) - self._id = data['CertificateArn'] + self._id = data["CertificateArn"] detail_op, param_name, detail_path = self.Meta.detail_spec - params = {param_name: data['CertificateArn']} + params = {param_name: data["CertificateArn"]} data = client.call(detail_op, **params) - - self.data = jmespath.search(detail_path, data) + self._data = jmespath.search(detail_path, data) diff --git a/skew/resources/aws/apigateway.py b/skew/resources/aws/apigateway.py index a460a0a..11c8176 100644 --- a/skew/resources/aws/apigateway.py +++ b/skew/resources/aws/apigateway.py @@ -1,3 +1,5 @@ +# Copyright (c) 2019 Christophe Morio +# # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of # the License is located at @@ -17,21 +19,28 @@ class RestAPI(AWSResource): - class Meta(object): - service = 'apigateway' - type = 'restapis' - enum_spec = ('get_rest_apis', 'items', None) - id = 'id' + service = "apigateway" + type = "restapis" + enum_spec = ("get_rest_apis", "items", None) + id = "id" filter_name = None filter_type = None detail_spec = None - name = 'name' - date = 'createdDate' - dimension = 'GatewayName' + name = "name" + date = "createdDate" + dimension = "GatewayName" @classmethod def filter(cls, arn, resource_id, data): api_id = data.get(cls.Meta.id) - LOG.debug('%s == %s', resource_id, api_id) + LOG.debug("%s == %s", resource_id, api_id) return resource_id == api_id + + @property + def arn(self): + # arn:aws:apigateway:us-east-1::/restapis/vwxyz12345 + return "arn:aws:apigateway:%s::restapis/%s" % ( + self._client.region_name, + self.id, + ) diff --git a/skew/resources/aws/autoscaling.py b/skew/resources/aws/autoscaling.py index 0d84ee2..668b587 100644 --- a/skew/resources/aws/autoscaling.py +++ b/skew/resources/aws/autoscaling.py @@ -1,5 +1,6 @@ # Copyright (c) 2014 Scopely, Inc. # Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -15,49 +16,76 @@ import jmespath from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient class AutoScalingGroup(AWSResource): - class Meta(object): - service = 'autoscaling' - type = 'autoScalingGroup' - name = 'AutoScalingGroupName' - date = 'CreatedTime' - dimension = 'AutoScalingGroupName' - enum_spec = ('describe_auto_scaling_groups', 'AutoScalingGroups', None) + service = "autoscaling" + type = "autoScalingGroup" + name = "AutoScalingGroupName" + date = "CreatedTime" + dimension = "AutoScalingGroupName" + enum_spec = ("describe_auto_scaling_groups", "AutoScalingGroups", None) detail_spec = None - id = 'AutoScalingGroupName' - filter_name = 'AutoScalingGroupNames' - filter_type = 'list' + id = "AutoScalingGroupName" + filter_name = "AutoScalingGroupNames" + filter_type = "list" def __init__(self, client, data, query=None): super(AutoScalingGroup, self).__init__(client, data, query) - self._arn_query = jmespath.compile('AutoScalingGroupARN') + self._arn_query = jmespath.compile("AutoScalingGroupARN") @property def arn(self): return self._arn_query.search(self.data) + def sleek(self): + # Always render lists in the same order to avoid false changes detection + self.data["EnabledMetrics"].sort(key=lambda item: item["Metric"]) + self.data["SuspendedProcesses"].sort(key=str) -class LaunchConfiguration(AWSResource): + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + client = get_awsclient(cls.Meta.service, region, account, **kwargs) + asg_name = arn.split(":")[7].split("/")[1] + addon = dict( + ResourceId=asg_name, + ResourceType="auto-scaling-group", + PropagateAtLaunch=False, + ) + tags_list = [dict(Key=k, Value=str(v), **addon) for k, v in tags.items()] + return client.call("create_or_update_tags", Tags=tags_list) + + @classmethod + def unset_tags(cls, arn, region, account, tag_keys, resource_id=None, **kwargs): + client = get_awsclient(cls.Meta.service, region, account, **kwargs) + asg_name = arn.split(":")[7].split("/")[1] + addon = dict( + ResourceId=asg_name, + ResourceType="auto-scaling-group", + PropagateAtLaunch=False, + ) + tags_list = [dict(Key=k, **addon) for k in tag_keys] + return client.call("delete_tags", Tags=tags_list) + +class LaunchConfiguration(AWSResource): class Meta(object): - service = 'autoscaling' - type = 'launchConfiguration' - name = 'LaunchConfigurationName' - date = 'CreatedTime' - dimension = 'AutoScalingGroupName' - enum_spec = ( - 'describe_launch_configurations', 'LaunchConfigurations', None) + service = "autoscaling" + type = "launchConfiguration" + name = "LaunchConfigurationName" + date = "CreatedTime" + dimension = "AutoScalingGroupName" + enum_spec = ("describe_launch_configurations", "LaunchConfigurations", None) detail_spec = None - id = 'LaunchConfigurationName' - filter_name = 'LaunchConfigurationNames' - filter_type = 'list' + id = "LaunchConfigurationName" + filter_name = "LaunchConfigurationNames" + filter_type = "list" def __init__(self, client, data, query=None): super(LaunchConfiguration, self).__init__(client, data, query) - self._arn_query = jmespath.compile('LaunchConfigurationARN') + self._arn_query = jmespath.compile("LaunchConfigurationARN") @property def arn(self): diff --git a/skew/resources/aws/cloudformation.py b/skew/resources/aws/cloudformation.py index 782af9f..14bfbdc 100644 --- a/skew/resources/aws/cloudformation.py +++ b/skew/resources/aws/cloudformation.py @@ -1,4 +1,5 @@ # Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -16,35 +17,22 @@ class Stack(AWSResource): - @classmethod def enumerate(cls, arn, region, account, resource_id=None, **kwargs): - resources = super(Stack, cls).enumerate(arn, region, account, - resource_id, **kwargs) - for stack in resources: - stack.data['Resources'] = [] - for stack_resource in stack: - resource_id = stack_resource.get('PhysicalResourceId') - if not resource_id: - resource_id = stack_resource.get('LogicalResourceId') - stack.data['Resources'].append( - { - 'id': resource_id, - 'type': stack_resource['ResourceType'] - } - ) + resources = list( + super(Stack, cls).enumerate(arn, region, account, resource_id, **kwargs) + ) return resources class Meta(object): - service = 'cloudformation' - type = 'stack' - enum_spec = ('describe_stacks', 'Stacks[]', None) - detail_spec = ('describe_stack_resources', 'StackName', - 'StackResources[]') - id = 'StackName' - filter_name = 'StackName' - name = 'StackName' - date = 'CreationTime' + service = "cloudformation" + type = "stack" + enum_spec = ("describe_stacks", "Stacks[]", None) + detail_spec = ("describe_stack_resources", "StackName", "StackResources[]") + id = "StackName" + filter_name = "StackName" + name = "StackName" + date = "CreationTime" dimension = None def __init__(self, client, data, query=None): @@ -63,4 +51,4 @@ def __iter__(self): @property def arn(self): - return self._data['StackId'] + return self._data["StackId"] diff --git a/skew/resources/aws/cloudfront.py b/skew/resources/aws/cloudfront.py index 5b096fa..ee97fa6 100644 --- a/skew/resources/aws/cloudfront.py +++ b/skew/resources/aws/cloudfront.py @@ -1,36 +1,47 @@ import logging from skew.resources.aws import AWSResource - +from skew.awsclient import get_awsclient LOG = logging.getLogger(__name__) class CloudfrontResource(AWSResource): - @property def arn(self): - return 'arn:aws:%s::%s:%s/%s' % ( + return "arn:aws:%s::%s:%s/%s" % ( self._client.service_name, - self._client.account_id, self.resourcetype, self.id) + self._client.account_id, + self.resourcetype, + self.id, + ) class Distribution(CloudfrontResource): - class Meta(object): - service = 'cloudfront' - type = 'distribution' - enum_spec = ('list_distributions', 'DistributionList.Items[]', None) + service = "cloudfront" + type = "distribution" + enum_spec = ("list_distributions", "DistributionList.Items[]", None) detail_spec = None - id = 'Id' - tags_spec = ('list_tags_for_resource', 'Tags.Items[]', - 'Resource', 'arn') - name = 'DomainName' + id = "Id" + tags_spec = ("list_tags_for_resource", "Tags.Items[]", "Resource", "arn") + name = "DomainName" filter_name = None - date = 'LastModifiedTime' + date = "LastModifiedTime" dimension = None @classmethod def filter(cls, arn, resource_id, data): - LOG.debug('%s == %s', resource_id, data) - return resource_id == data['Id'] + LOG.debug("%s == %s", resource_id, data) + return resource_id == data["Id"] + + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + client = get_awsclient(cls.Meta.service, region, account, **kwargs) + tags_list = [dict(Key=k, Value=str(v)) for k, v in tags.items()] + return client.call("tag_resource", Resource=arn, Tags=dict(Items=tags_list)) + + @classmethod + def unset_tags(cls, arn, region, account, tag_keys, resource_id=None, **kwargs): + client = get_awsclient(cls.Meta.service, region, account, **kwargs) + return client.call("untag_resource", Resource=arn, TagKeys=dict(Items=tag_keys)) diff --git a/skew/resources/aws/cloudsearch.py b/skew/resources/aws/cloudsearch.py new file mode 100644 index 0000000..5195d83 --- /dev/null +++ b/skew/resources/aws/cloudsearch.py @@ -0,0 +1,42 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2019 Christophe Morio +# Copyright (c) 2020 Jerome Guibert +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Domain(AWSResource): + class Meta(object): + service = "cloudsearch" + type = "domain" + enum_spec = ("describe_domains", "DomainStatusList", None) + detail_spec = None + id = "ARN" + tags_spec = None + filter_name = None + name = "DomainName" + date = "Created" + dimension = None + + @property + def arn(self): + return self._data["ARN"] diff --git a/skew/resources/aws/cloudtrail.py b/skew/resources/aws/cloudtrail.py index e4c82cb..6740e01 100644 --- a/skew/resources/aws/cloudtrail.py +++ b/skew/resources/aws/cloudtrail.py @@ -1,5 +1,7 @@ # Copyright (c) 2014 Scopely, Inc. # Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2019 Christophe Morio +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -13,24 +15,81 @@ # language governing permissions and limitations under the License. import jmespath import logging +from botocore.exceptions import ClientError from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient LOG = logging.getLogger(__name__) class CloudTrail(AWSResource): + @classmethod + def enumerate(cls, arn, region, account, resource_id=None, **kwargs): + client = get_awsclient(cls.Meta.service, region, account, **kwargs) + try: + data = client.call("list_trails", query="Trails[]") + if data: + if account and account != "*": + data = filter( + lambda d: account == d["TrailARN"].split(":")[4], data + ) + if region and region != "*": + data = filter(lambda d: region == d["HomeRegion"], data) + if resource_id and resource_id != "*": + data = filter(lambda d: cls.filter(arn, resource_id, d), data) + return map(lambda d: cls(client, d, arn.query), data) + except ClientError as e: + LOG.debug(e) + # if the error is because the resource was not found, be quiet + if "NotFound" not in e.response["Error"]["Code"]: + raise + return [] + + @classmethod + def filter(cls, arn, resource_id, data): + return resource_id == data["Name"] class Meta(object): - service = 'cloudtrail' - type = 'trail' - enum_spec = ('describe_trails', 'trailList[]', None) - attr_spec = None - detail_spec = None - id = 'Name' - tags_spec = ('list_tags', 'ResourceTagList[].TagsList[]', - 'ResourceIdList', 'name') - filter_name = 'trailNameList' - filter_type = 'arn' - name = 'TrailARN' + service = "cloudtrail" + type = "trail" + + filter_name = None + detail_spec = ("get_trail", "Trail", "Name", "name") + status_spec = ("get_trail_status", None, "Name", "name") + id = "Name" + name = "Name" + tags_spec = ( + "list_tags", + "ResourceTagList[].TagsList[]", + "ResourceIdList[]", + "name", + ) + date = None dimension = None + + def __init__(self, client, data, query=None): + super(CloudTrail, self).__init__(client, data, query) + self._data = { + **self._data, + **self._feed_from_spec(attr_spec=self.Meta.detail_spec), + } + self._data["Status"] = self._feed_from_spec(attr_spec=self.Meta.status_spec) + + @property + def arn(self): + return self._data["TrailARN"] + + @property + def tags(self): + if self._tags is None: + self._tags = {} + self._data["Tags"] = self._client.call( + "list_tags", + query="ResourceTagList[].TagsList[]", + ResourceIdList=[self.arn], + ) + if "Tags" in self._data: + self._tags = self._normalize_tags(self._data["Tags"]) + + return self._tags \ No newline at end of file diff --git a/skew/resources/aws/cloudwatch.py b/skew/resources/aws/cloudwatch.py index b0c6efc..078958f 100644 --- a/skew/resources/aws/cloudwatch.py +++ b/skew/resources/aws/cloudwatch.py @@ -1,5 +1,6 @@ # Copyright (c) 2014 Scopely, Inc. # Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -19,81 +20,134 @@ class Alarm(AWSResource): - class Meta(object): - service = 'cloudwatch' - type = 'alarm' - enum_spec = ('describe_alarms', 'MetricAlarms', None) - id = 'AlarmName' - filter_name = 'AlarmNames' + service = "cloudwatch" + type = "alarm" + enum_spec = ("describe_alarms", "MetricAlarms", None) + id = "AlarmName" + filter_name = "AlarmNames" filter_type = None detail_spec = None - name = 'AlarmName' - date = 'AlarmConfigurationUpdatedTimestamp' + name = "AlarmName" + date = "AlarmConfigurationUpdatedTimestamp" dimension = None - tags_spec = ('list_tags_for_resource', 'Tags[]', 'ResourceARN', 'arn') + tags_spec = ("list_tags_for_resource", "Tags[]", "ResourceARN", "arn") @property def arn(self): - return 'arn:aws:%s:%s:%s:%s:%s' % ( + return "arn:aws:%s:%s:%s:%s:%s" % ( self._client.service_name, self._client.region_name, self._client.account_id, - self.resourcetype, self.id) + self.resourcetype, + self.id, + ) class LogGroup(AWSResource): - class Meta(object): - service = 'logs' - type = 'log-group' - enum_spec = ('describe_log_groups', 'logGroups[]', None) - attr_spec = [ - ('describe_log_streams', 'logGroupName', - 'logStreams', 'logStreams'), - ('describe_metric_filters', 'logGroupName', - 'metricFilters', 'metricFilters'), - ('describe_subscription_filters', 'logGroupName', - 'subscriptionFilters', 'subscriptionFilters'), - ('describe_queries', 'logGroupName', - 'queries', 'queries'), - ] + service = "logs" + type = "log-group" + enum_spec = ("describe_log_groups", "logGroups[]", None) + attr_spec = { + "logStreams": ("describe_log_streams", "logStreams", "logGroupName", "id"), + "metricFilters": ( + "describe_metric_filters", + "metricFilters", + "logGroupName", + "id", + ), + "queries": ("describe_queries", "queries", "logGroupName", "id"), + "subscription": ( + "describe_subscription_filters", + "subscriptionFilters", + "logGroupName", + "id", + ), + } + detail_spec = None - id = 'logGroupName' - tags_spec = ('list_tags_log_group', 'tags', - 'logGroupName', 'id') - filter_name = 'logGroupNamePrefix' - filter_type = 'dict' - name = 'logGroupName' - date = 'creationTime' - dimension = 'logGroupName' + id = "logGroupName" + tags_spec = ("list_tags_log_group", "tags", "logGroupName", "id") + filter_name = "logGroupNamePrefix" + filter_type = "dict" + name = "logGroupName" + date = "creationTime" + dimension = "logGroupName" def __init__(self, client, data, query=None): super(LogGroup, self).__init__(client, data, query) - self._data = data self._keys = [] - self._id = data['logGroupName'] - - # add addition attribute data - for attr in self.Meta.attr_spec: - LOG.debug(attr) - detail_op, param_name, detail_path, detail_key = attr - params = {param_name: self._id} - data = self._client.call(detail_op, **params) - if not (detail_path is None): - data = jmespath.search(detail_path, data) - if 'ResponseMetadata' in data: - del data['ResponseMetadata'] - self.data[detail_key] = data - LOG.debug(data) + self._id = data["logGroupName"] + + @property + def log_streams(self): + if "logStreams" not in self._data: + self._data["logStreams"] = self._remove_response_metadata( + self._feed_from_spec(attr_spec=self.Meta.attr_spec["logStreams"]) + ) + return self._data["logStreams"] + + @property + def metric_filters(self): + if "metricFilters" not in self._data: + self._data["metricFilters"] = self._remove_response_metadata( + self._feed_from_spec(attr_spec=self.Meta.attr_spec["metricFilters"]) + ) + return self._data["metricFilters"] + + @property + def queries(self): + if "queries" not in self._data: + self._data["queries"] = self._remove_response_metadata( + self._feed_from_spec(attr_spec=self.Meta.attr_spec["queries"]) + ) + return self._data["queries"] + + @property + def subscriptions(self): + if "subscriptionFilters" not in self._data: + self._data["subscriptionFilters"] = self._remove_response_metadata( + self._feed_from_spec(attr_spec=self.Meta.attr_spec["subscription"]) + ) + return self._data["subscriptionFilters"] + + def _remove_response_metadata(self, data): + if "ResponseMetadata" in data: + del data["ResponseMetadata"] + return data @property def logGroupName(self): - return self.data.get('logGroupName') + return self.data.get("logGroupName") @property def arn(self): - return 'arn:aws:%s:%s:%s:%s:%s' % ( + return "arn:aws:%s:%s:%s:%s:%s" % ( self._client.service_name, self._client.region_name, - self._client.account_id, self.resourcetype, self.id) + self._client.account_id, + self.resourcetype, + self.id, + ) + + +class CloudWatchEventRule(AWSResource): + class Meta(object): + service = "events" + type = "rule" + enum_spec = ("list_rules", "Rules[]", None) + id = "Name" + filter_name = None + filter_type = None + name = "Name" + attr_spec = ("list_targets_by_rule", "Targets[]", "Rule", "id") + # tags_spec = ("list_tags_log_group", "tags", "logGroupName", "id") + + @classmethod + def filter(cls, arn, resource_id, data): + return resource_id == data["None"] + + def __init__(self, client, data, query=None): + super(CloudWatchEventRule, self).__init__(client, data, query) + self._data["Targets"] = self._feed_from_spec(attr_spec=self.Meta.attr_spec) diff --git a/skew/resources/aws/dynamodb.py b/skew/resources/aws/dynamodb.py index 196c6b7..b20f457 100644 --- a/skew/resources/aws/dynamodb.py +++ b/skew/resources/aws/dynamodb.py @@ -1,5 +1,6 @@ # Copyright (c) 2014 Scopely, Inc. # Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -23,29 +24,27 @@ class Table(AWSResource): - class Meta(object): - service = 'dynamodb' - type = 'table' - enum_spec = ('list_tables', 'TableNames', None) - detail_spec = ('describe_table', 'TableName', 'Table') - id = 'Table' - tags_spec = ('list_tags_of_resource', 'Tags[]', - 'ResourceArn', 'arn') + service = "dynamodb" + type = "table" + enum_spec = ("list_tables", "TableNames", None) + id = "TableName" + detail_spec = ("describe_table", "TableName", "Table") + tags_spec = ("list_tags_of_resource", "Tags[]", "ResourceArn", "arn") filter_name = None - name = 'TableName' - date = 'CreationDateTime' - dimension = 'TableName' + name = "TableName" + date = "CreationDateTime" + dimension = "TableName" @classmethod def filter(cls, arn, resource_id, data): - LOG.debug('%s == %s', resource_id, data) + LOG.debug("%s: %s == %s", arn, resource_id, data) return resource_id == data def __init__(self, client, data, query=None): - super(Table, self).__init__(client, data, query) - self._id = data + # data from list_tables operation is a table name + super(Table, self).__init__(client, data={"TableName": data}, query=query) detail_op, param_name, detail_path = self.Meta.detail_spec params = {param_name: self.id} data = client.call(detail_op, **params) - self.data = jmespath.search(detail_path, data) + self._data = jmespath.search(detail_path, data) diff --git a/skew/resources/aws/ec2.py b/skew/resources/aws/ec2.py index cd61c70..e8896ed 100644 --- a/skew/resources/aws/ec2.py +++ b/skew/resources/aws/ec2.py @@ -16,282 +16,267 @@ class Instance(AWSResource): - class Meta(object): - service = 'ec2' - type = 'instance' - enum_spec = ('describe_instances', 'Reservations[].Instances[]', None) + service = "ec2" + type = "instance" + enum_spec = ("describe_instances", "Reservations[].Instances[]", None) detail_spec = None - id = 'InstanceId' - filter_name = 'InstanceIds' - filter_type = 'list' - name = 'PublicDnsName' - date = 'LaunchTime' - dimension = 'InstanceId' + id = "InstanceId" + filter_name = "InstanceIds" + filter_type = "list" + name = "InstanceId" + date = "LaunchTime" + dimension = "InstanceId" @property def parent(self): - return self.data['ImageId'] + return self._data["ImageId"] + def __init__(self, client, data, query=None): + super(Instance, self).__init__(client, data, query) + # Asset name is get by tags if defined, or is InstanceId + self._name = self.tags.get("Name", self._data["InstanceId"]) -class SecurityGroup(AWSResource): +class SecurityGroup(AWSResource): class Meta(object): - service = 'ec2' - type = 'security-group' - enum_spec = ('describe_security_groups', 'SecurityGroups', None) + service = "ec2" + type = "security-group" + enum_spec = ("describe_security_groups", "SecurityGroups", None) detail_spec = None - id = 'GroupId' - filter_name = 'GroupNames' - filter_type = 'list' - name = 'GroupName' + id = "GroupId" + filter_name = "GroupNames" + filter_type = "list" + name = "GroupName" date = None dimension = None class KeyPair(AWSResource): - class Meta(object): - service = 'ec2' - type = 'key-pair' - enum_spec = ('describe_key_pairs', 'KeyPairs', None) + service = "ec2" + type = "key-pair" + enum_spec = ("describe_key_pairs", "KeyPairs", None) detail_spec = None - id = 'KeyPairId' - filter_name = 'KeyNames' - name = 'KeyName' + id = "KeyPairId" + filter_name = "KeyNames" + name = "KeyName" date = None dimension = None class Address(AWSResource): - class Meta(object): - service = 'ec2' - type = 'address' - enum_spec = ('describe_addresses', 'Addresses', None) + service = "ec2" + type = "address" + enum_spec = ("describe_addresses", "Addresses", None) detail_spec = None - id = 'AllocationId' - filter_name = 'PublicIps' - filter_type = 'list' - name = 'PublicIp' + id = "AllocationId" + filter_name = "PublicIps" + filter_type = "list" + name = "PublicIp" date = None dimension = None class Volume(AWSResource): - class Meta(object): - service = 'ec2' - type = 'volume' - enum_spec = ('describe_volumes', 'Volumes', None) + service = "ec2" + type = "volume" + enum_spec = ("describe_volumes", "Volumes", None) detail_spec = None - id = 'VolumeId' - filter_name = 'VolumeIds' - filter_type = 'list' - name = 'VolumeId' - date = 'createTime' - dimension = 'VolumeId' + id = "VolumeId" + filter_name = "VolumeIds" + filter_type = "list" + name = "VolumeId" + date = "createTime" + dimension = "VolumeId" @property def parent(self): - if len(self.data['Attachments']): - return self.data['Attachments'][0]['InstanceId'] + if len(self.data["Attachments"]): + return self.data["Attachments"][0]["InstanceId"] else: return None class Snapshot(AWSResource): - class Meta(object): - service = 'ec2' - type = 'snapshot' - enum_spec = ( - 'describe_snapshots', 'Snapshots', {'OwnerIds': ['self']}) + service = "ec2" + type = "snapshot" + enum_spec = ("describe_snapshots", "Snapshots", {"OwnerIds": ["self"]}) detail_spec = None - id = 'SnapshotId' - filter_name = 'SnapshotIds' - filter_type = 'list' - name = 'SnapshotId' - date = 'StartTime' + id = "SnapshotId" + filter_name = "SnapshotIds" + filter_type = "list" + name = "SnapshotId" + date = "StartTime" dimension = None @property def parent(self): - if self.data['VolumeId']: - return self.data['VolumeId'] + if self.data["VolumeId"]: + return self.data["VolumeId"] else: return None class Image(AWSResource): - class Meta(object): - service = 'ec2' - type = 'image' - enum_spec = ( - 'describe_images', 'Images', {'Owners': ['self']}) + service = "ec2" + type = "image" + enum_spec = ("describe_images", "Images", {"Owners": ["self"]}) detail_spec = None - id = 'ImageId' - filter_name = 'ImageIds' - filter_type = 'list' - name = 'ImageId' - date = 'StartTime' + id = "ImageId" + filter_name = "ImageIds" + filter_type = "list" + name = "ImageId" + date = "StartTime" dimension = None @property def parent(self): - if self.data['VolumeId']: - return self.data['VolumeId'] + if self.data["VolumeId"]: + return self.data["VolumeId"] else: return None class Vpc(AWSResource): - class Meta(object): - service = 'ec2' - type = 'vpc' - enum_spec = ('describe_vpcs', 'Vpcs', None) + service = "ec2" + type = "vpc" + enum_spec = ("describe_vpcs", "Vpcs", None) detail_spec = None - id = 'VpcId' - filter_name = 'VpcIds' - filter_type = 'list' - name = 'VpcId' + id = "VpcId" + filter_name = "VpcIds" + filter_type = "list" + name = "VpcId" date = None dimension = None class Subnet(AWSResource): - class Meta(object): - service = 'ec2' - type = 'subnet' - enum_spec = ('describe_subnets', 'Subnets', None) + service = "ec2" + type = "subnet" + enum_spec = ("describe_subnets", "Subnets", None) detail_spec = None - id = 'SubnetId' - filter_name = 'SubnetIds' - filter_type = 'list' - name = 'SubnetId' + id = "SubnetId" + filter_name = "SubnetIds" + filter_type = "list" + name = "SubnetId" date = None dimension = None class CustomerGateway(AWSResource): - class Meta(object): - service = 'ec2' - type = 'customer-gateway' - enum_spec = ('describe_customer_gateways', 'CustomerGateways', None) + service = "ec2" + type = "customer-gateway" + enum_spec = ("describe_customer_gateways", "CustomerGateways", None) detail_spec = None - id = 'CustomerGatewayId' - filter_name = 'CustomerGatewayIds' - filter_type = 'list' - name = 'CustomerGatewayId' + id = "CustomerGatewayId" + filter_name = "CustomerGatewayIds" + filter_type = "list" + name = "CustomerGatewayId" date = None dimension = None class InternetGateway(AWSResource): - class Meta(object): - service = 'ec2' - type = 'internet-gateway' - enum_spec = ('describe_internet_gateways', 'InternetGateways', None) + service = "ec2" + type = "internet-gateway" + enum_spec = ("describe_internet_gateways", "InternetGateways", None) detail_spec = None - id = 'InternetGatewayId' - filter_name = 'InternetGatewayIds' - filter_type = 'list' - name = 'InternetGatewayId' + id = "InternetGatewayId" + filter_name = "InternetGatewayIds" + filter_type = "list" + name = "InternetGatewayId" date = None dimension = None class RouteTable(AWSResource): - class Meta(object): - service = 'ec2' - type = 'route-table' - enum_spec = ('describe_route_tables', 'RouteTables', None) + service = "ec2" + type = "route-table" + enum_spec = ("describe_route_tables", "RouteTables", None) detail_spec = None - id = 'RouteTableId' - filter_name = 'RouteTableIds' - filter_type = 'list' - name = 'RouteTableId' + id = "RouteTableId" + filter_name = "RouteTableIds" + filter_type = "list" + name = "RouteTableId" date = None dimension = None class NatGateway(AWSResource): - class Meta(object): - service = 'ec2' - type = 'natgateway' - enum_spec = ('describe_nat_gateways', 'NatGateways', None) + service = "ec2" + type = "natgateway" + enum_spec = ("describe_nat_gateways", "NatGateways", None) detail_spec = None - id = 'NatGatewayId' - filter_name = 'NatGatewayIds' - filter_type = 'list' - name = 'NatGatewayId' - date = 'CreateTime' + id = "NatGatewayId" + filter_name = "NatGatewayIds" + filter_type = "list" + name = "NatGatewayId" + date = "CreateTime" dimension = None class NetworkAcl(AWSResource): - class Meta(object): - service = 'ec2' - type = 'network-acl' - enum_spec = ('describe_network_acls', 'NetworkAcls', None) + service = "ec2" + type = "network-acl" + enum_spec = ("describe_network_acls", "NetworkAcls", None) detail_spec = None - id = 'NetworkAclId' - filter_name = 'NetworkAclIds' - filter_type = 'list' - name = 'NetworkAclId' + id = "NetworkAclId" + filter_name = "NetworkAclIds" + filter_type = "list" + name = "NetworkAclId" date = None dimension = None class VpcPeeringConnection(AWSResource): - class Meta(object): - service = 'ec2' - type = 'vpc-peering-connection' - enum_spec = ('describe_vpc_peering_connections', - 'VpcPeeringConnections', None) + service = "ec2" + type = "vpc-peering-connection" + enum_spec = ("describe_vpc_peering_connections", "VpcPeeringConnections", None) detail_spec = None - id = 'VpcPeeringConnectionId' - filter_name = 'VpcPeeringConnectionIds' - filter_type = 'list' - name = 'VpcPeeringConnectionId' + id = "VpcPeeringConnectionId" + filter_name = "VpcPeeringConnectionIds" + filter_type = "list" + name = "VpcPeeringConnectionId" date = None dimension = None class LaunchTemplate(AWSResource): - class Meta(object): - service = 'ec2' - type = 'launch-template' - enum_spec = ('describe_launch_templates', 'LaunchTemplates', None) + service = "ec2" + type = "launch-template" + enum_spec = ("describe_launch_templates", "LaunchTemplates", None) detail_spec = None - id = 'LaunchTemplateId' - filter_name = 'LaunchTemplateIds' - filter_type = 'list' - name = 'LaunchTemplateName' - date = 'CreateTime' + id = "LaunchTemplateId" + filter_name = "LaunchTemplateIds" + filter_type = "list" + name = "LaunchTemplateName" + date = "CreateTime" dimension = None class FlowLog(AWSResource): - class Meta(object): - service = 'ec2' - type = 'flow-log' - enum_spec = ('describe_flow_logs', 'FlowLogs', None) + service = "ec2" + type = "flow-log" + enum_spec = ("describe_flow_logs", "FlowLogs", None) detail_spec = None - id = 'FlowLogId' - filter_name = 'FlowLogIds' - filter_type = 'list' - name = 'LogGroupName' - date = 'CreationTime' + id = "FlowLogId" + filter_name = "FlowLogIds" + filter_type = "list" + name = "LogGroupName" + date = "CreationTime" dimension = None diff --git a/skew/resources/aws/ecr.py b/skew/resources/aws/ecr.py new file mode 100644 index 0000000..c760895 --- /dev/null +++ b/skew/resources/aws/ecr.py @@ -0,0 +1,85 @@ +# Copyright (c) 2020 Jerome Guibert +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from botocore.exceptions import ClientError +from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient + + +LOG = logging.getLogger(__name__) + + +class Registery(AWSResource): + @classmethod + def enumerate(cls, arn, region, account, resource_id=None, **kwargs): + client = get_awsclient(cls.Meta.service, region, account, **kwargs) + try: + data = client.call("describe_registry") + if data: + if "ResponseMetadata" in data: + del data["ResponseMetadata"] + data[ + "RegisteryUri" + ] = f"{data['registryId']}.dkr.ecr.{region}.amazonaws.com" + + return [Registery(client, data, arn.query)] + except ClientError as e: + LOG.debug(e) + # if the error is because the resource was not found, be quiet + if "NotFound" not in e.response["Error"]["Code"]: + raise + return [] + + class Meta(object): + service = "ecr" + type = "registery" + id = "registryId" + + +class Repository(AWSResource): + @classmethod + def enumerate(cls, arn, region, account, resource_id=None, **kwargs): + client = get_awsclient(cls.Meta.service, region, account, **kwargs) + try: + param = {"registryId": account} + if resource_id and resource_id != "*": + param.update({"repositoryNames": [resource_id]}) + data = client.call("describe_repositories", query="repositories[]", **param) + + if data: + if "ResponseMetadata" in data: + del data["ResponseMetadata"] + + return map(lambda d: Repository(client, d, arn.query), data) + except ClientError as e: + LOG.debug(e) + # if the error is because the resource was not found, be quiet + if "NotFound" not in e.response["Error"]["Code"]: + raise + return [] + + class Meta(object): + service = "ecr" + type = "repository" + id = "repositoryName" + + def __init__(self, client, data, query=None): + super(Repository, self).__init__(client, data, query) + + @property + def arn(self): + return self._data["repositoryArn"] \ No newline at end of file diff --git a/skew/resources/aws/ecs.py b/skew/resources/aws/ecs.py new file mode 100644 index 0000000..3fefbae --- /dev/null +++ b/skew/resources/aws/ecs.py @@ -0,0 +1,90 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Cluster(AWSResource): + class Meta(object): + service = "ecs" + type = "cluster" + enum_spec = ("list_clusters", "clusterArns", None) + detail_spec = ("describe_clusters", "clusters", "clusters[0]") + id = None + tags_spec = ("list_tags_for_resource", "tags[]", "resourceArn", "arn") + + filter_name = None + name = "clusterName" + date = None + dimension = None + + attr_spec = ("list_services", "serviceArns[]", "cluster", "id") + + @property + def arn(self): + return self.data["clusterArn"] + + def __init__(self, client, data, query=None): + super(Cluster, self).__init__(client, data, query) + self._id = data + detail_op, param_name, detail_path = self.Meta.detail_spec + params = {param_name: [self.id]} + data = client.call(detail_op, **params) + self._data = jmespath.search(detail_path, data) + + service_arns = self._feed_from_spec(attr_spec=self.Meta.attr_spec) + self._data["services"] = {} + for service_arn in service_arns: + kwargs = { + "cluster": self._id, + "services": [service_arn], + "include": ["TAGS"], + } + service_def = self._client.call( + "describe_services", query="services[0]", **kwargs + ) + self._data["services"][service_def["serviceName"]] = service_def + + +class TaskDefinition(AWSResource): + class Meta(object): + service = "ecs" + type = "task-definition" + enum_spec = ("list_task_definitions", "taskDefinitionArns", None) + detail_spec = ("describe_task_definition", "taskDefinition", "taskDefinition") + id = None + name = None + filter_name = None + date = None + dimension = None + + def __init__(self, client, data, query=None): + super(TaskDefinition, self).__init__(client, data, query) + self._id = data + detail_op, param_name, detail_path = self.Meta.detail_spec + params = {param_name: self.id} + data = client.call(detail_op, **params) + self._data = jmespath.search(detail_path, data) + + @property + def arn(self): + return self._data["taskDefinitionArn"] diff --git a/skew/resources/aws/efs.py b/skew/resources/aws/efs.py new file mode 100644 index 0000000..e1fa2cb --- /dev/null +++ b/skew/resources/aws/efs.py @@ -0,0 +1,71 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient + + +LOG = logging.getLogger(__name__) + + +class Filesystem(AWSResource): + class Meta(object): + service = "efs" + type = "filesystem" + enum_spec = ("describe_file_systems", "FileSystems", None) + detail_spec = None + id = "FileSystemId" + tags_spec = ("describe_tags", "Tags[]", "FileSystemId", "id") + filter_name = None + name = "Name" + date = "CreationTime" + dimension = None + + @property + def arn(self): + # arn:aws:elasticfilesystem:us-east-1:123456789012:file-system-id/fs12345678 + return "arn:aws:%s:%s:%s:%s/%s" % ( + "elasticfilesystem", + self._client.region_name, + self._client.account_id, + "file-system-id", + self.id, + ) + + def __init__(self, client, data, query=None): + super(Filesystem, self).__init__(client, data, query) + # Asset name is get by tags if defined, or is FileSystemId + self._name = self.tags.get("Name", self.data["FileSystemId"]) + + def sleek(self): + self._data["SizeInBytes"] = 0 + + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + client = get_awsclient(cls.Meta.service, region, account, **kwargs) + tags_list = [dict(Key=k, Value=str(v)) for k, v in tags.items()] + x = client.call("create_tags", FileSystemId=arn.split("/")[-1], Tags=tags_list) + return x + + @classmethod + def unset_tags(cls, arn, region, account, tag_keys, resource_id=None, **kwargs): + client = get_awsclient(cls.Meta.service, region, account, **kwargs) + x = client.call( + "delete_tags", FileSystemId=arn.split("/")[-1], TagKeys=tag_keys + ) + return x diff --git a/skew/resources/aws/elasticbeanstalk.py b/skew/resources/aws/elasticbeanstalk.py index 33a60b6..0d21e5b 100644 --- a/skew/resources/aws/elasticbeanstalk.py +++ b/skew/resources/aws/elasticbeanstalk.py @@ -50,6 +50,6 @@ def arn(self): self._client.region_name, self._client.account_id, self.resourcetype, - self.data['ApplicationName'], + self._data['ApplicationName'], self.id ) diff --git a/skew/resources/aws/elb.py b/skew/resources/aws/elb.py index 0f11451..51612ad 100644 --- a/skew/resources/aws/elb.py +++ b/skew/resources/aws/elb.py @@ -1,5 +1,7 @@ # Copyright (c) 2014 Scopely, Inc. # Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2019 Christophe Morio +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -19,31 +21,41 @@ class LoadBalancer(AWSResource): - class Meta(object): - service = 'elb' - type = 'loadbalancer' - enum_spec = ('describe_load_balancers', - 'LoadBalancerDescriptions', None) + service = "elb" + type = "loadbalancer" + enum_spec = ("describe_load_balancers", "LoadBalancerDescriptions", None) detail_spec = None attr_spec = [ - ('describe_load_balancer_attributes', 'LoadBalancerName', - 'LoadBalancerAttributes', 'LoadBalancerAttributes'), - ('describe_load_balancer_policies', 'LoadBalancerName', - 'PolicyDescriptions', 'PolicyDescriptions'), + ( + "describe_load_balancer_attributes", + "LoadBalancerName", + "LoadBalancerAttributes", + "LoadBalancerAttributes", + ), + ( + "describe_load_balancer_policies", + "LoadBalancerName", + "PolicyDescriptions", + "PolicyDescriptions", + ), ] - id = 'LoadBalancerName' - filter_name = 'LoadBalancerNames' - filter_type = 'list' - name = 'DNSName' - date = 'CreatedTime' - dimension = 'LoadBalancerName' - tags_spec = ('describe_tags', 'TagDescriptions[].Tags[]', - 'LoadBalancerNames', 'id') + id = "LoadBalancerName" + filter_name = "LoadBalancerNames" + filter_type = "list" + name = "DNSName" + date = "CreatedTime" + dimension = "LoadBalancerName" + tags_spec = ( + "describe_tags", + "TagDescriptions[].Tags[]", + "LoadBalancerNames", + "id", + ) def __init__(self, client, data, query=None): super(LoadBalancer, self).__init__(client, data, query) - self._id = data['LoadBalancerName'] + self._id = data["LoadBalancerName"] # add addition attribute data for attr in self.Meta.attr_spec: @@ -53,7 +65,16 @@ def __init__(self, client, data, query=None): data = self._client.call(detail_op, **params) if not (detail_path is None): data = jmespath.search(detail_path, data) - if 'ResponseMetadata' in data: - del data['ResponseMetadata'] - self.data[detail_key] = data + if "ResponseMetadata" in data: + del data["ResponseMetadata"] + self._data[detail_key] = data LOG.debug(data) + + @property + def arn(self): + return "arn:aws:elb:%s:%s:%s/%s" % ( + self._client.region_name, + self._client.account_id, + self.resourcetype, + self.id, + ) diff --git a/skew/resources/aws/elbv2.py b/skew/resources/aws/elbv2.py new file mode 100644 index 0000000..20f8768 --- /dev/null +++ b/skew/resources/aws/elbv2.py @@ -0,0 +1,70 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2019 Christophe Morio +# Copyright (c) 2020 Jerome Guibert +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import jmespath + +from skew.resources.aws import AWSResource + + +class LoadBalancer(AWSResource): + class Meta(object): + service = "elbv2" + type = "loadbalancer" + enum_spec = ("describe_load_balancers", "LoadBalancers", None) + detail_spec = ("describe_listeners", "LoadBalancerArn", "Listeners") + id = "LoadBalancerArn" + filter_name = "Names" + filter_type = "list" + name = "LoadBalancerName" + date = "CreatedTime" + dimension = None + tags_spec = ("describe_tags", "TagDescriptions[].Tags[]", "ResourceArns", "id") + + @property + def arn(self): + return self._data["LoadBalancerArn"] + + def __init__(self, client, data, query=None): + super(LoadBalancer, self).__init__(client, data, query) + if data and "LoadBalancerArn" in data: + detail_op, param_name, detail_path = self.Meta.detail_spec + params = {param_name: self._data["LoadBalancerArn"]} + data = client.call(detail_op, **params) + self._data["Listeners"] = jmespath.search(detail_path, data) + + +class TargetGroup(AWSResource): + class Meta(object): + service = "elbv2" + type = "targetgroup" + enum_spec = ("describe_target_groups", "TargetGroups", None) + detail_spec = None + id = "TargetGroupArn" + filter_name = "Names" + filter_type = "list" + name = "TargetGroupName" + date = "CreatedTime" + dimension = "LoadBalancerName" + tags_spec = ( + "describe_tags", + "TagDescriptions[].Tags[]", + "LoadBalancerNames", + "id", + ) + + @property + def arn(self): + return self._data["TargetGroupArn"] diff --git a/skew/resources/aws/es.py b/skew/resources/aws/es.py index 3243ebf..4c06c54 100644 --- a/skew/resources/aws/es.py +++ b/skew/resources/aws/es.py @@ -38,4 +38,4 @@ def __init__(self, client, data, query=None): detail_op, param_name, detail_path = self.Meta.detail_spec params = {param_name: self.id} data = client.call(detail_op, **params) - self.data = jmespath.search(detail_path, data) + self._data = jmespath.search(detail_path, data) diff --git a/skew/resources/aws/firehose.py b/skew/resources/aws/firehose.py index c4c8600..6045edc 100644 --- a/skew/resources/aws/firehose.py +++ b/skew/resources/aws/firehose.py @@ -34,4 +34,4 @@ def __init__(self, client, data, query=None): detail_op, param_name, detail_path = self.Meta.detail_spec params = {param_name: self.id} data = client.call(detail_op, **params) - self.data = jmespath.search(detail_path, data) + self._data = jmespath.search(detail_path, data) diff --git a/skew/resources/aws/iam.py b/skew/resources/aws/iam.py index fbbed17..f650f65 100644 --- a/skew/resources/aws/iam.py +++ b/skew/resources/aws/iam.py @@ -1,5 +1,6 @@ # Copyright (c) 2014 Scopely, Inc. # Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -20,60 +21,81 @@ class IAMResource(AWSResource): - @property def arn(self): - return 'arn:aws:%s::%s:%s/%s' % ( + return "arn:aws:%s::%s:%s/%s" % ( self._client.service_name, - self._client.account_id, self.resourcetype, self.name) + self._client.account_id, + self.resourcetype, + self.name, + ) class Group(IAMResource): - class Meta(object): - service = 'iam' - type = 'group' - enum_spec = ('list_groups', 'Groups', None) + service = "iam" + type = "group" + enum_spec = ("list_groups", "Groups", None) detail_spec = None - id = 'GroupId' - name = 'GroupName' + id = "GroupId" + name = "GroupName" filter_name = None - date = 'CreateDate' + date = "CreateDate" dimension = None + attr_spec = { + "users": ("get_group", "Users", "GroupName", "name"), + "policy_names": ("list_group_policies", "PolicyNames", "GroupName", "name"), + "attached_group_policies": ( + "list_attached_group_policies", + "AttachedPolicies", + "GroupName", + "name", + ), + } + @classmethod def filter(cls, arn, resource_id, data): - LOG.debug('%s == %s', resource_id, data) - return resource_id == data['GroupName'] + LOG.debug("%s == %s", resource_id, data) + return resource_id == data["GroupName"] + def _load_extra_attribute(self): + self._data["Users"] = self._feed_from_spec( + attr_spec=self.Meta.attr_spec["users"] + ) + self._data["PolicyNames"] = self._feed_from_spec( + attr_spec=self.Meta.attr_spec["policy_names"] + ) + self._data["AttachedPolicies"] = self._feed_from_spec( + attr_spec=self.Meta.attr_spec["attached_group_policies"] + ) -class User(IAMResource): +class User(IAMResource): class Meta(object): - service = 'iam' - type = 'user' - enum_spec = ('list_users', 'Users', None) - detail_spec = ('get_user', 'UserName', 'User') + service = "iam" + type = "user" + enum_spec = ("list_users", "Users", None) + detail_spec = ("get_user", "UserName", "User") attr_spec = [ - ('list_access_keys', 'UserName', - 'AccessKeyMetadata', 'AccessKeyMetadata'), - ('list_groups_for_user', 'UserName', - 'Groups', 'Groups'), - ('list_user_policies', 'UserName', - 'PolicyNames', 'PolicyNames'), - ('list_attached_user_policies', 'UserName', - 'AttachedPolicies', 'AttachedPolicies'), - ('list_ssh_public_keys', 'UserName', - 'SSHPublicKeys', 'SSHPublicKeys'), + ("list_access_keys", "UserName", "AccessKeyMetadata", "AccessKeyMetadata"), + ("list_groups_for_user", "UserName", "Groups", "Groups"), + ("list_user_policies", "UserName", "PolicyNames", "PolicyNames"), + ( + "list_attached_user_policies", + "UserName", + "AttachedPolicies", + "AttachedPolicies", + ), + ("list_ssh_public_keys", "UserName", "SSHPublicKeys", "SSHPublicKeys"), # ('list_mfa_devices', 'UserName', 'MFADevices', 'MFADevices'), ] - id = 'UserId' + id = "UserId" filter_name = None - name = 'UserName' - date = 'CreateDate' + name = "UserName" + date = "CreateDate" dimension = None - tags_spec = ('list_user_tags', 'Tags[]', - 'UserName', 'name') + tags_spec = ("list_user_tags", "Tags[]", "UserName", "name") def __init__(self, client, data, query=None): super(User, self).__init__(client, data, query) @@ -82,9 +104,9 @@ def __init__(self, client, data, query=None): # add details if self.Meta.detail_spec is not None: detail_op, param_name, detail_path = self.Meta.detail_spec - params = {param_name: self.data[param_name]} + params = {param_name: self._data[param_name]} data = client.call(detail_op, **params) - self.data = jmespath.search(detail_path, data) + self._data = jmespath.search(detail_path, data) # add attribute data if self.Meta.attr_spec is not None: @@ -92,109 +114,102 @@ def __init__(self, client, data, query=None): LOG.debug(attr) LOG.debug(data) detail_op, param_name, detail_path, detail_key = attr - params = {param_name: self.data[param_name]} + params = {param_name: self._data[param_name]} tmp_data = self._client.call(detail_op, **params) if not (detail_path is None): tmp_data = jmespath.search(detail_path, tmp_data) - if 'ResponseMetadata' in tmp_data: - del tmp_data['ResponseMetadata'] - self.data[detail_key] = tmp_data + if "ResponseMetadata" in tmp_data: + del tmp_data["ResponseMetadata"] + self._data[detail_key] = tmp_data LOG.debug(data) # retrieve all of the inline IAM policies - if 'PolicyNames' in self.data \ - and self.data['PolicyNames']: + if "PolicyNames" in self._data and self._data["PolicyNames"]: tmp_dict = {} - for policy_name in self.data['PolicyNames']: + for policy_name in self._data["PolicyNames"]: params = { - 'UserName': self.data['UserName'], - 'PolicyName': policy_name + "UserName": self._data["UserName"], + "PolicyName": policy_name, } - tmp_data = self._client.call('get_user_policy', **params) - tmp_data = jmespath.search('PolicyDocument', tmp_data) + tmp_data = self._client.call("get_user_policy", **params) + tmp_data = jmespath.search("PolicyDocument", tmp_data) tmp_dict[policy_name] = tmp_data - self.data['PolicyNames'] = tmp_dict + self._data["PolicyNames"] = tmp_dict @classmethod def filter(cls, arn, resource_id, data): - LOG.debug('%s == %s', resource_id, data) - return resource_id == data['UserName'] + LOG.debug("%s == %s", resource_id, data) + return resource_id == data["UserName"] class Role(IAMResource): - class Meta(object): - service = 'iam' - type = 'role' - enum_spec = ('list_roles', 'Roles', None) + service = "iam" + type = "role" + enum_spec = ("list_roles", "Roles", None) detail_spec = None - id = 'RoleId' + id = "RoleId" filter_name = None - name = 'RoleName' - date = 'CreateDate' + name = "RoleName" + date = "CreateDate" dimension = None - tags_spec = ('list_role_tags', 'Tags[]', 'RoleName', 'name') + tags_spec = ("list_role_tags", "Tags[]", "RoleName", "name") @classmethod def filter(cls, arn, resource_id, data): - LOG.debug('%s == %s', resource_id, data) - return resource_id == data['RoleName'] + LOG.debug("%s == %s", resource_id, data) + return resource_id == data["RoleName"] class InstanceProfile(IAMResource): - class Meta(object): - service = 'iam' - type = 'instance-profile' - enum_spec = ('list_instance_profiles', 'InstanceProfiles', None) + service = "iam" + type = "instance-profile" + enum_spec = ("list_instance_profiles", "InstanceProfiles", None) detail_spec = None - id = 'InstanceProfileId' + id = "InstanceProfileId" filter_name = None - name = 'InstanceProfileId' - date = 'CreateDate' + name = "InstanceProfileId" + date = "CreateDate" dimension = None @classmethod def filter(cls, arn, resource_id, data): - LOG.debug('%s == %s', resource_id, data) - return resource_id == data['InstanceProfileId'] + LOG.debug("%s == %s", resource_id, data) + return resource_id == data["InstanceProfileId"] class Policy(IAMResource): - class Meta(object): - service = 'iam' - type = 'policy' - enum_spec = ('list_policies', 'Policies', None) + service = "iam" + type = "policy" + enum_spec = ("list_policies", "Policies", None) detail_spec = None - id = 'PolicyArn' + id = "PolicyArn" filter_name = None - name = 'PolicyName' - date = 'CreateDate' + name = "PolicyName" + date = "CreateDate" dimension = None @classmethod def filter(cls, arn, resource_id, data): - LOG.debug('%s == %s', resource_id, data) - return resource_id == data['PolicyName'] + LOG.debug("%s == %s", resource_id, data) + return resource_id == data["PolicyName"] class ServerCertificate(IAMResource): - class Meta(object): - service = 'iam' - type = 'server-certificate' - enum_spec = ('list_server_certificates', - 'ServerCertificateMetadataList', - None) + service = "iam" + type = "server-certificate" + enum_spec = ("list_server_certificates", "ServerCertificateMetadataList", None) detail_spec = None - id = 'ServerCertificateId' + id = "ServerCertificateId" filter_name = None - name = 'ServerCertificateName' - date = 'Expiration' + name = "ServerCertificateName" + date = "Expiration" dimension = None @classmethod def filter(cls, arn, resource_id, data): - LOG.debug('%s == %s', resource_id, data) - return resource_id == data['ServerCertificateName'] + LOG.debug("%s == %s", resource_id, data) + return resource_id == data["ServerCertificateName"] diff --git a/skew/resources/aws/kinesis.py b/skew/resources/aws/kinesis.py index 69f009c..64200c5 100644 --- a/skew/resources/aws/kinesis.py +++ b/skew/resources/aws/kinesis.py @@ -1,5 +1,6 @@ # Copyright (c) 2014 Scopely, Inc. # Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -16,22 +17,22 @@ class Stream(AWSResource): - class Meta(object): - service = 'kinesis' - type = 'stream' - enum_spec = ('list_streams', 'StreamNames', None) - detail_spec = None - id = 'StreamName' + service = "kinesis" + type = "stream" + enum_spec = ("list_streams", "StreamNames", None) + detail_spec = ("describe_stream", "StreamDescription", "StreamName", "name") + + id = "StreamName" filter_name = None filter_type = None - name = 'StreamName' + name = "StreamName" date = None - dimension = 'StreamName' - tags_spec = ('list_tags_for_stream', 'Tags[]', - 'StreamName', 'id') + dimension = "StreamName" + tags_spec = ("list_tags_for_stream", "Tags[]", "StreamName", "id") def __init__(self, client, data, query=None): super(Stream, self).__init__(client, data, query) - self.data = {self.Meta.id: data} - self._id = self.data[self.Meta.id] + self._data = {self.Meta.id: data} + self._id = self._data[self.Meta.id] + self._data = self._feed_from_spec(attr_spec=self.Meta.detail_spec) diff --git a/skew/resources/aws/kms.py b/skew/resources/aws/kms.py new file mode 100644 index 0000000..e6c07b5 --- /dev/null +++ b/skew/resources/aws/kms.py @@ -0,0 +1,66 @@ +# Copyright (c) 2020 Jerome Guibert +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from botocore.exceptions import ClientError +from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient + + +LOG = logging.getLogger(__name__) + + +class Key(AWSResource): + class Meta(object): + service = "kms" + type = "key" + + enum_spec = ("list_keys", "Keys[]", None) + filter_name = None + id = "KeyId" + + attr_spec = { + "describe": ("describe_key", "KeyMetadata", "KeyId", "id"), + "key_policy": ("get_key_policy", "Policy", "KeyId", "id"), + "key_rotation_status": ( + "get_key_rotation_status", + "KeyRotationEnabled", + "KeyId", + "id", + ), + "aliases": ("list_aliases", "Aliases[]", "KeyId", "id"), + } + + tags_spec = ("list_resource_tags", "Tags[]", "KeyId", "arn") + + @classmethod + def filter(cls, arn, resource_id, data): + return resource_id == data["KeyId"] + + def __init__(self, client, data, query=None): + super(Key, self).__init__(client, data, query) + self._data["KeyMetadata"] = self._feed_from_spec( + attr_spec=self.Meta.attr_spec["describe"] + ) + self._data["Policy"] = self._feed_from_spec( + attr_spec=self.Meta.attr_spec["key_policy"] + ) + self._data["KeyRotationEnabled"] = self._feed_from_spec( + attr_spec=self.Meta.attr_spec["key_rotation_status"] + ) + self._data["Aliases"] = self._feed_from_spec( + attr_spec=self.Meta.attr_spec["aliases"] + ) diff --git a/skew/resources/aws/lambda.py b/skew/resources/aws/lambda.py index c42cc87..e16cea3 100644 --- a/skew/resources/aws/lambda.py +++ b/skew/resources/aws/lambda.py @@ -1,5 +1,6 @@ # Copyright (c) 2014 Scopely, Inc. # Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -21,38 +22,38 @@ class Function(AWSResource): - @classmethod def enumerate(cls, arn, region, account, resource_id=None, **kwargs): - resources = super(Function, cls).enumerate(arn, region, account, - resource_id, **kwargs) + resources = list( + super(Function, cls).enumerate(arn, region, account, resource_id, **kwargs) + ) for r in resources: - r.data['EventSources'] = [] - kwargs = {'FunctionName': r.data['FunctionName']} - response = r._client.call('list_event_source_mappings', **kwargs) - for esm in response['EventSourceMappings']: - r.data['EventSources'].append(esm['EventSourceArn']) + r.data["EventSources"] = [] + kwargs = {"FunctionName": r.data["FunctionName"]} + response = r._client.call("list_event_source_mappings", **kwargs) + for esm in response["EventSourceMappings"]: + r.data["EventSources"].append(esm["EventSourceArn"]) return resources class Meta(object): - service = 'lambda' - type = 'function' - enum_spec = ('list_functions', 'Functions', None) + service = "lambda" + type = "function" + enum_spec = ("list_functions", "Functions", None) detail_spec = None - id = 'FunctionName' + + id = "FunctionName" filter_name = None - name = 'FunctionName' - date = 'LastModified' - dimension = 'FunctionName' - tags_spec = ('list_tags', 'Tags', - 'Resource', 'arn') + name = "FunctionName" + date = "LastModified" + dimension = "FunctionName" + tags_spec = ("list_tags", "Tags", "Resource", "arn") @classmethod def filter(cls, arn, resource_id, data): function_name = data.get(cls.Meta.id) - LOG.debug('%s == %s', resource_id, function_name) + LOG.debug("%s == %s", resource_id, function_name) return resource_id == function_name @property def arn(self): - return self.data.get('FunctionArn') + return self._data.get("FunctionArn") diff --git a/skew/resources/aws/opsworks.py b/skew/resources/aws/opsworks.py new file mode 100644 index 0000000..a9291a4 --- /dev/null +++ b/skew/resources/aws/opsworks.py @@ -0,0 +1,57 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2019 Christophe Morio +# Copyright (c) 2020 Jerome Guibert +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +from skew.resources.aws import AWSResource +from skew.awsclient import get_awsclient + + +LOG = logging.getLogger(__name__) + + +class Stack(AWSResource): + class Meta(object): + service = "opsworks" + type = "stack" + enum_spec = ("describe_stacks", "Stacks", None) + detail_spec = None + id = "StackId" + filter_name = None + name = "Name" + date = "CreatedAt" + dimension = None + tags_spec = ("list_tags", "Tags", "ResourceArn", "arn") + + @property + def arn(self): + return self._data.get("Arn") + + @classmethod + def set_tags(cls, arn, region, account, tags, resource_id=None, **kwargs): + # ResourceGroupsTaggingAPI supports regional stacks, but not classic (us-east-1) + # opsworks.tag_resource() supports both + client = get_awsclient(cls.Meta.service, region, account, **kwargs) + r = client.call("tag_resource", ResourceArn=arn, Tags=tags) + LOG.debug("Tag ARN %s, r=%s", arn, r) + + @classmethod + def unset_tags(cls, arn, region, account, tag_keys, resource_id=None, **kwargs): + # ResourceGroupsTaggingAPI supports regional stacks, but not classic (us-east-1) + # opsworks.untag_resource() supports both + client = get_awsclient(cls.Meta.service, region, account, **kwargs) + r = client.call("untag_resource", ResourceArn=arn, TagKeys=tag_keys) + LOG.debug("UnTag ARN %s, r=%s", arn, r) diff --git a/skew/resources/aws/rds.py b/skew/resources/aws/rds.py index 4826a8e..85cb7d8 100644 --- a/skew/resources/aws/rds.py +++ b/skew/resources/aws/rds.py @@ -27,7 +27,7 @@ class Meta(object): id = 'DBInstanceIdentifier' filter_name = 'DBInstanceIdentifier' filter_type = 'scalar' - name = 'Endpoint.Address' + name = 'DBInstanceIdentifier' date = 'InstanceCreateTime' dimension = 'DBInstanceIdentifier' @@ -38,6 +38,9 @@ def arn(self): self._client.region_name, self._client.account_id, self.resourcetype, self.id) + def sleek(self): + self._data['LatestRestorableTime'] = '' + class DBSecurityGroup(AWSResource): diff --git a/skew/resources/aws/s3.py b/skew/resources/aws/s3.py index 98a2b25..a0599fa 100644 --- a/skew/resources/aws/s3.py +++ b/skew/resources/aws/s3.py @@ -1,4 +1,5 @@ # Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -24,46 +25,219 @@ class Bucket(AWSResource): @classmethod def enumerate(cls, arn, region, account, resource_id=None, **kwargs): - resources = super(Bucket, cls).enumerate(arn, region, account, - resource_id, - **kwargs) + resources = super(Bucket, cls).enumerate( + arn, region, account, resource_id, **kwargs + ) region_resources = [] if region is None: - region = 'us-east-1' + region = "us-east-1" for r in resources: location = cls._location_cache.get(r.id) if location is None: - LOG.debug('finding location for %s', r.id) - kwargs = {'Bucket': r.id} - response = r._client.call('get_bucket_location', **kwargs) - location = response.get('LocationConstraint', 'us-east-1') + LOG.debug("finding location for %s", r.id) + kwargs = {"Bucket": r.id} + response = r._client.call("get_bucket_location", **kwargs) + location = response.get("LocationConstraint", "us-east-1") if location is None: - location = 'us-east-1' - if location is 'EU': - location = 'eu-west-1' + location = "us-east-1" + if location == "EU": + location = "eu-west-1" cls._location_cache[r.id] = location if location == region: region_resources.append(r) return region_resources class Meta(object): - service = 's3' - type = 'bucket' - enum_spec = ('list_buckets', 'Buckets[]', None) - detail_spec = ('list_objects', 'Bucket', 'Contents[]') - id = 'Name' + service = "s3" + type = "bucket" + enum_spec = ("list_buckets", "Buckets[]", None) + detail_spec = ("list_objects", "Bucket", "Contents[]") + id = "Name" filter_name = None - name = 'BucketName' - date = 'CreationDate' + name = "BucketName" + date = "CreationDate" dimension = None - tags_spec = ('get_bucket_tagging', 'TagSet[]', - 'Bucket', 'id') + tags_spec = ("get_bucket_tagging", "TagSet[]", "Bucket", "id") + + attr_spec = { + "location": ("get_bucket_location", "LocationConstraint", "Bucket", "id"), + "acl": ("get_bucket_acl", "Grants", "Bucket", "id"), + "cors": ("get_bucket_cors", "CORSRules", "Bucket", "id"), + "encryption": ( + "get_bucket_encryption", + "ServerSideEncryptionConfiguration", + "Bucket", + "id", + ), + "lifecycle": ( + "get_bucket_lifecycle_configuration", + "Rules", + "Bucket", + "id", + ), + "logging": ("get_bucket_logging", "LoggingEnabled", "Bucket", "id"), + "policy": ("get_bucket_policy", "Policy", "Bucket", "id"), + "policy_status": ( + "get_bucket_policy_status", + "PolicyStatus", + "Bucket", + "id", + ), + "notifications": ( + "get_bucket_notification_configuration", + None, + "Bucket", + "id", + ), + "versioning": ( + "get_bucket_versioning", + None, + "Bucket", + "id", + ), + "website": ( + "get_bucket_website", + None, + "Bucket", + "id", + ), + } + + @classmethod + def filter(cls, arn, resource_id, data): + _id = data.get(cls.Meta.id) + return resource_id == _id def __init__(self, client, data, query=None): super(Bucket, self).__init__(client, data, query) self._data = data self._keys = [] + @property + def name(self): + return self._id + + @property + def arn(self): + return f"arn:aws:s3:::{self.id}" + + def _load_extra_attribute(self): + # loaded when self.data is called + self.location + self.acl + self.cors + self.encryption + self.lifecycle + self.logging + self.policy + self.policy_status + + @property + def location(self): + if "LocationConstraint" not in self._data: + self._data["LocationConstraint"] = self._feed_from_spec( + attr_spec=self.Meta.attr_spec["location"] + ) + return self._data["LocationConstraint"] + + @property + def acl(self): + if "Acl" not in self._data: + self._data["Acl"] = { + "Grants": self._feed_from_spec(attr_spec=self.Meta.attr_spec["acl"]) + } + return self._data["Acl"] + + @property + def cors(self): + if "CORSRules" not in self._data: + self._data["CORSRules"] = self._feed_from_spec( + attr_spec=self.Meta.attr_spec["cors"] + ) + return self._data["CORSRules"] + + @property + def encryption(self): + if "ServerSideEncryptionConfiguration" not in self._data: + self._data["ServerSideEncryptionConfiguration"] = self._feed_from_spec( + attr_spec=self.Meta.attr_spec["encryption"] + ) + return self._data["ServerSideEncryptionConfiguration"] + + @property + def lifecycle(self): + if "LifecycleConfiguration" not in self._data: + self._data["LifecycleConfiguration"] = { + "Rules": self._feed_from_spec( + attr_spec=self.Meta.attr_spec["lifecycle"] + ) + } + return self._data["LifecycleConfiguration"] + + @property + def logging(self): + if "Logging" not in self._data: + self._data["Logging"] = { + "LoggingEnabled": self._feed_from_spec( + attr_spec=self.Meta.attr_spec["logging"] + ) + } + return self._data["Logging"] + + @property + def policy(self): + if "Policy" not in self._data: + self._data["Policy"] = self._feed_from_spec( + attr_spec=self.Meta.attr_spec["policy"] + ) + return self._data["Policy"] + + @property + def policy_status(self): + if "PolicyStatus" not in self._data: + self._data["PolicyStatus"] = self._feed_from_spec( + attr_spec=self.Meta.attr_spec["policy_status"] + ) + return self._data["PolicyStatus"] + + @property + def notifications(self): + if "NotificationConfiguration" not in self._data: + _rep = self._feed_from_spec(attr_spec=self.Meta.attr_spec["notifications"]) + self._data["NotificationConfiguration"] = {} + if "TopicConfigurations" in _rep: + self._data["NotificationConfiguration"]["TopicConfigurations"] = _rep[ + "TopicConfigurations" + ] + if "QueueConfigurations" in _rep: + self._data["NotificationConfiguration"]["QueueConfigurations"] = _rep[ + "QueueConfigurations" + ] + if "LambdaFunctionConfigurations" in _rep: + self._data["NotificationConfiguration"][ + "LambdaFunctionConfigurations" + ] = _rep["LambdaFunctionConfigurations"] + return self._data["NotificationConfiguration"] + + @property + def versioning(self): + if "Versioning" not in self._data: + _rep = self._feed_from_spec(attr_spec=self.Meta.attr_spec["versioning"]) + self._data["Versioning"] = {} + if "Status" in _rep: + self._data["Versioning"]["Status"] = _rep["Status"] + if "MFADelete" in _rep: + self._data["Versioning"]["MFADelete"] = _rep["MFADelete"] + return self._data["Versioning"] + + @property + def website(self): + if "Website" not in self._data: + self._data["Website"] = self._feed_from_spec( + attr_spec=self.Meta.attr_spec["website"] + ) + return self._data["Website"] + def __iter__(self): detail_op, param_name, detail_path = self.Meta.detail_spec params = {param_name: self.id} diff --git a/skew/resources/aws/ses.py b/skew/resources/aws/ses.py new file mode 100644 index 0000000..fdc36df --- /dev/null +++ b/skew/resources/aws/ses.py @@ -0,0 +1,53 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2019 Christophe Morio +# Copyright (c) 2020 Jerome Guibert +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Identity(AWSResource): + class Meta(object): + service = "ses" + type = "identity" + enum_spec = ("list_identities", "Identities", None) + detail_spec = ("describe_table", "TableName", "Table") + id = "Identity" + tags_spec = None + filter_name = None + name = "IdentityName" + date = None + dimension = "IdentityName" + + def __init__(self, client, data, query=None): + super(Identity, self).__init__(client, data, query) + arn = self._data + self._data = {"IdentityName": arn} + + @property + def arn(self): + return "arn:aws:%s:%s:%s:%s/%s" % ( + self._client.service_name, + self._client.region_name, + self._client.account_id, + self.resourcetype, + self._data["IdentityName"], + ) diff --git a/skew/resources/aws/sns.py b/skew/resources/aws/sns.py index 1df3f08..eaf472e 100644 --- a/skew/resources/aws/sns.py +++ b/skew/resources/aws/sns.py @@ -19,82 +19,81 @@ class Topic(AWSResource): - class Meta(object): - service = 'sns' - type = 'topic' - enum_spec = ('list_topics', 'Topics', None) - detail_spec = ('get_topic_attributes', 'TopicArn', 'Attributes') - id = 'TopicArn' + service = "sns" + type = "topic" + enum_spec = ("list_topics", "Topics", None) + detail_spec = ("get_topic_attributes", "TopicArn", "Attributes") + id = "TopicArn" filter_name = None filter_type = None - name = 'DisplayName' + name = "DisplayName" date = None - dimension = 'TopicName' - tags_spec = ('list_tags_for_resource', 'Tags[]', 'ResourceArn', 'arn') + dimension = "TopicName" + tags_spec = ("list_tags_for_resource", "Tags[]", "ResourceArn", "arn") @classmethod def filter(cls, arn, resource_id, data): topic_arn = data.get(cls.Meta.id) - LOG.debug('%s == %s', arn, topic_arn) + LOG.debug("%s == %s", arn, topic_arn) return arn == topic_arn @property def arn(self): - return self.data.get('TopicArn') + return self._data.get("TopicArn") def __init__(self, client, data, query=None): super(Topic, self).__init__(client, data, query) - self._id = data['TopicArn'].split(':', 5)[5] + self._id = data["TopicArn"].split(":", 5)[5] detail_op, param_name, detail_path = self.Meta.detail_spec - params = {param_name: data['TopicArn']} + params = {param_name: data["TopicArn"]} data = client.call(detail_op, **params) - self.data = jmespath.search(detail_path, data) + self._data = jmespath.search(detail_path, data) class Subscription(AWSResource): - invalid_arns = ['PendingConfirmation', 'Deleted'] + invalid_arns = ["PendingConfirmation", "Deleted"] class Meta(object): - service = 'sns' - type = 'subscription' - enum_spec = ('list_subscriptions', 'Subscriptions', None) - detail_spec = ('get_subscription_attributes', 'SubscriptionArn', - 'Attributes') - id = 'SubscriptionArn' + service = "sns" + type = "subscription" + enum_spec = ("list_subscriptions", "Subscriptions", None) + detail_spec = ("get_subscription_attributes", "SubscriptionArn", "Attributes") + id = "SubscriptionArn" filter_name = None filter_type = None - name = 'SubscriptionArn' + name = "SubscriptionArn" date = None dimension = None @property def arn(self): - return self.data.get('SubscriptionArn') + return self._data.get("SubscriptionArn") @classmethod def enumerate(cls, arn, region, account, resource_id=None, **kwargs): resources = super(Subscription, cls).enumerate( - arn, region, account, resource_id, **kwargs) + arn, region, account, resource_id, **kwargs + ) return [r for r in resources if r.id not in cls.invalid_arns] def __init__(self, client, data, query=None): super(Subscription, self).__init__(client, data, query) - if data['SubscriptionArn'] in self.invalid_arns: - self._id = 'PendingConfirmation' + if data["SubscriptionArn"] in self.invalid_arns: + self._id = "PendingConfirmation" return - self._id = data['SubscriptionArn'].split(':', 6)[6] + self._id = data["SubscriptionArn"].split(":", 6)[6] self._name = "" detail_op, param_name, detail_path = self.Meta.detail_spec - params = {param_name: data['SubscriptionArn']} + params = {param_name: data["SubscriptionArn"]} data = client.call(detail_op, **params) - self.data = jmespath.search(detail_path, data) + self._data = jmespath.search(detail_path, data) diff --git a/skew/resources/aws/sqs.py b/skew/resources/aws/sqs.py index 7d11c59..197d9b5 100644 --- a/skew/resources/aws/sqs.py +++ b/skew/resources/aws/sqs.py @@ -16,22 +16,20 @@ class Queue(AWSResource): - class Meta(object): - service = 'sqs' - type = 'queue' - enum_spec = ('list_queues', 'QueueUrls', None) - detail_spec = ('get_queue_attributes', 'QueueUrl', 'QueueUrl') - id = 'QueueUrl' - filter_name = 'QueueNamePrefix' - filter_type = 'scalar' - name = 'QueueUrl' + service = "sqs" + type = "queue" + enum_spec = ("list_queues", "QueueUrls", None) + detail_spec = ("get_queue_attributes", "QueueUrl", "QueueUrl") + id = "QueueUrl" + filter_name = "QueueNamePrefix" + filter_type = "scalar" + name = "QueueName" date = None - dimension = 'QueueName' - tags_spec = ('list_queue_tags', 'Tags', 'QueueUrl', 'name') + dimension = "QueueName" + tags_spec = ("list_queue_tags", "Tags", "QueueUrl", "name") def __init__(self, client, data, query=None): super(Queue, self).__init__(client, data, query) - self.data = {self.Meta.id: data, - 'QueueName': data.split('/')[-1]} - self._id = self.data['QueueName'] + self._data = {self.Meta.id: data, "QueueName": data.split("/")[-1]} + self._id = self._data["QueueName"] diff --git a/skew/resources/aws/stepfunctions.py b/skew/resources/aws/stepfunctions.py new file mode 100755 index 0000000..7beff28 --- /dev/null +++ b/skew/resources/aws/stepfunctions.py @@ -0,0 +1,56 @@ +# Copyright (c) 2020 Jerome Guibert +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class StateMachines(AWSResource): + class Meta(object): + service = "stepfunctions" + type = "stateMachine" + enum_spec = ("list_state_machines", "stateMachines[]", None) + detail_spec = ("describe_state_machine", None, "stateMachineArn", "arn") + filter_name = None + filter_type = None + id = "name" + name = "name" + date = "creationDate" + tags_spec = ("list_tags_for_resource", "tags", "resourceArn", "arn") + + @classmethod + def filter(cls, arn, resource_id, data): + return resource_id == data["name"] + + def __init__(self, client, data, query=None): + super(StateMachines, self).__init__(client, data, query) + self._arn = self._data["stateMachineArn"] + + detail = self._feed_from_spec(attr_spec=self.Meta.detail_spec) + if "ResponseMetadata" in detail: + del detail["ResponseMetadata"] + + self._data = { + **self._data, + **detail, + } + + @property + def arn(self): + return self._arn \ No newline at end of file diff --git a/skew/resources/aws/support.py b/skew/resources/aws/support.py new file mode 100644 index 0000000..c15631c --- /dev/null +++ b/skew/resources/aws/support.py @@ -0,0 +1,37 @@ +# Copyright (c) 2014 Scopely, Inc. +# Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import logging + +import jmespath + +from skew.resources.aws import AWSResource + + +LOG = logging.getLogger(__name__) + + +class Check(AWSResource): + class Meta(object): + service = "support" + type = "check" + enum_spec = ("describe_trusted_advisor_checks", "checks", None) + detail_spec = None + id = "id" + tags_spec = None + filter_name = None + name = "name" + date = None + dimension = "IdentityName" diff --git a/skew/resources/json_dump.py b/skew/resources/json_dump.py new file mode 100644 index 0000000..cbc316c --- /dev/null +++ b/skew/resources/json_dump.py @@ -0,0 +1,62 @@ +# Copyright (c) 2020 Jerome Guibert +# +# Licensed under the Apache License, Version 2.0 (the "License"). You +# may not use this file except in compliance with the License. A copy of +# the License is located at +# +# http://aws.amazon.com/apache2.0/ +# +# or in the "license" file accompanying this file. This file is +# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF +# ANY KIND, either express or implied. See the License for the specific +# language governing permissions and limitations under the License. + +import urllib +import re +from typing import Dict, List +import json +import datetime + +__all__ = ["json_dump"] + + +def json_dump(data, normalize=True): + return json.dumps( + obj=_normalize(data) if normalize else data, + indent=4, + sort_keys=True, + default=_custom_json_serializer, + ) + + +def _custom_json_serializer(x): + if isinstance(x, datetime.datetime): + return x.isoformat() + elif isinstance(x, bytes): + return x.decode() + raise TypeError("Unknown type") + + +# _camel_to_snake optimisation pattern +_pattern_1 = re.compile("(.)([A-Z][a-z]+)") +_pattern_2 = re.compile("([a-z0-9])([A-Z])") + + +def _camel_to_snake(name: str) -> str: + """Convert camel case string to snake case.""" + name = _pattern_1.sub(r"\1_\2", name) + return _pattern_2.sub(r"\1_\2", name).lower() + + +def _normalize(data: Dict) -> Dict: + """Normalize dictionary keys.""" + new_data = dict(map(lambda item: (_camel_to_snake(item[0]), item[1]), data.items())) + for key, value in new_data.items(): + if isinstance(value, dict): + new_data[key] = _normalize(value) + if isinstance(value, list): + for i in range(len(value)): + if isinstance(value[i], dict): + value[i] = _normalize(value[i]) + + return new_data diff --git a/skew/resources/resource.py b/skew/resources/resource.py index 49001d4..158a1ed 100644 --- a/skew/resources/resource.py +++ b/skew/resources/resource.py @@ -1,5 +1,6 @@ # Copyright (c) 2014 Scopely, Inc. # Copyright (c) 2015 Mitch Garnaat +# Copyright (c) 2020 Jerome Guibert # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -15,7 +16,8 @@ import logging import jmespath -import skew.awsclient +from skew.awsclient import get_awsclient +from skew.resources.json_dump import json_dump from botocore.exceptions import ClientError @@ -23,14 +25,12 @@ class Resource(object): - @classmethod def enumerate(cls, arn, region, account, resource_id=None, **kwargs): - client = skew.awsclient.get_awsclient( - cls.Meta.service, region, account, **kwargs) + client = get_awsclient(cls.Meta.service, region, account, **kwargs) kwargs = {} do_client_side_filtering = False - if resource_id and resource_id != '*': + if resource_id and resource_id != "*": # If we are looking for a specific resource and the # API provides a way to filter on a specific resource # id then let's insert the right parameter to do the filtering. @@ -38,9 +38,9 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): # after we get all of the results. filter_name = cls.Meta.filter_name if filter_name: - if cls.Meta.filter_type == 'arn': + if cls.Meta.filter_type == "arn": kwargs[filter_name] = [str(arn)] - elif cls.Meta.filter_type == 'list': + elif cls.Meta.filter_type == "list": kwargs[filter_name] = [resource_id] else: kwargs[filter_name] = resource_id @@ -49,31 +49,22 @@ def enumerate(cls, arn, region, account, resource_id=None, **kwargs): enum_op, path, extra_args = cls.Meta.enum_spec if extra_args: kwargs.update(extra_args) - LOG.debug('enum_spec=%s' % str(cls.Meta.enum_spec)) + LOG.debug("enum_spec=%s" % str(cls.Meta.enum_spec)) try: data = client.call(enum_op, query=path, **kwargs) + if data: + if do_client_side_filtering: + data = filter(lambda d: cls.filter(arn, resource_id, d), data) + return map(lambda d: cls(client, d, arn.query), data) except ClientError as e: LOG.debug(e) - data = {} # if the error is because the resource was not found, be quiet - if 'NotFound' not in e.response['Error']['Code']: + if "NotFound" not in e.response["Error"]["Code"]: raise - LOG.debug(data) - resources = [] - if data: - for d in data: - if do_client_side_filtering: - # If the API does not support filtering, the resource - # class should provide a filter method that will - # return True if the returned data matches the - # resource ID we are looking for. - if not cls.filter(arn, resource_id, d): - continue - resources.append(cls(client, d, arn.query)) - return resources + return [] class Meta(object): - type = 'resource' + type = "resource" dimension = None tags_spec = None id = None @@ -82,14 +73,12 @@ class Meta(object): def __init__(self, client, data): self._client = client - if data is None: - data = {} - self.data = data - if hasattr(self.Meta, 'id') and isinstance(self.data, dict): - self._id = self.data.get(self.Meta.id, '') + self._data = data if data else {} + if hasattr(self.Meta, "id") and isinstance(self._data, dict): + self._id = self._data.get(self.Meta.id, "") else: - self._id = '' - self._metrics = list() + self._id = "" + self._metrics = None self._name = None self._date = None @@ -98,11 +87,17 @@ def __repr__(self): @property def arn(self): - return 'arn:aws:%s:%s:%s:%s/%s' % ( + return "arn:aws:%s:%s:%s:%s/%s" % ( self._client.service_name, self._client.region_name, self._client.account_id, - self.resourcetype, self.id) + self.resourcetype, + self.id, + ) + + @property + def data(self): + return self._data @property def resourcetype(self): @@ -115,7 +110,7 @@ def parent(self): @property def name(self): if not self._name: - self._name = jmespath.search(self.Meta.name, self.data) + self._name = jmespath.search(self.Meta.name, self._data) return self._name @property @@ -125,21 +120,22 @@ def id(self): @property def date(self): if not self._date: - self._date = jmespath.search(self.Meta.date, self.data) + self._date = jmespath.search(self.Meta.date, self._data) return self._date @property def metrics(self): - if self._metrics is None: - self._metrics = [] - return self._metrics + return self._metrics if self._metrics else [] @property def metric_names(self): - return [m['MetricName'] for m in self.metrics] + return [m["MetricName"] for m in self.metrics] def find_metric(self, metric_name): for m in self.metrics: - if m['MetricName'] == metric_name: + if m["MetricName"] == metric_name: return m return None + + def json_dump(self, normalize=True): + return json_dump(self.data, normalize=normalize) \ No newline at end of file diff --git a/tests/unit/responses/trail/cloudtrail.GetTrailStatus_1.json b/tests/unit/responses/trail/cloudtrail.GetTrailStatus_1.json new file mode 100644 index 0000000..33fca73 --- /dev/null +++ b/tests/unit/responses/trail/cloudtrail.GetTrailStatus_1.json @@ -0,0 +1,8 @@ +{ + "status_code": 200, + "data": { + "Status": { + "IsLogging": true + } + } +} \ No newline at end of file diff --git a/tests/unit/responses/trail/cloudtrail.GetTrail_1.json b/tests/unit/responses/trail/cloudtrail.GetTrail_1.json new file mode 100644 index 0000000..b59247c --- /dev/null +++ b/tests/unit/responses/trail/cloudtrail.GetTrail_1.json @@ -0,0 +1,18 @@ +{ + "status_code": 200, + "data": { + "Trail": { + "Name": "awslog", + "S3BucketName": "awslog", + "IncludeGlobalServiceEvents": true, + "IsMultiRegionTrail": true, + "HomeRegion": "us-east-1", + "TrailARN": "arn:aws:cloudtrail:us-east-1:123456789012:trail/awslog", + "LogFileValidationEnabled": false, + "CloudWatchLogsLogGroupArn": "arn:aws:logs:us-east-1:123456789012:log-group:CloudTrail/DefaultLogGroup:*", + "CloudWatchLogsRoleArn": "arn:aws:iam::123456789012:role/CloudTrail_CloudWatchLogs_Role", + "HasCustomEventSelectors": false, + "IsOrganizationTrail": false + } + } +} \ No newline at end of file diff --git a/tests/unit/responses/trail/cloudtrail.ListTrails_1.json b/tests/unit/responses/trail/cloudtrail.ListTrails_1.json new file mode 100644 index 0000000..da36066 --- /dev/null +++ b/tests/unit/responses/trail/cloudtrail.ListTrails_1.json @@ -0,0 +1,12 @@ +{ + "status_code": 200, + "data": { + "Trails": [ + { + "TrailARN": "arn:aws:cloudtrail:us-east-1:123456789012:trail/awslog", + "HomeRegion": "us-east-1", + "Name": "awslog" + } + ] + } +} \ No newline at end of file diff --git a/tests/unit/test_arn.py b/tests/unit/test_arn.py index 85f27c7..9663fad 100644 --- a/tests/unit/test_arn.py +++ b/tests/unit/test_arn.py @@ -21,74 +21,79 @@ class TestARN(unittest.TestCase): - def _get_response_path(self, test_case): - p = os.path.join(os.path.dirname(__file__), 'responses') + p = os.path.join(os.path.dirname(__file__), "responses") return os.path.join(p, test_case) def setUp(self): self.environ = {} - self.environ_patch = mock.patch('os.environ', self.environ) + self.environ_patch = mock.patch("os.environ", self.environ) self.environ_patch.start() - credential_path = os.path.join(os.path.dirname(__file__), 'cfg', - 'aws_credentials') - self.environ['AWS_CONFIG_FILE'] = credential_path - config_path = os.path.join(os.path.dirname(__file__), 'cfg', - 'skew.yml') - self.environ['SKEW_CONFIG'] = config_path + credential_path = os.path.join( + os.path.dirname(__file__), "cfg", "aws_credentials" + ) + self.environ["AWS_CONFIG_FILE"] = credential_path + config_path = os.path.join(os.path.dirname(__file__), "cfg", "skew.yml") + self.environ["SKEW_CONFIG"] = config_path def tearDown(self): pass def test_ec2(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('instances_1'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:ec2:us-west-2:123456789012:instance/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("instances_1"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:ec2:us-west-2:123456789012:instance/*", **placebo_cfg) # Fetch all Instance resources l = list(arn) self.assertEqual(len(l), 2) # Fetch a single resource placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('instances_2'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:ec2:us-west-2:123456789012:instance/i-db530902', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("instances_2"), + "placebo_mode": "playback", + } + arn = scan( + "arn:aws:ec2:us-west-2:123456789012:instance/i-db530902", **placebo_cfg + ) l = list(arn) self.assertEqual(len(l), 1) # check filters - arn = scan('arn:aws:ec2:us-west-2:123456789012:instance/i-db530902|InstanceType', - **placebo_cfg) + arn = scan( + "arn:aws:ec2:us-west-2:123456789012:instance/i-db530902|InstanceType", + **placebo_cfg + ) l = list(arn) self.assertEqual(len(l), 1) r = l[0] - self.assertEqual(r.filtered_data, 't2.small') + self.assertEqual(r.filtered_data, "t2.small") def test_ec2_instance_not_found(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('instances_3'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:ec2:us-west-2:123456789012:instance/i-87654321', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("instances_3"), + "placebo_mode": "playback", + } + arn = scan( + "arn:aws:ec2:us-west-2:123456789012:instance/i-87654321", **placebo_cfg + ) # Fetch all Instance resources l = list(arn) self.assertEqual(len(l), 0) def test_ec2_volumes(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('volumes'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:ec2:us-west-2:123456789012:volume/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("volumes"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:ec2:us-west-2:123456789012:volume/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 4) r = l[0] - self.assertEqual(r.data['VolumeId'], "vol-b85e475f") + self.assertEqual(r.data["VolumeId"], "vol-b85e475f") # def test_ec2_images(self): # arn = scan('arn:aws:ec2:us-west-2:234567890123:image/*') @@ -97,129 +102,149 @@ def test_ec2_volumes(self): def test_ec2_keypairs(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('keypairs'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:ec2:us-west-2:123456789012:key-pair/*', - debug=True, **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("keypairs"), + "placebo_mode": "playback", + } + arn = scan( + "arn:aws:ec2:us-west-2:123456789012:key-pair/*", debug=True, **placebo_cfg + ) l = list(arn) self.assertEqual(len(l), 2) - self.assertEqual(l[0].id, 'key-1274ea12942819e24') - self.assertEqual(l[1].id, 'key-1274ea12942819e25') - self.assertEqual(l[0].name, 'admin') - self.assertEqual(l[1].name, 'FooBar') + self.assertEqual(l[0].id, "key-1274ea12942819e24") + self.assertEqual(l[1].id, "key-1274ea12942819e25") + self.assertEqual(l[0].name, "admin") + self.assertEqual(l[1].name, "FooBar") self.assertEqual( - l[0].data['KeyFingerprint'], - "85:83:08:25:fa:96:45:ea:c9:15:04:12:af:45:3f:c0:ef:e8:b8:ce") + l[0].data["KeyFingerprint"], + "85:83:08:25:fa:96:45:ea:c9:15:04:12:af:45:3f:c0:ef:e8:b8:ce", + ) def test_ec2_securitygroup(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('secgrp'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:ec2:us-west-2:123456789012:security-group/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("secgrp"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:ec2:us-west-2:123456789012:security-group/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 3) def test_elb_loadbalancer(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('elbs'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:elb:us-east-1:123456789012:loadbalancer/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("elbs"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:elb:us-east-1:123456789012:loadbalancer/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 1) - self.assertEqual(l[0].arn, 'arn:aws:elb:us-east-1:123456789012:loadbalancer/example') - self.assertEqual(l[0].data['DNSName'], 'example-1111111111.us-east-1.elb.amazonaws.com') - self.assertEqual(l[0].tags['Name'], 'example-web') - self.assertEqual(l[0].data['LoadBalancerAttributes']['CrossZoneLoadBalancing']['Enabled'], False) - self.assertEqual(l[0].data['PolicyDescriptions'][0]['PolicyName'], 'AWSConsole-SSLNegotiationPolicy-example-1111111111111') + self.assertEqual( + l[0].arn, "arn:aws:elb:us-east-1:123456789012:loadbalancer/example" + ) + self.assertEqual( + l[0].data["DNSName"], "example-1111111111.us-east-1.elb.amazonaws.com" + ) + self.assertEqual(l[0].tags["Name"], "example-web") + self.assertEqual( + l[0].data["LoadBalancerAttributes"]["CrossZoneLoadBalancing"]["Enabled"], + False, + ) + self.assertEqual( + l[0].data["PolicyDescriptions"][0]["PolicyName"], + "AWSConsole-SSLNegotiationPolicy-example-1111111111111", + ) def test_ec2_vpcs(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('vpcs'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:ec2:us-west-2:123456789012:vpc/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("vpcs"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:ec2:us-west-2:123456789012:vpc/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 3) def test_ec2_routetable(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('routetables'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:ec2:us-west-2:123456789012:route-table/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("routetables"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:ec2:us-west-2:123456789012:route-table/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 5) def test_ec2_network_acls(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('networkacls'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:ec2:us-west-2:123456789012:network-acl/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("networkacls"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:ec2:us-west-2:123456789012:network-acl/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 8) def test_s3_buckets(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('buckets'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:s3:us-east-1:234567890123:bucket/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("buckets"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:s3:us-east-1:234567890123:bucket/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 5) def test_iam_groups(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('groups'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:iam::234567890123:group/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("groups"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:iam::234567890123:group/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 3) group_resource = l[0] - self.assertEqual(group_resource.arn, - 'arn:aws:iam::234567890123:group/Administrators') + self.assertEqual( + group_resource.arn, "arn:aws:iam::234567890123:group/Administrators" + ) def test_iam_users(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('users'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:iam::123456789012:user/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("users"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:iam::123456789012:user/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 1) - self.assertEqual(l[0].arn, 'arn:aws:iam::123456789012:user/testuser') - self.assertEqual(l[0].data['UserName'], 'testuser') - self.assertEqual(l[0].tags['TestKey'], 'TestValue') - self.assertEqual(l[0].data['AccessKeyMetadata'][0]['AccessKeyId'], - 'AKIAAAAAAAAAAAAAAAAA') - self.assertEqual(l[0].data['Groups'][0]['GroupId'], - 'AGPAAAAAAAAAAAAAAAAAA') - self.assertEqual(l[0].data['PolicyNames']['TestInlinePolicy']['Version'], - '2012-10-17') - self.assertEqual(l[0].data['AttachedPolicies'][0]['PolicyArn'], - 'arn:aws:iam::aws:policy/AdministratorAccess') - self.assertEqual(l[0].data['SSHPublicKeys'][0]['SSHPublicKeyId'], - 'APKAAAAAAAAAAAAAAAAA') + self.assertEqual(l[0].arn, "arn:aws:iam::123456789012:user/testuser") + self.assertEqual(l[0].data["UserName"], "testuser") + self.assertEqual(l[0].tags["TestKey"], "TestValue") + self.assertEqual( + l[0].data["AccessKeyMetadata"][0]["AccessKeyId"], "AKIAAAAAAAAAAAAAAAAA" + ) + self.assertEqual(l[0].data["Groups"][0]["GroupId"], "AGPAAAAAAAAAAAAAAAAAA") + self.assertEqual( + l[0].data["PolicyNames"]["TestInlinePolicy"]["Version"], "2012-10-17" + ) + self.assertEqual( + l[0].data["AttachedPolicies"][0]["PolicyArn"], + "arn:aws:iam::aws:policy/AdministratorAccess", + ) + self.assertEqual( + l[0].data["SSHPublicKeys"][0]["SSHPublicKeyId"], "APKAAAAAAAAAAAAAAAAA" + ) def test_cloudformation_stacks(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('stacks'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:cloudformation:us-west-2:123456789012:stack/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("stacks"), + "placebo_mode": "playback", + } + arn = scan( + "arn:aws:cloudformation:us-west-2:123456789012:stack/*", **placebo_cfg + ) l = list(arn) self.assertEqual(len(l), 1) stack_resource = l[0] @@ -228,177 +253,235 @@ def test_cloudformation_stacks(self): def test_nat_gateways(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('natgateways'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:ec2:us-west-2:123456789012:natgateway/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("natgateways"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:ec2:us-west-2:123456789012:natgateway/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 1) natgateways = l[0] - self.assertEqual(natgateways.arn, - 'arn:aws:ec2:us-west-2:123456789012:natgateway/nat-443d3ea762d00ee83') + self.assertEqual( + natgateways.arn, + "arn:aws:ec2:us-west-2:123456789012:natgateway/nat-443d3ea762d00ee83", + ) def test_ec2_launchtemplates(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('launchtemplates'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:ec2:us-west-2:123456789012:launch-template/*', - debug=True, **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("launchtemplates"), + "placebo_mode": "playback", + } + arn = scan( + "arn:aws:ec2:us-west-2:123456789012:launch-template/*", + debug=True, + **placebo_cfg + ) l = list(arn) self.assertEqual(len(l), 4) - self.assertEqual(l[0].id, 'lt-000005555511111888') - self.assertEqual(l[1].id, 'lt-000007777744444999') - self.assertEqual(l[2].id, 'lt-000006666633333888') - self.assertEqual(l[3].id, 'lt-000777772222211223') + self.assertEqual(l[0].id, "lt-000005555511111888") + self.assertEqual(l[1].id, "lt-000007777744444999") + self.assertEqual(l[2].id, "lt-000006666633333888") + self.assertEqual(l[3].id, "lt-000777772222211223") - self.assertEqual(l[1].data['Tags'][0]['Key'], "costcenter") - self.assertTrue('Tags' not in l[3].data) + self.assertEqual(l[1].data["Tags"][0]["Key"], "costcenter") + self.assertTrue("Tags" not in l[3].data) def test_acm(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('certificates'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:acm:us-west-2:123456789012:certificate/*', - debug=True, **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("certificates"), + "placebo_mode": "playback", + } + arn = scan( + "arn:aws:acm:us-west-2:123456789012:certificate/*", + debug=True, + **placebo_cfg + ) l = list(arn) self.assertEqual(len(l), 2) - self.assertEqual(l[0].arn, 'arn:aws:acm:us-west-2:123456789012:certificate/aaaaaaaa-bbbb-cccc-dddd-000000000001') - self.assertEqual(l[0].data['DomainName'], 'example.com') - self.assertEqual(l[0].tags['tld'], '.com') - self.assertEqual(l[1].arn, 'arn:aws:acm:us-west-2:123456789012:certificate/aaaaaaaa-bbbb-cccc-dddd-000000000002') - self.assertEqual(l[1].data['DomainName'], 'example.net') - self.assertEqual(l[1].tags['tld'], '.net') + self.assertEqual( + l[0].arn, + "arn:aws:acm:us-west-2:123456789012:certificate/aaaaaaaa-bbbb-cccc-dddd-000000000001", + ) + self.assertEqual(l[0].data["DomainName"], "example.com") + self.assertEqual(l[0].tags["tld"], ".com") + self.assertEqual( + l[1].arn, + "arn:aws:acm:us-west-2:123456789012:certificate/aaaaaaaa-bbbb-cccc-dddd-000000000002", + ) + self.assertEqual(l[1].data["DomainName"], "example.net") + self.assertEqual(l[1].tags["tld"], ".net") def test_cloudwatch_loggroup(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('loggroups'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:logs:us-east-1:123456789012:log-group/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("loggroups"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:logs:us-east-1:123456789012:log-group/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 1) - self.assertEqual(l[0].arn, 'arn:aws:logs:us-east-1:123456789012:log-group:CloudTrail/DefaultLogGroup') - self.assertEqual(l[0].data['logGroupName'], 'CloudTrail/DefaultLogGroup') - self.assertEqual(l[0].tags['TestKey'], 'TestValue') - self.assertEqual(l[0].data['logStreams'][0]['logStreamName'], '123456789012_CloudTrail_us-east-1') - self.assertEqual(l[0].data['metricFilters'][0]['filterName'], 'EventCount') - self.assertEqual(l[0].data['subscriptionFilters'][0]['filterName'], 'TestLambdaTrigger') - self.assertEqual(l[0].data['queries'][0]['queryId'], '11111111-cfe3-43db-8eca-8862fee615a3') + self.assertEqual( + l[0].arn, + "arn:aws:logs:us-east-1:123456789012:log-group:CloudTrail/DefaultLogGroup", + ) + self.assertEqual(l[0].data["logGroupName"], "CloudTrail/DefaultLogGroup") + self.assertEqual(l[0].tags["TestKey"], "TestValue") + + self.assertNotIn("logStreams", l[0].data) + l[0].log_streams + self.assertEqual( + l[0].data["logStreams"][0]["logStreamName"], + "123456789012_CloudTrail_us-east-1", + ) + + self.assertNotIn("metricFilters", l[0].data) + l[0].metric_filters + self.assertEqual(l[0].data["metricFilters"][0]["filterName"], "EventCount") + + self.assertNotIn("subscriptionFilters", l[0].data) + l[0].subscriptions + self.assertEqual( + l[0].data["subscriptionFilters"][0]["filterName"], "TestLambdaTrigger" + ) + + self.assertNotIn("queries", l[0].data) + l[0].queries + self.assertEqual( + l[0].data["queries"][0]["queryId"], "11111111-cfe3-43db-8eca-8862fee615a3" + ) def test_vpc_flowlog(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('flowlogs'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:ec2:us-east-1:123456789012:flow-log/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("flowlogs"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:ec2:us-east-1:123456789012:flow-log/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 2) - self.assertEqual(l[0].arn, 'arn:aws:ec2:us-east-1:123456789012:flow-log/fl-1234abcd') - self.assertEqual(l[0].data['LogGroupName'], 'CloudTrail/DefaultLogGroup') - self.assertEqual(str(l[0].data['CreationTime']), '2017-01-23 19:47:49') + self.assertEqual( + l[0].arn, "arn:aws:ec2:us-east-1:123456789012:flow-log/fl-1234abcd" + ) + self.assertEqual(l[0].data["LogGroupName"], "CloudTrail/DefaultLogGroup") + self.assertEqual(str(l[0].data["CreationTime"]), "2017-01-23 19:47:49+00:00") def test_cloudtrail(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('trail'), - 'placebo_mode': 'playback'} - arn = scan( - 'arn:aws:cloudtrail:us-east-1:123456789012:trail/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("trail"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:cloudtrail:us-east-1:123456789012:trail/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 1) print(l[0].tags) - self.assertEqual(l[0].arn, 'arn:aws:cloudtrail:us-east-1:123456789012:trail/awslog') - self.assertEqual(l[0].data['CloudWatchLogsLogGroupArn'], - 'arn:aws:logs:us-east-1:123456789012:log-group:CloudTrail/DefaultLogGroup:*') + self.assertEqual( + l[0].arn, "arn:aws:cloudtrail:us-east-1:123456789012:trail/awslog" + ) + self.assertEqual( + l[0].data["CloudWatchLogsLogGroupArn"], + "arn:aws:logs:us-east-1:123456789012:log-group:CloudTrail/DefaultLogGroup:*", + ) print(l[0].tags) - self.assertEqual(l[0].tags['TestKey'], 'TestValue') + self.assertEqual(l[0].tags["TestKey"], "TestValue") def test_no_provider(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('trail'), - 'placebo_mode': 'playback'} - arn = scan( - '::cloudtrail:us-east-1:123456789012:trail/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("trail"), + "placebo_mode": "playback", + } + arn = scan("::cloudtrail:us-east-1:123456789012:trail/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 1) def test_alarm(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('alarms'), - 'placebo_mode': 'playback'} - arn = scan( - 'arn:aws:cloudwatch:us-east-1:123456789012:alarm/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("alarms"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:cloudwatch:us-east-1:123456789012:alarm/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 1) - self.assertEqual(l[0].arn, 'arn:aws:cloudwatch:us-east-1:123456789012:alarm:some-alarm') - self.assertEqual(l[0].data['AlarmArn'], - 'arn:aws:cloudwatch:us-east-1:123456789012:alarm:some-alarm') + self.assertEqual( + l[0].arn, "arn:aws:cloudwatch:us-east-1:123456789012:alarm:some-alarm" + ) + self.assertEqual( + l[0].data["AlarmArn"], + "arn:aws:cloudwatch:us-east-1:123456789012:alarm:some-alarm", + ) def test_customer_gateway(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('customergateways'), - 'placebo_mode': 'playback'} + "placebo": placebo, + "placebo_dir": self._get_response_path("customergateways"), + "placebo_mode": "playback", + } arn = scan( - 'arn:aws:ec2:us-east-1:123456789012:customer-gateway/*', - **placebo_cfg) + "arn:aws:ec2:us-east-1:123456789012:customer-gateway/*", **placebo_cfg + ) l = list(arn) self.assertEqual(len(l), 1) - self.assertEqual(l[0].arn, 'arn:aws:ec2:us-east-1:123456789012:customer-gateway/cgw-030d9af8cdbcdc12f') - self.assertEqual(l[0].data['CustomerGatewayId'], - 'cgw-030d9af8cdbcdc12f') + self.assertEqual( + l[0].arn, + "arn:aws:ec2:us-east-1:123456789012:customer-gateway/cgw-030d9af8cdbcdc12f", + ) + self.assertEqual(l[0].data["CustomerGatewayId"], "cgw-030d9af8cdbcdc12f") def test_beanstalk_environments(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('environments'), - 'placebo_mode': 'playback'} - arn = scan('arn:aws:elasticbeanstalk:us-west-2:123456789012:environment/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("environments"), + "placebo_mode": "playback", + } + arn = scan( + "arn:aws:elasticbeanstalk:us-west-2:123456789012:environment/*", + **placebo_cfg + ) l = list(arn) r = l[0] - self.assertEqual(r.data['EnvironmentName'], "Env1") - self.assertEqual(r.arn, "arn:aws:elasticbeanstalk:us-west-2:123456789012:environment/sample-application/Env1") - self.assertEqual(r.data['ApplicationName'], "sample-application") + self.assertEqual(r.data["EnvironmentName"], "Env1") + self.assertEqual( + r.arn, + "arn:aws:elasticbeanstalk:us-west-2:123456789012:environment/sample-application/Env1", + ) + self.assertEqual(r.data["ApplicationName"], "sample-application") def test_ec2_address(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('addresses'), - 'placebo_mode': 'playback'} - arn = scan( - 'arn:aws:ec2:us-east-1:123456789012:address/*', - **placebo_cfg) + "placebo": placebo, + "placebo_dir": self._get_response_path("addresses"), + "placebo_mode": "playback", + } + arn = scan("arn:aws:ec2:us-east-1:123456789012:address/*", **placebo_cfg) l = list(arn) self.assertEqual(len(l), 3) - self.assertEqual(l[0].arn, 'arn:aws:ec2:us-east-1:123456789012:address/eipalloc-091f2b843804f008c') - self.assertEqual(l[0].data['AllocationId'], - 'eipalloc-091f2b843804f008c') - - self.assertEqual(l[2].data['Tags'], - [{'Key': 'Name', 'Value': 'some-name'}, {'Key': 'Env', 'Value': 'Prod'}]) + self.assertEqual( + l[0].arn, + "arn:aws:ec2:us-east-1:123456789012:address/eipalloc-091f2b843804f008c", + ) + self.assertEqual(l[0].data["AllocationId"], "eipalloc-091f2b843804f008c") + self.assertEqual( + l[2].data["Tags"], + [{"Key": "Name", "Value": "some-name"}, {"Key": "Env", "Value": "Prod"}], + ) def test_vpc_peering_connection(self): placebo_cfg = { - 'placebo': placebo, - 'placebo_dir': self._get_response_path('peeringconnections'), - 'placebo_mode': 'playback'} + "placebo": placebo, + "placebo_dir": self._get_response_path("peeringconnections"), + "placebo_mode": "playback", + } arn = scan( - 'arn:aws:ec2:us-east-1:123456789012:vpc-peering-connection/*', - **placebo_cfg) + "arn:aws:ec2:us-east-1:123456789012:vpc-peering-connection/*", **placebo_cfg + ) l = list(arn) self.assertEqual(len(l), 1) - self.assertEqual(l[0].arn, 'arn:aws:ec2:us-east-1:123456789012:vpc-peering-connection/pcx-027a582b95db2af78') - - + self.assertEqual( + l[0].arn, + "arn:aws:ec2:us-east-1:123456789012:vpc-peering-connection/pcx-027a582b95db2af78", + ) diff --git a/tests/unit/test_resource.py b/tests/unit/test_resource.py index 8cc560e..02c34be 100644 --- a/tests/unit/test_resource.py +++ b/tests/unit/test_resource.py @@ -61,4 +61,8 @@ def test_all_providers(self): def test_all_services(self): all_providers = skew.resources.all_services('aws') - self.assertEqual(len(all_providers), 24) + self.assertEqual(len(all_providers), 35) + + def test_all_regions(self): + all_regions = skew.arn.Region('arn:aws:*:*:*:*', 'arn:aws:*:*:*:*').__getattribute__('_all_region_names') + self.assertEqual(len(all_regions), 22) diff --git a/tox.ini b/tox.ini index 5114e5a..1f5a7b9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26,py27,py33,py34 +envlist = py36,py37,py38,py39 [testenv] commands = nosetests tests/unit