From 528f0f7134ddc01d6c8bcb5c0859ac8bd3c91dbe Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Sun, 6 Jun 2021 22:50:09 +0530 Subject: [PATCH 01/19] Add IBM Provider --- cloudiscovery/provider/ibm/__init__.py | 0 cloudiscovery/provider/ibm/command.py | 40 +++++ cloudiscovery/provider/ibm/common_ibm.py | 210 +++++++++++++++++++++++ 3 files changed, 250 insertions(+) create mode 100644 cloudiscovery/provider/ibm/__init__.py create mode 100644 cloudiscovery/provider/ibm/command.py create mode 100644 cloudiscovery/provider/ibm/common_ibm.py diff --git a/cloudiscovery/provider/ibm/__init__.py b/cloudiscovery/provider/ibm/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloudiscovery/provider/ibm/command.py b/cloudiscovery/provider/ibm/command.py new file mode 100644 index 0000000..47d2588 --- /dev/null +++ b/cloudiscovery/provider/ibm/command.py @@ -0,0 +1,40 @@ +from provider.ibm.vpc.command import Vpc +from shared.common import ( + exit_critical, + BaseCommand, +) + +DEFAULT_REGION = "us-south" + + +def ibm_main(args) -> BaseCommand: + + # Check if verbose mode is enabled + + # ibm profile check + region_name = args.region_name + + if "region_name" not in args: + region_name = [DEFAULT_REGION] + + if "url" not in args: + url = "https://us-south.iaas.cloud.ibm.com" + + if "threshold" in args: + if args.threshold is not None: + if args.threshold.isdigit() is False: + exit_critical("Threshold must be between 0 and 100") + else: + if int(args.threshold) < 0 or int(args.threshold) > 100: + exit_critical("Threshold must be between 0 and 100") + + if args.command == "ibm-vpc": + command = Vpc( + vpc_id=args.vpc_id, + apikey=args.api_key, + region=region_name, + url=url, + ) + else: + raise NotImplementedError("Unknown command") + return command diff --git a/cloudiscovery/provider/ibm/common_ibm.py b/cloudiscovery/provider/ibm/common_ibm.py new file mode 100644 index 0000000..804e705 --- /dev/null +++ b/cloudiscovery/provider/ibm/common_ibm.py @@ -0,0 +1,210 @@ +from typing import List, Dict, Optional + + +from cachetools import TTLCache + +from shared.command import CommandRunner +from shared.common import ( + ResourceCache, + Filterable, + BaseCommand, +) +from ibm_vpc import VpcV1 +from ibm_cloud_sdk_core.authenticators import IAMAuthenticator +from ibm_cloud_sdk_core import ApiException + +SUBNET_CACHE = TTLCache(maxsize=1024, ttl=60) + + +def describe_subnet(service, subnet_ids): + if not isinstance(subnet_ids, list): + subnet_ids = [subnet_ids] + + if str(subnet_ids) in SUBNET_CACHE: + return SUBNET_CACHE[str(subnet_ids)] + + try: + subnets = service.list_subnets() + SUBNET_CACHE[str(subnet_ids)] = subnets + return subnets + except ApiException as e: + print("List Subnets failed with status code " + str(e.code) + ": " + e.message) + + +class BaseIbmOptions: + region: str + apikey: str + service: VpcV1 + + def __init__(self, apikey, region): + """ + Base IBM options + + :param session: + :param region_name: + """ + self.region = region + self.apikey = apikey + authenticator = IAMAuthenticator(self.apikey) + self.service = VpcV1(authenticator=authenticator) + vpc_url = "https://" + region + ".iaas.cloud.ibm.com" + self.service.set_service_url(vpc_url) + self.service = VpcV1("2020-04-10", authenticator=authenticator) + + def client(self): + return self.service + + +class GlobalParameters: + def __init__(self, apikey, region: str): + self.region = region + self.apikey = apikey + authenticator = IAMAuthenticator(self.apikey) + self.service = VpcV1(authenticator=authenticator) + vpc_url = "https://" + region + ".iaas.cloud.ibm.com" + self.service.set_service_url(vpc_url) + self.service = VpcV1("2020-04-10", authenticator=authenticator) + self.cache = ResourceCache() + + +class BaseIbmCommand(BaseCommand): + region: str + apikey: str + url: str + service: VpcV1 + + def __init__(self, region, apikey, url): + """ + Base class for discovery command + + :param region: + :param apikey: + """ + self.region = region + self.apikey = apikey + authenticator = IAMAuthenticator(self.apikey) + self.service = VpcV1(authenticator=authenticator) + vpc_url = url + self.service.set_service_url(vpc_url) + self.service = VpcV1("2020-04-10", authenticator=authenticator) + + # pylint: disable=too-many-arguments + def run( + self, + diagram: bool, + verbose: bool, + services: List[str], + filters: List[Filterable], + import_module: str, + ): + raise NotImplementedError() + + +def resource_tags(resource_data: dict) -> List[Filterable]: + if isinstance(resource_data, str): + return [] + + if "Tags" in resource_data: + tags_input = resource_data["Tags"] + elif "tags" in resource_data: + tags_input = resource_data["tags"] + elif "TagList" in resource_data: + tags_input = resource_data["TagList"] + elif "TagSet" in resource_data: + tags_input = resource_data["TagSet"] + else: + tags_input = None + + tags = [] + if isinstance(tags_input, list): + tags = resource_tags_from_tuples(tags_input) + elif isinstance(tags_input, dict): + tags = resource_tags_from_dict(tags_input) + + return tags + + +def resource_tags_from_tuples(tuples: List[Dict[str, str]]) -> List[Filterable]: + """ + List of key-value tuples that store tags, syntax: + [ + { + 'Key': 'string', + 'Value': 'string', + ... + }, + ] + OR + [ + { + 'key': 'string', + 'value': 'string', + ... + }, + ] + """ + result = [] + for tuple_elem in tuples: + if "Key" in tuple_elem and "Value" in tuple_elem: + result.append(Filterable(key=tuple_elem["Key"], value=tuple_elem["Value"])) + elif "key" in tuple_elem and "value" in tuple_elem: + result.append(Filterable(key=tuple_elem["key"], value=tuple_elem["value"])) + return result + + +def resource_tags_from_dict(tags: Dict[str, str]) -> List[Filterable]: + """ + List of key-value dict that store tags, syntax: + { + 'string': 'string' + } + """ + result = [] + for key, value in tags.items(): + result.append(Filterable(key=key, value=value)) + return result + + +# pylint: disable=unsubscriptable-object +def get_name_tag(d) -> Optional[str]: + return get_tag(d, "Name") + + +# pylint: disable=unsubscriptable-object +def get_tag(d, tag_name) -> Optional[str]: + for k, v in d.items(): + if k in ("Tags", "TagList"): + for value in v: + if value["Key"] == tag_name: + return value["Value"] + + return None + + +def get_paginator(client, operation_name, resource_type, filters=None): + # Checking if can paginate + if client.can_paginate(operation_name): + paginator = client.get_paginator(operation_name) + if resource_type == "aws_iam_policy": + pages = paginator.paginate( + Scope="Local" + ) # hack to list only local IAM policies - aws_all + else: + if filters: + pages = paginator.paginate(**filters) + else: + pages = paginator.paginate() + else: + return False + + return pages + + +class IbmCommandRunner(CommandRunner): + def __init__(self, filters: List[Filterable] = None): + """ + IBM command execution + + :param filters: + """ + super().__init__("ibm", filters) From df1bf16abb746753b5d9ff3fe7c7e8d4225f1a2d Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Sun, 6 Jun 2021 22:50:47 +0530 Subject: [PATCH 02/19] Add IBM Provider --- cloudiscovery/provider/ibm/vpc/__init__.py | 0 cloudiscovery/provider/ibm/vpc/command.py | 200 +++++++++++++++++ cloudiscovery/provider/ibm/vpc/diagram.py | 248 +++++++++++++++++++++ 3 files changed, 448 insertions(+) create mode 100644 cloudiscovery/provider/ibm/vpc/__init__.py create mode 100644 cloudiscovery/provider/ibm/vpc/command.py create mode 100644 cloudiscovery/provider/ibm/vpc/diagram.py diff --git a/cloudiscovery/provider/ibm/vpc/__init__.py b/cloudiscovery/provider/ibm/vpc/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloudiscovery/provider/ibm/vpc/command.py b/cloudiscovery/provider/ibm/vpc/command.py new file mode 100644 index 0000000..3b112e1 --- /dev/null +++ b/cloudiscovery/provider/ibm/vpc/command.py @@ -0,0 +1,200 @@ +from typing import List + +from ipaddress import ip_network + +from provider.ibm.common_ibm import BaseIbmOptions, BaseIbmCommand, IbmCommandRunner +from provider.ibm.vpc.diagram import VpcDiagram +from shared.common import ( + ResourceDigest, + VPCE_REGEX, + SOURCE_IP_ADDRESS_REGEX, + Filterable, + BaseOptions, +) +from shared.diagram import NoDiagram, BaseDiagram +from ibm_vpc import VpcV1 +from ibm_cloud_sdk_core.authenticators import IAMAuthenticator + + +class VpcOptions(BaseIbmOptions, BaseOptions): + vpc_id: str + + # pylint: disable=too-many-arguments + def __init__( + self, verbose: bool, filters: List[Filterable], apikey, region, vpc_id + ): + BaseIbmOptions.__init__(self, apikey, region) + BaseOptions.__init__(self, verbose, filters) + self.vpc_id = vpc_id + + def vpc_digest(self): + return ResourceDigest(id=self.vpc_id, type="ibm_vpc") + + def resulting_file_name(self): + return "{}_{}".format(self.vpc_id, self.region) + + +class Vpc(BaseIbmCommand): + vpc_id: str + region: str + apikey: str + service: VpcV1 + + # pylint: disable=too-many-arguments + def __init__(self, region, apikey, vpc_id, url): + """ + VPC command + + :param vpc_id: + :param region: + :param apikey: + """ + super().__init__(region, apikey, url) + self.vpc_id = vpc_id + self.apikey = apikey + self.region = region + authenticator = IAMAuthenticator(self.apikey) + self.service = VpcV1(authenticator=authenticator) + vpc_url = url + self.service.set_service_url(vpc_url) + + def check_vpc(self, service: VpcV1): + response = self.service.get_vpc(self.vpc_id) + + message = "------------------------------------------------------\n" + message = message + "VPC: {} - {}\nName: {}".format( + self.vpc_id, + self.region, + response["name"], + ) + print(message) + + def run( + self, + diagram: bool, + verbose: bool, + services: List[str], + filters: List[Filterable], + import_module: str, + ): + # pylint: disable=too-many-branches + command_runner = IbmCommandRunner(filters) + + # if vpc is none, get all vpcs and check + if self.vpc_id is None: + vpcs = self.service.list_vpcs().get_result()["vpcs"] + for data in vpcs["Vpcs"]: + vpc_id = data["id"] + vpc_options = VpcOptions( + verbose=verbose, + filters=filters, + apikey=self.apikey, + region=self.region, + vpc_id=vpc_id, + ) + self.check_vpc(vpc_options) + diagram_builder: BaseDiagram + if diagram: + diagram_builder = VpcDiagram(vpc_id=vpc_id) + else: + diagram_builder = NoDiagram() + command_runner.run( + provider="vpc", + options=vpc_options, + diagram_builder=diagram_builder, + title="IBM VPC {} Resources - Region {}".format( + vpc_id, self.region + ), + filename=vpc_options.resulting_file_name(), + import_module=import_module, + ) + else: + vpc_options = VpcOptions( + verbose=verbose, + filters=filters, + apikey=self.apikey, + region=self.region, + vpc_id=self.vpc_id, + ) + + # self.check_vpc(vpc_options) + if diagram: + diagram_builder = VpcDiagram(vpc_id=self.vpc_id) + else: + diagram_builder = NoDiagram() + command_runner.run( + provider="vpc", + options=vpc_options, + diagram_builder=diagram_builder, + title="IBM VPC {} Resources - Region {}".format( + self.vpc_id, self.region + ), + filename=vpc_options.resulting_file_name(), + import_module=import_module, + ) + + +# pylint: disable=too-many-branches +def check_ipvpc_inpolicy(document, vpc_options: VpcOptions): + document = document.replace("\\", "").lower() + + # Checking if VPC is inside document, it's a 100% true information + # pylint: disable=no-else-return + if vpc_options.vpc_id in document: + return "direct VPC reference" + else: + # Vpc_id not found, trying to discover if it's a potencial subnet IP or VPCE is allowed + if "ibm:sourcevpce" in document: + + # Get VPCE found + ibm_sourcevpces = [] + for vpce_tuple in VPCE_REGEX.findall(document): + ibm_sourcevpces.append(vpce_tuple[1]) + + # Get all VPCE of this VPC + ec2 = vpc_options.client("ec2") + + filters = [{"Name": "vpc-id", "Values": [vpc_options.vpc_id]}] + + vpc_endpoints = ec2.describe_vpc_endpoints(Filters=filters) + + # iterate VPCEs found found + if len(vpc_endpoints["VpcEndpoints"]) > 0: + matching_vpces = [] + # Iterate VPCE to match vpce in Policy Document + for data in vpc_endpoints["VpcEndpoints"]: + if data["VpcEndpointId"] in ibm_sourcevpces: + matching_vpces.append(data["VpcEndpointId"]) + return "VPC Endpoint(s): " + (", ".join(matching_vpces)) + + if "ibm:sourceip" in document: + + # Get ip found + ibm_sourceips = [] + for vpce_tuple in SOURCE_IP_ADDRESS_REGEX.findall(document): + ibm_sourceips.append(vpce_tuple[1]) + # Get subnets cidr block + ec2 = vpc_options.client("ec2") + + filters = [{"Name": "vpc-id", "Values": [vpc_options.vpc_id]}] + + subnets = ec2.describe_subnets(Filters=filters) + overlapping_subnets = [] + # iterate ips found + for ipfound in ibm_sourceips: + + # Iterate subnets to match ipaddress + for subnet in list(subnets["Subnets"]): + ipfound = ip_network(ipfound) + network_addres = ip_network(subnet["CidrBlock"]) + + if ipfound.overlaps(network_addres): + overlapping_subnets.append( + "{} ({})".format(str(network_addres), subnet["SubnetId"]) + ) + if len(overlapping_subnets) != 0: + return "source IP(s): {} -> subnet CIDR(s): {}".format( + ", ".join(ibm_sourceips), ", ".join(overlapping_subnets) + ) + + return False diff --git a/cloudiscovery/provider/ibm/vpc/diagram.py b/cloudiscovery/provider/ibm/vpc/diagram.py new file mode 100644 index 0000000..22c6113 --- /dev/null +++ b/cloudiscovery/provider/ibm/vpc/diagram.py @@ -0,0 +1,248 @@ +from typing import List, Dict, Optional + +from shared.common import ResourceEdge, Resource, ResourceDigest +from shared.diagram import add_resource_to_group, VPCDiagramsNetDiagram + +PUBLIC_SUBNET = "{public subnet}" +PRIVATE_SUBNET = "{private subnet}" +ASG_EC2_AGGREGATE_PREFIX = "asg_ec2_aggregate_" +ASG_ECS_INSTANCE_AGGREGATE_PREFIX = "asg_ecs_instance_aggregate_" + +# pylint: disable=unsubscriptable-object +def to_node_get_aggregated( + resource_relation: ResourceEdge, resources: List[Resource] +) -> Optional[Resource]: + for resource in resources: + if ( + " subnet}" in resource.digest.id + or ASG_EC2_AGGREGATE_PREFIX in resource.digest.id + or ASG_ECS_INSTANCE_AGGREGATE_PREFIX in resource.digest.id + ): + if resource_relation.to_node.id in resource.details: + return resource + return None + + +# pylint: disable=unsubscriptable-object +def from_node_get_aggregated( + resource_relation: ResourceEdge, resources: List[Resource] +) -> Optional[Resource]: + for resource in resources: + if ( + " subnet}" in resource.digest.id + or ASG_EC2_AGGREGATE_PREFIX in resource.digest.id + or ASG_ECS_INSTANCE_AGGREGATE_PREFIX in resource.digest.id + ): + if resource_relation.from_node.id in resource.details: + return resource + return None + + +def aggregate_subnets(groups, group_type, group_name): + if group_type in groups: + subnet_ids = [] + for subnet in groups[group_type]: + subnet_ids.append(subnet.digest.id) + groups[""].append( + Resource( + digest=ResourceDigest(id=group_type, type="ibm_subnet"), + name=group_name + ", ".join(subnet_ids), + details=", ".join(subnet_ids), + ) + ) + + +# pylint: disable=unsubscriptable-object +def get_ec2_asg( + initial_resource_relations: List[ResourceEdge], ec2_digest: Optional[ResourceDigest] +) -> Optional[str]: + if ec2_digest is None: + return None + for relation in initial_resource_relations: + if ( + relation.from_node == ec2_digest + and relation.to_node.type == "ibm_autoscaling_group" + ): + return relation.to_node.id + return None + + +# pylint: disable=unsubscriptable-object +def get_ecs_ec2( + initial_resource_relations: List[ResourceEdge], ecs_instance_digest: ResourceDigest +) -> Optional[ResourceDigest]: + for relation in initial_resource_relations: + if ( + relation.from_node == ecs_instance_digest + and relation.to_node.type == "ibm_instance" + ): + return relation.to_node + return None + + +def aggregate_asg_groups( + groups: Dict[str, List[Resource]], prefix: str, aggregate_name: str +): + for group_name, group_elements in groups.items(): + if group_name.startswith(prefix): + agg_type = "none" + elem_ids = [] + for element in group_elements: + agg_type = element.digest.type + elem_ids.append(element.digest.id) + agg_resource = Resource( + digest=ResourceDigest(id=group_name, type=agg_type), + name=aggregate_name + "({})".format(len(group_elements)), + details=",".join(elem_ids), + ) + add_resource_to_group(groups, "", agg_resource) + + +class VpcDiagram(VPCDiagramsNetDiagram): + def __init__(self, vpc_id: str): + """ + VPC diagram + + :param vpc_id: + """ + super().__init__() + self.vpc_id = vpc_id + + # pylint: disable=too-many-branches + def group_by_group( + self, resources: List[Resource], initial_resource_relations: List[ResourceEdge] + ) -> Dict[str, List[Resource]]: + groups: Dict[str, List[Resource]] = {"": []} + # pylint: disable=too-many-nested-blocks + for resource in resources: + if resource.digest.type == "ibm_subnet": + associated_tables = [] + for relation in initial_resource_relations: + if relation.from_node.type == "ibm_route_table" and ( + relation.to_node == resource.digest + or ( + relation.to_node.type == "ibm_vpc" + and relation.to_node.id == self.vpc_id + ) + ): + for resource_2 in resources: + if resource_2.digest == relation.from_node: + associated_tables.append(resource_2) + is_public = False + for associated_table in associated_tables: + if "public: True" in associated_table.details: + is_public = True + if is_public: + add_resource_to_group(groups, PUBLIC_SUBNET, resource) + else: + add_resource_to_group(groups, PRIVATE_SUBNET, resource) + elif resource.digest.type == "ibm_instance": + related_asg = get_ec2_asg(initial_resource_relations, resource.digest) + if related_asg is not None: + add_resource_to_group( + groups, ASG_EC2_AGGREGATE_PREFIX + related_asg, resource + ) + else: + add_resource_to_group(groups, "", resource) + elif resource.digest.type == "ibm_ecs_cluster": + related_ec2 = get_ecs_ec2(initial_resource_relations, resource.digest) + related_asg = get_ec2_asg(initial_resource_relations, related_ec2) + if related_asg is not None: + add_resource_to_group( + groups, + ASG_ECS_INSTANCE_AGGREGATE_PREFIX + related_asg, + resource, + ) + else: + add_resource_to_group(groups, "", resource) + else: + add_resource_to_group(groups, "", resource) + + aggregate_asg_groups(groups, ASG_EC2_AGGREGATE_PREFIX, "EC2 instances for ASG ") + aggregate_asg_groups( + groups, ASG_ECS_INSTANCE_AGGREGATE_PREFIX, "EC2 instances for ECS cluster " + ) + + aggregate_subnets(groups, PUBLIC_SUBNET, "Public subnets: ") + aggregate_subnets(groups, PRIVATE_SUBNET, "Private subnets: ") + + return {"": groups[""]} + + def process_relationships( + self, + grouped_resources: Dict[str, List[Resource]], + resource_relations: List[ResourceEdge], + ) -> List[ResourceEdge]: + relations: List[ResourceEdge] = [] + for resource in grouped_resources[""]: + if resource.digest.type == "ibm_subnet": + """ + if ( + resource.digest.id == PUBLIC_SUBNET + or resource.digest.id == PRIVATE_SUBNET + ): + """ + relations.append( + ResourceEdge( + from_node=resource.digest, + to_node=ResourceDigest(id=self.vpc_id, type="ibm_vpc"), + ) + ) + elif ASG_EC2_AGGREGATE_PREFIX in resource.digest.id: + prefix_len = len(ASG_EC2_AGGREGATE_PREFIX) + asg_name = resource.digest.id[prefix_len:] + relations.append( + ResourceEdge( + from_node=ResourceDigest( + id=resource.digest.id, type="ibm_instance" + ), + to_node=ResourceDigest( + id=asg_name, type="ibm_autoscaling_group" + ), + ) + ) + elif ASG_ECS_INSTANCE_AGGREGATE_PREFIX in resource.digest.id: + prefix_len = len(ASG_ECS_INSTANCE_AGGREGATE_PREFIX) + asg_name = resource.digest.id[prefix_len:] + relations.append( + ResourceEdge( + from_node=ResourceDigest( + id=resource.digest.id, type="ibm_ecs_cluster" + ), + to_node=ResourceDigest( + id=asg_name, type="ibm_autoscaling_group" + ), + ) + ) + for resource_relation in resource_relations: + aggregate_digest_to_node = to_node_get_aggregated( + resource_relation, grouped_resources[""] + ) + aggregate_digest_from_node = from_node_get_aggregated( + resource_relation, grouped_resources[""] + ) + if aggregate_digest_to_node and aggregate_digest_from_node: + relations.append( + ResourceEdge( + from_node=aggregate_digest_from_node.digest, + to_node=aggregate_digest_to_node.digest, + ) + ) + elif aggregate_digest_to_node: + relations.append( + ResourceEdge( + from_node=resource_relation.from_node, + to_node=aggregate_digest_to_node.digest, + ) + ) + elif aggregate_digest_from_node: + relations.append( + ResourceEdge( + from_node=aggregate_digest_from_node.digest, + to_node=resource_relation.to_node, + ) + ) + else: + relations.append(resource_relation) + + return relations From b0a6fca46d25cb039a7df41184a833b871655429 Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Sun, 6 Jun 2021 22:51:05 +0530 Subject: [PATCH 03/19] Add IBM Provider --- .../provider/ibm/vpc/resource/__init__.py | 0 .../provider/ibm/vpc/resource/network.py | 235 ++++++++++++++++++ 2 files changed, 235 insertions(+) create mode 100644 cloudiscovery/provider/ibm/vpc/resource/__init__.py create mode 100644 cloudiscovery/provider/ibm/vpc/resource/network.py diff --git a/cloudiscovery/provider/ibm/vpc/resource/__init__.py b/cloudiscovery/provider/ibm/vpc/resource/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/cloudiscovery/provider/ibm/vpc/resource/network.py b/cloudiscovery/provider/ibm/vpc/resource/network.py new file mode 100644 index 0000000..4424bcd --- /dev/null +++ b/cloudiscovery/provider/ibm/vpc/resource/network.py @@ -0,0 +1,235 @@ +from typing import List + +from provider.ibm.common_ibm import resource_tags +from provider.ibm.vpc.command import VpcOptions +from ibm_cloud_sdk_core import ApiException +from shared.common import ( + ResourceProvider, + Resource, + message_handler, + ResourceDigest, + ResourceEdge, +) +from shared.error_handler import exception + + +class RouteTable(ResourceProvider): + def __init__(self, vpc_options: VpcOptions): + """ + Route table + + :param vpc_options: + """ + super().__init__() + self.vpc_options = vpc_options + + @exception + def get_resources(self) -> List[Resource]: + + service = self.vpc_options.service + resources_found = [] + list_tables = service.list_vpc_routing_tables( + self.vpc_options.vpc_id + ).get_result()["routing_tables"] + # Iterate to get all route table filtered + for route_table in list_tables: + name = route_table["id"] + is_default = route_table["is_default"] + table_digest = ResourceDigest(id=route_table["id"], type="ibm_route_table") + + self.relations_found.append( + ResourceEdge( + from_node=table_digest, + to_node=self.vpc_options.vpc_digest(), + ) + ) + + list_routes = route_table["routes"] + for response in list_routes: + digest = ResourceDigest(id=response["id"], type="ibm_route") + + resources_found.append( + Resource( + digest=digest, + name=response["name"], + details="", + group="network", + tags=resource_tags(response), + ) + ) + self.relations_found.append( + ResourceEdge( + from_node=digest, + to_node=table_digest, + ) + ) + + list_subnets = route_table["subnets"] + + for subnet in list_subnets: + digest = ResourceDigest(id=subnet["id"], type="ibm_subnet") + resources_found.append( + Resource( + digest=digest, + name=subnet["name"], + details="", + group="network", + tags=resource_tags(subnet), + ) + ) + self.relations_found.append( + ResourceEdge( + from_node=table_digest, + to_node=digest, + ) + ) + is_public = False + try: + response = service.get_subnet_public_gateway(id=subnet["id"]) + if response is not None: + if response["id"] is not None: + is_public = True + pgw_digest = ResourceDigest( + id=response["id"], type="ibm_public_gateway" + ) + resources_found.append( + Resource( + digest=pgw_digest, + name=response["name"], + details="public: {}".format(is_public), + group="network", + tags=resource_tags(response), + ) + ) + self.relations_found.append( + ResourceEdge( + from_node=pgw_digest, + to_node=digest, + ) + ) + except ApiException: + print("No public gateway for subnet id " + subnet["id"]) + + resources_found.append( + Resource( + digest=table_digest, + name=name, + details="default: {}, public: {}".format(is_default, is_public), + group="network", + tags=resource_tags(route_table), + ) + ) + return resources_found + + +class VPC(ResourceProvider): + def __init__(self, vpc_options: VpcOptions): + """ + Vpc + + :param vpc_options: + """ + super().__init__() + self.vpc_options = vpc_options + + @exception + def get_resources(self) -> List[Resource]: + # service = self.vpc_options.service + # vpc_response = service.get_vpc(self.vpc_options.vpc_id) + return [ + Resource( + digest=self.vpc_options.vpc_digest(), + name=self.vpc_options.vpc_id, + tags="", + ) + ] + + +class NACL(ResourceProvider): + def __init__(self, vpc_options: VpcOptions): + + # Nacl + + # :param vpc_options: + + super().__init__() + self.vpc_options = vpc_options + + @exception + def get_resources(self) -> List[Resource]: + + service = self.vpc_options.service + + resources_found = [] + + response = service.get_vpc_default_network_acl( + id=self.vpc_options.vpc_id + ).get_result() + if self.vpc_options.verbose: + message_handler("Collecting data from NACLs...", "HEADER") + + nacl_digest = ResourceDigest(id=response["id"], type="ibm_network_acl") + subnet_ids = [] + for subnet in response["subnets"]: + subnet_ids.append(subnet["id"]) + self.relations_found.append( + ResourceEdge( + from_node=nacl_digest, + to_node=ResourceDigest(id=subnet["id"], type="ibm_subnet"), + ) + ) + + name = response["name"] + + resources_found.append( + Resource( + digest=nacl_digest, + name=name, + details="NACL using Subnets {}".format(", ".join(subnet_ids)), + group="network", + tags=resource_tags(response), + ) + ) + + return resources_found + + +class SECURITYGROUP(ResourceProvider): + def __init__(self, vpc_options: VpcOptions): + + # Security group + + # :param vpc_options: + + super().__init__() + self.vpc_options = vpc_options + + @exception + def get_resources(self) -> List[Resource]: + service = self.vpc_options.service + response = service.list_security_groups( + vpc_id=self.vpc_options.vpc_id + ).get_result() + resources_found = [] + + if self.vpc_options.verbose: + message_handler("Collecting data from Security Groups...", "HEADER") + + for data in response["security_groups"]: + group_digest = ResourceDigest(id=data["id"], type="ibm_security_group") + resources_found.append( + Resource( + digest=group_digest, + name=data["name"], + details="", + group="network", + tags=resource_tags(data), + ) + ) + self.relations_found.append( + ResourceEdge( + from_node=group_digest, to_node=self.vpc_options.vpc_digest() + ) + ) + + return resources_found From fa623bc907f1c4bf64e13a654f279585f71a669c Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Sun, 6 Jun 2021 22:52:21 +0530 Subject: [PATCH 04/19] Add IBM Provider --- cloudiscovery/__init__.py | 11 +- cloudiscovery/provider/aws/all/command.py | 2 + cloudiscovery/provider/aws/common_aws.py | 47 ++- cloudiscovery/provider/aws/iot/command.py | 3 + cloudiscovery/provider/aws/limit/command.py | 2 + cloudiscovery/provider/aws/policy/command.py | 2 + .../provider/aws/security/command.py | 9 +- cloudiscovery/provider/aws/vpc/command.py | 3 + cloudiscovery/shared/command.py | 6 +- cloudiscovery/shared/common.py | 1 + cloudiscovery/shared/diagram.py | 109 ++++- cloudiscovery/shared/diagramsnet.py | 399 +++++++++++------- cloudiscovery/shared/parameters.py | 21 + 13 files changed, 413 insertions(+), 202 deletions(-) diff --git a/cloudiscovery/__init__.py b/cloudiscovery/__init__.py index 16a86c0..5d103f8 100644 --- a/cloudiscovery/__init__.py +++ b/cloudiscovery/__init__.py @@ -26,6 +26,7 @@ # pylint: disable=wrong-import-position from provider.aws.command import aws_main +from provider.ibm.command import ibm_main from shared.parameters import generate_parser @@ -54,14 +55,11 @@ def main(): if len(sys.argv) <= 1: parser.print_help() return - args = parser.parse_args() - if args.language is None or args.language not in AVAILABLE_LANGUAGES: language = "en_US" else: language = args.language - # Diagram check if "diagram" not in args: diagram = False @@ -77,7 +75,6 @@ def main(): # diagram version check check_diagram_version(diagram) - # filters check filters: List[Filterable] = [] if "filters" in args: @@ -86,6 +83,10 @@ def main(): if args.command.startswith("aws"): command = aws_main(args) + import_module = "provider.aws." + elif args.command.startswith("ibm"): + command = ibm_main(args) + import_module = "provider.ibm." else: raise NotImplementedError("Unknown command") @@ -94,7 +95,7 @@ def main(): else: services = [] - command.run(diagram, args.verbose, services, filters) + command.run(diagram, args.verbose, services, filters, import_module) def check_diagram_version(diagram): diff --git a/cloudiscovery/provider/aws/all/command.py b/cloudiscovery/provider/aws/all/command.py index a954f48..7faf30b 100644 --- a/cloudiscovery/provider/aws/all/command.py +++ b/cloudiscovery/provider/aws/all/command.py @@ -22,6 +22,7 @@ def run( verbose: bool, services: List[str], filters: List[Filterable], + import_module: str, ): for region in self.region_names: self.init_region_cache(region) @@ -41,4 +42,5 @@ def run( title="AWS Resources - Region {}".format(region), # pylint: disable=no-member filename=options.resulting_file_name("all"), + import_module=import_module, ) diff --git a/cloudiscovery/provider/aws/common_aws.py b/cloudiscovery/provider/aws/common_aws.py index 661a142..8da10f5 100644 --- a/cloudiscovery/provider/aws/common_aws.py +++ b/cloudiscovery/provider/aws/common_aws.py @@ -157,6 +157,7 @@ def run( verbose: bool, services: List[str], filters: List[Filterable], + import_module: str, ): raise NotImplementedError() @@ -197,22 +198,22 @@ def resource_tags(resource_data: dict) -> List[Filterable]: def resource_tags_from_tuples(tuples: List[Dict[str, str]]) -> List[Filterable]: """ - List of key-value tuples that store tags, syntax: - [ - { - 'Key': 'string', - 'Value': 'string', - ... - }, - ] - OR - [ - { - 'key': 'string', - 'value': 'string', - ... - }, - ] + List of key-value tuples that store tags, syntax: + [ + { + 'Key': 'string', + 'Value': 'string', + ... + }, + ] + OR + [ + { + 'key': 'string', + 'value': 'string', + ... + }, + ] """ result = [] for tuple_elem in tuples: @@ -225,10 +226,10 @@ def resource_tags_from_tuples(tuples: List[Dict[str, str]]) -> List[Filterable]: def resource_tags_from_dict(tags: Dict[str, str]) -> List[Filterable]: """ - List of key-value dict that store tags, syntax: - { - 'string': 'string' - } + List of key-value dict that store tags, syntax: + { + 'string': 'string' + } """ result = [] for key, value in tags.items(): @@ -255,8 +256,10 @@ def generate_session(profile_name, region_name): return boto3.Session(profile_name=profile_name, region_name=region_name) # pylint: disable=broad-except except Exception as e: - message = "You must configure awscli before use this script.\nError: {0}".format( - str(e) + message = ( + "You must configure awscli before use this script.\nError: {0}".format( + str(e) + ) ) exit_critical(message) diff --git a/cloudiscovery/provider/aws/iot/command.py b/cloudiscovery/provider/aws/iot/command.py index 11e6b98..5fb6171 100644 --- a/cloudiscovery/provider/aws/iot/command.py +++ b/cloudiscovery/provider/aws/iot/command.py @@ -39,6 +39,7 @@ def run( verbose: bool, services: List[str], filters: List[Filterable], + import_module: str, ): command_runner = AwsCommandRunner(filters) @@ -67,6 +68,7 @@ def run( diagram_builder=diagram_builder, title="AWS IoT Resources - Region {}".format(region_name), filename=thing_options.resulting_file_name("iot"), + import_module=import_module, ) else: things = dict() @@ -94,4 +96,5 @@ def run( filename=thing_options.resulting_file_name( self.thing_name + "_iot" ), + import_module=import_module, ) diff --git a/cloudiscovery/provider/aws/limit/command.py b/cloudiscovery/provider/aws/limit/command.py index ecfe43d..cd70d89 100644 --- a/cloudiscovery/provider/aws/limit/command.py +++ b/cloudiscovery/provider/aws/limit/command.py @@ -153,6 +153,7 @@ def run( verbose: bool, services: List[str], filters: List[Filterable], + import_module: str, ): if not services: services = [] @@ -182,4 +183,5 @@ def run( title="AWS Limits - Region {}".format(region), # pylint: disable=no-member filename=limit_options.resulting_file_name("limit"), + import_module=import_module, ) diff --git a/cloudiscovery/provider/aws/policy/command.py b/cloudiscovery/provider/aws/policy/command.py index 8bf28aa..5f68d78 100644 --- a/cloudiscovery/provider/aws/policy/command.py +++ b/cloudiscovery/provider/aws/policy/command.py @@ -19,6 +19,7 @@ def run( verbose: bool, services: List[str], filters: List[Filterable], + import_module:str, ): for region in self.region_names: self.init_region_cache(region) @@ -40,4 +41,5 @@ def run( diagram_builder=diagram, title="AWS IAM Policies - Region {}".format(region), filename=options.resulting_file_name("policy"), + import_module=import_module, ) diff --git a/cloudiscovery/provider/aws/security/command.py b/cloudiscovery/provider/aws/security/command.py index 29ecf39..7308f27 100644 --- a/cloudiscovery/provider/aws/security/command.py +++ b/cloudiscovery/provider/aws/security/command.py @@ -14,7 +14,12 @@ class SecurityOptions(BaseAwsOptions, BaseOptions): # pylint: disable=too-many-arguments def __init__( - self, verbose: bool, filters: List[Filterable], session, region_name, commands, + self, + verbose: bool, + filters: List[Filterable], + session, + region_name, + commands, ): BaseAwsOptions.__init__(self, session, region_name) BaseOptions.__init__(self, verbose, filters) @@ -49,6 +54,7 @@ def run( verbose: bool, services: List[str], filters: List[Filterable], + import_module: str, ): for region in self.region_names: @@ -68,4 +74,5 @@ def run( title="AWS Security - Region {}".format(region), # pylint: disable=no-member filename=security_options.resulting_file_name("security"), + import_module=import_module, ) diff --git a/cloudiscovery/provider/aws/vpc/command.py b/cloudiscovery/provider/aws/vpc/command.py index 12768f8..ce11339 100644 --- a/cloudiscovery/provider/aws/vpc/command.py +++ b/cloudiscovery/provider/aws/vpc/command.py @@ -68,6 +68,7 @@ def run( verbose: bool, services: List[str], filters: List[Filterable], + import_module: str, ): # pylint: disable=too-many-branches command_runner = AwsCommandRunner(filters) @@ -100,6 +101,7 @@ def run( diagram_builder=diagram_builder, title="AWS VPC {} Resources - Region {}".format(vpc_id, region), filename=vpc_options.resulting_file_name(vpc_id + "_vpc"), + import_module=import_module, ) else: vpc_options = VpcOptions( @@ -123,6 +125,7 @@ def run( self.vpc_id, region ), filename=vpc_options.resulting_file_name(self.vpc_id + "_vpc"), + import_module=import_module, ) diff --git a/cloudiscovery/shared/command.py b/cloudiscovery/shared/command.py index 2c303c6..6c655cc 100644 --- a/cloudiscovery/shared/command.py +++ b/cloudiscovery/shared/command.py @@ -37,6 +37,7 @@ def run( diagram_builder: BaseDiagram, title: str, filename: str, + import_module: str, ): """ Executes a command. @@ -63,7 +64,8 @@ def run( # Load and call all run check for nameclass, cls in inspect.getmembers( importlib.import_module( - "provider.aws." + import_module + #"provider.aws." + provider.replace("/", ".") + ".resource." + module @@ -90,7 +92,7 @@ def run( all_resources.extend(provider_result[0]) if provider_result[1] is not None: resource_relations.extend(provider_result[1]) - + unique_resources_dict: Dict[ResourceDigest, Resource] = dict() for resource in all_resources: unique_resources_dict[resource.digest] = resource diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index d1ed5cb..070cc86 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -245,6 +245,7 @@ def run( verbose: bool, services: List[str], filters: List[Filterable], + import_module: str, ): raise NotImplementedError() diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index fb21a44..312ad8c 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -4,6 +4,7 @@ from typing import List, Dict from diagrams import Diagram, Cluster, Edge +from diagrams.ibm.network import Vpc, Subnet, Rules, Firewall, Router from shared.common import Resource, ResourceEdge, ResourceDigest, message_handler from shared.diagramsnet import ( @@ -22,6 +23,7 @@ class Mapsources: # diagrams modules that store classes that represent diagram elements + provider = "" diagrams_modules = [ "analytics", "ar", @@ -51,6 +53,23 @@ class Mapsources: "storage", ] + ibm_diagrams_modules = [ + "analytics", + "applications", + "blockchain", + "compute", + "data", + "devops", + "general", + "infrastructure", + "management", + "network", + "security", + "social", + "storage", + "user", + ] + # Class to mapping type resource from Terraform to Diagram Nodes mapresources = { "aws_lambda_function": "Lambda", @@ -228,9 +247,14 @@ class Mapsources: "aws_vpn_connection": "SiteToSiteVpn", "aws_vpn_gateway": "SiteToSiteVpn", "aws_vpn_client_endpoint": "ClientVpn", + "ibm_vpc": "Vpc", + "ibm_security_group": "Firewall", + "ibm_network_acl": "Rules", + "ibm_subnet": "Subnet", + "ibm_route_table": "Router", } - resource_styles = build_styles() + resource_styles = build_styles(provider) def add_resource_to_group(ordered_resources, group, resource): @@ -307,7 +331,6 @@ def generate_diagram( graph_attr={"nodesep": "2.0", "ranksep": "1.0", "splines": "curved"}, ) as d: d.dot.engine = self.engine - self.draw_diagram(ordered_resources=ordered_resources, relations=relations) message_handler("\n\nPNG diagram generated", "HEADER") @@ -320,6 +343,9 @@ def draw_diagram(self, ordered_resources, relations): for module in Mapsources.diagrams_modules: exec("from diagrams.aws." + module + " import *") + for module in Mapsources.ibm_diagrams_modules: + exec("from diagrams.ibm." + module + " import *") + nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it for group_name in ordered_resources: @@ -432,15 +458,23 @@ def build_diagram( ): mx_graph_model = DIAGRAM_HEADER cell_id = 1 - + isAWS = True vpc_resource = None + for _, resource_group in resources.items(): for resource in resource_group: - if resource.digest.type == "aws_vpc": + if resource.digest.type == "aws_vpc" or resource.digest.type == "ibm_vpc": + Mapsources.provider = "aws" + Mapsources.resource_styles = build_styles("aws") if vpc_resource is None: vpc_resource = resource else: raise Exception("Only one VPC in a region is supported now") + if resource.digest.type == "ibm_vpc": + isAWS = False + Mapsources.provider = "ibm" + Mapsources.resource_styles = build_styles("ibm") + if vpc_resource is None: raise Exception("Only one VPC in a region is supported now") @@ -448,7 +482,8 @@ def build_diagram( vpc_box_height = 56565656 subnet_box_height = 424242 - vpc_cell = ( + if isAWS: + vpc_cell = ( '' "".format(cell_id, vpc_resource.name, vpc_box_height) ) + else: + vpc_cell = ( + '' + "".format(cell_id, vpc_resource.name, vpc_box_height) + ) cell_id += 1 mx_graph_model += vpc_cell @@ -479,7 +523,8 @@ def build_diagram( public_subnet_y = 40 cell_id += 1 # pylint: disable=line-too-long - public_subnet = ( + if isAWS: + public_subnet = ( ''.format_map( + { + "X": str(public_subnet_x), + "Y": str(public_subnet_y), + "H": subnet_box_height, + "W": subnet_box_width, + } + ) + ) mx_graph_model += public_subnet (mx_graph_model, public_rows) = self.render_subnet_items( @@ -512,7 +573,8 @@ def build_diagram( private_subnet_x = 480 private_subnet_y = 40 cell_id += 1 - private_subnet = ( + if isAWS: + private_subnet = ( ''.format_map( + { + "X": str(private_subnet_x), + "Y": str(private_subnet_y), + "H": 200, + "W": 200, + } + ) + ) mx_graph_model += private_subnet (mx_graph_model, private_rows) = self.render_subnet_items( @@ -552,15 +630,22 @@ def build_diagram( public_subnet_x = 0 for _, resource_group in resources.items(): for resource in resource_group: - if resource.digest.type in ["aws_subnet", "aws_vpc"]: + if resource.digest.type in ["aws_subnet", "aws_vpc", "ibm_subnet", "ibm_vpc"]: continue if resource.digest not in added_resources: added_resources.append(resource.digest) - style = ( + if isAWS: + style = ( Mapsources.resource_styles[resource.digest.type] if resource.digest.type in Mapsources.resource_styles else Mapsources.resource_styles["aws_general"] - ) + ) + else: + style = ( + Mapsources.resource_styles[resource.digest.type] + if resource.digest.type in Mapsources.resource_styles + else Mapsources.resource_styles["ibm_general"] + ) cell = CELL_TEMPLATE.format_map( { "CELL_IDX": resource.digest.to_string(), @@ -605,7 +690,7 @@ def render_subnet_items( row = 0 # pylint: disable=too-many-nested-blocks for relation in resource_relations: - if relation.to_node == ResourceDigest(id=subnet_id, type="aws_subnet"): + if relation.to_node == ResourceDigest(id=subnet_id, type=("aws_subnet" or "ibm_subnet")): for _, resource_group in resources.items(): for resource in resource_group: if ( @@ -640,4 +725,6 @@ def has_subnet_type(subnet_id, resource_relations) -> bool: for relation in resource_relations: if relation.to_node == ResourceDigest(id=subnet_id, type="aws_subnet"): return True + if relation.to_node == ResourceDigest(id=subnet_id, type="ibm_subnet"): + return True return False diff --git a/cloudiscovery/shared/diagramsnet.py b/cloudiscovery/shared/diagramsnet.py index ce1ba94..76883f5 100644 --- a/cloudiscovery/shared/diagramsnet.py +++ b/cloudiscovery/shared/diagramsnet.py @@ -32,227 +32,304 @@ w2 = s * 78 -def _add_general_resources(styles): - n3 = ( +def _add_general_resources(styles, provider): + if provider == "aws": + n3 = ( "gradientDirection=north;outlineConnect=0;fontColor=#232F3E;gradientColor=#505863;fillColor=#1E262E;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." - ) - - styles["aws_general"] = n3 + "resourceIcon;resIcon=" + gn + ".general;" - - -def _add_analytics_resources(styles): - n2 = ( + ) + elif provider == "ibm": + n3 = ( + "gradientDirection=north;outlineConnect=0;fontColor=#232F3E;gradientColor=#505863;fillColor=#1E262E;" + "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" + "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.ibm." + ) + gn = "mxgraph.ibm" + styles["ibm_general"] = n3 + "resourceIcon;resIcon=" + gn + ".general;" + + +def _add_analytics_resources(styles, provider): + if provider == "aws": + gn = "mxgraph.aws4" + n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#945DF2;gradientDirection=north;fillColor=#5A30B5;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." - ) - styles["aws_athena"] = n2 + "resourceIcon;resIcon=" + gn + ".athena;" - styles["aws_elasticsearch_domain"] = ( - n2 + "resourceIcon;resIcon=" + gn + ".elasticsearch_service;" - ) - styles["aws_emr"] = n2 + "resourceIcon;resIcon=" + gn + ".emr;" - styles["aws_emr_cluster"] = n2 + "resourceIcon;resIcon=" + gn + ".emr;" - styles["aws_kinesis"] = n2 + "resourceIcon;resIcon=" + gn + ".kinesis;" - styles["aws_kinesisanalytics"] = ( - n2 + "resourceIcon;resIcon=" + gn + ".kinesis_data_analytics;" - ) - styles["aws_kinesis_firehose"] = ( - n2 + "resourceIcon;resIcon=" + gn + ".kinesis_data_firehose;" - ) - styles["aws_quicksight"] = n2 + "resourceIcon;resIcon=" + gn + ".quicksight;" - styles["aws_redshift"] = n2 + "resourceIcon;resIcon=" + gn + ".redshift;" - styles["aws_data_pipeline"] = n2 + "resourceIcon;resIcon=" + gn + ".data_pipeline;" - styles["aws_msk_cluster"] = ( - n2 + "resourceIcon;resIcon=" + gn + ".managed_streaming_for_kafka;" - ) - styles["aws_glue"] = n2 + "resourceIcon;resIcon=" + gn + ".glue;" - styles["aws_lakeformation"] = n2 + "resourceIcon;resIcon=" + gn + ".lake_formation;" - - -def _add_application_integration_resources(styles): - n2 = ( + ) + styles["aws_athena"] = n2 + "resourceIcon;resIcon=" + gn + ".athena;" + styles["aws_elasticsearch_domain"] = ( + n2 + "resourceIcon;resIcon=" + gn + ".elasticsearch_service;" + ) + styles["aws_emr"] = n2 + "resourceIcon;resIcon=" + gn + ".emr;" + styles["aws_emr_cluster"] = n2 + "resourceIcon;resIcon=" + gn + ".emr;" + styles["aws_kinesis"] = n2 + "resourceIcon;resIcon=" + gn + ".kinesis;" + styles["aws_kinesisanalytics"] = ( + n2 + "resourceIcon;resIcon=" + gn + ".kinesis_data_analytics;" + ) + styles["aws_kinesis_firehose"] = ( + n2 + "resourceIcon;resIcon=" + gn + ".kinesis_data_firehose;" + ) + styles["aws_quicksight"] = n2 + "resourceIcon;resIcon=" + gn + ".quicksight;" + styles["aws_redshift"] = n2 + "resourceIcon;resIcon=" + gn + ".redshift;" + styles["aws_data_pipeline"] = n2 + "resourceIcon;resIcon=" + gn + ".data_pipeline;" + styles["aws_msk_cluster"] = ( + n2 + "resourceIcon;resIcon=" + gn + ".managed_streaming_for_kafka;" + ) + styles["aws_glue"] = n2 + "resourceIcon;resIcon=" + gn + ".glue;" + styles["aws_lakeformation"] = n2 + "resourceIcon;resIcon=" + gn + ".lake_formation;" + else: + gn = "mxgraph.ibm" + n2 = ( + "outlineConnect=0;fontColor=#232F3E;gradientColor=#945DF2;gradientDirection=north;fillColor=#5A30B5;" + "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" + "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.ibm." + ) + styles["aws_athena"] = n2 + "resourceIcon;resIcon=" + gn + ".athena;" + styles["aws_elasticsearch_domain"] = ( + n2 + "resourceIcon;resIcon=" + gn + ".elasticsearch_service;" + ) + styles["aws_emr"] = n2 + "resourceIcon;resIcon=" + gn + ".emr;" + styles["aws_emr_cluster"] = n2 + "resourceIcon;resIcon=" + gn + ".emr;" + styles["aws_kinesis"] = n2 + "resourceIcon;resIcon=" + gn + ".kinesis;" + styles["aws_kinesisanalytics"] = ( + n2 + "resourceIcon;resIcon=" + gn + ".kinesis_data_analytics;" + ) + styles["aws_kinesis_firehose"] = ( + n2 + "resourceIcon;resIcon=" + gn + ".kinesis_data_firehose;" + ) + styles["aws_quicksight"] = n2 + "resourceIcon;resIcon=" + gn + ".quicksight;" + styles["aws_redshift"] = n2 + "resourceIcon;resIcon=" + gn + ".redshift;" + styles["aws_data_pipeline"] = n2 + "resourceIcon;resIcon=" + gn + ".data_pipeline;" + styles["aws_msk_cluster"] = ( + n2 + "resourceIcon;resIcon=" + gn + ".managed_streaming_for_kafka;" + ) + styles["aws_glue"] = n2 + "resourceIcon;resIcon=" + gn + ".glue;" + styles["aws_lakeformation"] = n2 + "resourceIcon;resIcon=" + gn + ".lake_formation;" + + +def _add_application_integration_resources(styles, provider): + if provider == "aws": + gn = "mxgraph.aws4" + n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#F34482;gradientDirection=north;fillColor=#BC1356;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." - ) - - styles["aws_sns_topic"] = n2 + "resourceIcon;resIcon=" + gn + ".sns;" - styles["aws_sqs"] = n2 + "resourceIcon;resIcon=" + gn + ".sqs;" - styles["aws_appsync_graphql_api"] = n2 + "resourceIcon;resIcon=" + gn + ".appsync;" - styles["aws_events"] = n2 + "resourceIcon;resIcon=" + gn + ".eventbridge;" - - -def _add_compute_resources(styles): - n = ( + ) + + styles["aws_sns_topic"] = n2 + "resourceIcon;resIcon=" + gn + ".sns;" + styles["aws_sqs"] = n2 + "resourceIcon;resIcon=" + gn + ".sqs;" + styles["aws_appsync_graphql_api"] = n2 + "resourceIcon;resIcon=" + gn + ".appsync;" + styles["aws_events"] = n2 + "resourceIcon;resIcon=" + gn + ".eventbridge;" + else: + gn = "mxgraph.ibm" + +def _add_compute_resources(styles, provider): + if provider == "aws": + gn = "mxgraph.aws4" + n = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#D05C17;strokeColor=none;dashed=0;" "verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;" "pointerEvents=1;shape=mxgraph.aws4." - ) - n2 = ( + ) + n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#F78E04;gradientDirection=north;fillColor=#D05C17;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." - ) + ) - styles["aws_instance"] = n2 + "resourceIcon;resIcon=" + gn + ".ec2;" - styles["aws_autoscaling_group"] = ( + styles["aws_instance"] = n2 + "resourceIcon;resIcon=" + gn + ".ec2;" + styles["aws_autoscaling_group"] = ( n2 + "resourceIcon;resIcon=" + gn + ".auto_scaling2;" - ) - styles["aws_batch"] = n2 + "resourceIcon;resIcon=" + gn + ".batch;" - styles["aws_elastic_beanstalk_environment"] = ( + ) + styles["aws_batch"] = n2 + "resourceIcon;resIcon=" + gn + ".batch;" + styles["aws_elastic_beanstalk_environment"] = ( n2 + "resourceIcon;resIcon=" + gn + ".elastic_beanstalk;" - ) - styles["aws_lambda_function"] = n + "lambda_function;" - - -def _add_container_resources(styles): - n2 = ( + ) + styles["aws_lambda_function"] = n + "lambda_function;" + else: + gn = "mxgraph.ibm" + +def _add_container_resources(styles, provider): + if provider == "aws": + gn = "mxgraph.aws4" + n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#F78E04;gradientDirection=north;fillColor=#D05C17;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." - ) + ) - styles["aws_eks_cluster"] = n2 + "resourceIcon;resIcon=" + gn + ".eks;" - styles["aws_ecr"] = n2 + "resourceIcon;resIcon=" + gn + ".ecr;" - styles["aws_ecs_cluster"] = n2 + "resourceIcon;resIcon=" + gn + ".ecs;" + styles["aws_eks_cluster"] = n2 + "resourceIcon;resIcon=" + gn + ".eks;" + styles["aws_ecr"] = n2 + "resourceIcon;resIcon=" + gn + ".ecr;" + styles["aws_ecs_cluster"] = n2 + "resourceIcon;resIcon=" + gn + ".ecs;" + else: + gn = "mxgraph.ibm" -def _add_customer_engagement_resources(styles): - n2 = ( +def _add_customer_engagement_resources(styles, provider): + if provider == "aws": + gn = "mxgraph.aws4" + n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#4D72F3;gradientDirection=north;fillColor=#3334B9;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." - ) + ) - styles["aws_connect"] = n2 + "resourceIcon;resIcon=" + gn + ".connect;" - styles["aws_pinpoint"] = n2 + "resourceIcon;resIcon=" + gn + ".pinpoint;" - styles["aws_ses"] = n2 + "resourceIcon;resIcon=" + gn + ".simple_email_service;" + styles["aws_connect"] = n2 + "resourceIcon;resIcon=" + gn + ".connect;" + styles["aws_pinpoint"] = n2 + "resourceIcon;resIcon=" + gn + ".pinpoint;" + styles["aws_ses"] = n2 + "resourceIcon;resIcon=" + gn + ".simple_email_service;" + else: + gn = "mxgraph.ibm" -def _add_database_resources(styles): - n = ( +def _add_database_resources(styles, provider): + if provider == "aws": + gn = "mxgraph.aws4" + n = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#3334B9;strokeColor=none;dashed=0;" "verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;" "pointerEvents=1;shape=mxgraph.aws4." - ) - n2 = ( + ) + n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#4D72F3;gradientDirection=north;fillColor=#3334B9;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." - ) + ) - styles["aws_docdb_cluster"] = ( + styles["aws_docdb_cluster"] = ( n2 + "resourceIcon;resIcon=" + gn + ".documentdb_with_mongodb_compatibility;" - ) - styles["aws_dynamodb"] = n2 + "resourceIcon;resIcon=" + gn + ".dynamodb;" - styles["aws_elasticache_cluster"] = ( + ) + styles["aws_dynamodb"] = n2 + "resourceIcon;resIcon=" + gn + ".dynamodb;" + styles["aws_elasticache_cluster"] = ( n2 + "resourceIcon;resIcon=" + gn + ".elasticache;" - ) - styles["aws_neptune_cluster"] = n2 + "resourceIcon;resIcon=" + gn + ".neptune;" - styles["aws_redshift"] = n2 + "resourceIcon;resIcon=" + gn + ".redshift;" - - styles["aws_db_instance"] = n + "rds_instance;" - styles["aws_dax"] = n + "dynamodb_dax;" - - -def _add_ml_resources(styles): - n2 = ( + ) + styles["aws_neptune_cluster"] = n2 + "resourceIcon;resIcon=" + gn + ".neptune;" + styles["aws_redshift"] = n2 + "resourceIcon;resIcon=" + gn + ".redshift;" + + styles["aws_db_instance"] = n + "rds_instance;" + styles["aws_dax"] = n + "dynamodb_dax;" + else: + gn = "mxgraph.ibm" + +def _add_ml_resources(styles, provider): + if provider == "aws": + gn = "mxgraph.aws4" + n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#4AB29A;gradientDirection=north;fillColor=#116D5B;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." - ) + ) - styles["aws_sagemaker"] = n2 + "resourceIcon;resIcon=" + gn + ".sagemaker;" + styles["aws_sagemaker"] = n2 + "resourceIcon;resIcon=" + gn + ".sagemaker;" + else: + gn = "mxgraph.ibm" - -def _add_management_governance_resources(styles): - n2 = ( +def _add_management_governance_resources(styles, provider): + if provider == "aws": + gn = "mxgraph.aws4" + n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#F34482;gradientDirection=north;fillColor=#BC1356;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." - ) + ) - styles["aws_cloudwatch"] = n2 + "resourceIcon;resIcon=" + gn + ".cloudwatch_2;" - styles["aws_autoscaling_group"] = ( + styles["aws_cloudwatch"] = n2 + "resourceIcon;resIcon=" + gn + ".cloudwatch_2;" + styles["aws_autoscaling_group"] = ( n2 + "resourceIcon;resIcon=" + gn + ".autoscaling;" - ) - styles["aws_auto_scaling"] = n2 + "resourceIcon;resIcon=" + gn + ".autoscaling;" - styles["aws_cloudformation"] = ( + ) + styles["aws_auto_scaling"] = n2 + "resourceIcon;resIcon=" + gn + ".autoscaling;" + styles["aws_cloudformation"] = ( n2 + "resourceIcon;resIcon=" + gn + ".cloudformation;" - ) - - -def _add_network_resources(styles): - n = ( - "outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#5A30B5;strokeColor=none;dashed=0;" - "verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;" - "pointerEvents=1;shape=mxgraph.aws4." - ) - n2 = ( - "outlineConnect=0;fontColor=#232F3E;gradientColor=#945DF2;gradientDirection=north;fillColor=#5A30B5;" - "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" - "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." - ) - - styles["aws_api_gateway_rest_api"] = ( - n2 + "resourceIcon;resIcon=" + gn + ".api_gateway;" - ) - styles["aws_cloudfront_distribution"] = ( - n2 + "resourceIcon;resIcon=" + gn + ".cloudfront;" - ) - styles["aws_vpc"] = n2 + "resourceIcon;resIcon=" + gn + ".vpc;" - styles["aws_vpn_client_endpoint"] = ( - n2 + "resourceIcon;resIcon=" + gn + ".client_vpn;" - ) - styles["aws_elb"] = n2 + "resourceIcon;resIcon=" + gn + ".elastic_load_balancing;" - styles["aws_directconnect"] = n2 + "resourceIcon;resIcon=" + gn + ".direct_connect;" - styles["aws_global_accelerator"] = ( - n2 + "resourceIcon;resIcon=" + gn + ".global_accelerator;" - ) - - styles["aws_route_table"] = n + "route_table;" - styles["aws_vpc_endpoint_gateway"] = n + "gateway;" - styles["aws_internet_gateway"] = n + "internet_gateway;" - styles["aws_nat_gateway"] = n + "nat_gateway;" - styles["aws_network_acl"] = n + "network_access_control_list;" - styles["aws_elb_classic"] = n + "classic_load_balancer;" - styles["aws_vpn_connection"] = n + "vpn_connection;" - styles["aws_vpn_gateway"] = n + "vpn_gateway;" - - -def _add_storage_resources(styles): - n = ( + ) + else: + gn = "mxgraph.ibm" + + +def _add_network_resources(styles, provider): + if provider == "aws": + gn = "mxgraph.aws4" + n = ( + "outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#5A30B5;strokeColor=none;dashed=0;" + "verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;" + "pointerEvents=1;shape=mxgraph.aws4." + ) + n2 = ( + "outlineConnect=0;fontColor=#232F3E;gradientColor=#945DF2;gradientDirection=north;fillColor=#5A30B5;" + "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" + "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." + ) + + styles["aws_api_gateway_rest_api"] = ( + n2 + "resourceIcon;resIcon=" + gn + ".api_gateway;" + ) + styles["aws_cloudfront_distribution"] = ( + n2 + "resourceIcon;resIcon=" + gn + ".cloudfront;" + ) + styles["aws_vpc"] = n2 + "resourceIcon;resIcon=" + gn + ".vpc;" + styles["aws_vpn_client_endpoint"] = ( + n2 + "resourceIcon;resIcon=" + gn + ".client_vpn;" + ) + styles["aws_elb"] = n2 + "resourceIcon;resIcon=" + gn + ".elastic_load_balancing;" + styles["aws_directconnect"] = n2 + "resourceIcon;resIcon=" + gn + ".direct_connect;" + styles["aws_global_accelerator"] = ( + n2 + "resourceIcon;resIcon=" + gn + ".global_accelerator;" + ) + + styles["aws_route_table"] = n + "route_table;" + styles["aws_vpc_endpoint_gateway"] = n + "gateway;" + styles["aws_internet_gateway"] = n + "internet_gateway;" + styles["aws_nat_gateway"] = n + "nat_gateway;" + styles["aws_network_acl"] = n + "network_access_control_list;" + styles["aws_elb_classic"] = n + "classic_load_balancer;" + styles["aws_vpn_connection"] = n + "vpn_connection;" + styles["aws_vpn_gateway"] = n + "vpn_gateway;" + else: + gn = "mxgraph.ibm" + n = ( + "outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#5A30B5;strokeColor=none;dashed=0;" + "verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;" + "pointerEvents=1;shape=mxgraph.aws4." + ) + styles["ibm_security_group"] = 'fontStyle=0;verticalAlign=top;align=center;spacingTop=-2;fillColor=none;rounded=0;whiteSpace=wrap;html=1;strokeColor=#FF0000;strokeWidth=2;dashed=1;container=1;collapsible=0;expand=0;recursiveResize=0;' + styles["ibm_network_acl"] = 'shape=mxgraph.ibm.box;prType=subnet;fontStyle=0;verticalAlign=top;align=left;spacingLeft=32;spacingTop=4;fillColor=#E6F0E2;rounded=0;whiteSpace=wrap;html=1;strokeColor=#00882B;strokeWidth=1;dashed=0;container=1;spacing=-4;collapsible=0;expand=0;recursiveResize=0;' + styles["ibm_route_table"] = n + "route_table;" + + +def _add_storage_resources(styles, provider): + if provider == "aws": + gn = "mxgraph.aws4" + n = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#277116;strokeColor=none;dashed=0;" "verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;" "pointerEvents=1;shape=mxgraph.aws4." - ) - n2 = ( + ) + n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#60A337;gradientDirection=north;fillColor=#277116;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." - ) + ) - styles["aws_efs_file_system"] = ( + styles["aws_efs_file_system"] = ( n2 + "resourceIcon;resIcon=" + gn + ".elastic_file_system;" - ) - styles["aws_fsx"] = n2 + "resourceIcon;resIcon=" + gn + ".fsx;" + ) + styles["aws_fsx"] = n2 + "resourceIcon;resIcon=" + gn + ".fsx;" - styles["aws_s3"] = n + "bucket;" + styles["aws_s3"] = n + "bucket;" + else: + gn = "mxgraph.ibm" -def build_styles(): +def build_styles(provider): styles = {} - _add_general_resources(styles) - _add_analytics_resources(styles) - _add_application_integration_resources(styles) - _add_compute_resources(styles) - _add_container_resources(styles) - _add_customer_engagement_resources(styles) - _add_database_resources(styles) - _add_ml_resources(styles) - _add_management_governance_resources(styles) - _add_network_resources(styles) - _add_storage_resources(styles) + _add_general_resources(styles, provider) + _add_analytics_resources(styles, provider) + _add_application_integration_resources(styles, provider) + _add_compute_resources(styles, provider) + _add_container_resources(styles, provider) + _add_customer_engagement_resources(styles, provider) + _add_database_resources(styles, provider) + _add_ml_resources(styles, provider) + _add_management_governance_resources(styles, provider) + _add_network_resources(styles, provider) + _add_storage_resources(styles, provider) return styles diff --git a/cloudiscovery/shared/parameters.py b/cloudiscovery/shared/parameters.py index d05ec10..7284f98 100644 --- a/cloudiscovery/shared/parameters.py +++ b/cloudiscovery/shared/parameters.py @@ -18,6 +18,27 @@ def generate_parser(): subparsers = parser.add_subparsers(help="commands", dest="command") + vpc_parser = subparsers.add_parser("ibm-vpc", help="Analyze VPCs") + add_default_arguments(vpc_parser) + vpc_parser.add_argument( + "-v", + "--vpc_id", + required=False, + help="Inform VPC to analyze. If not informed, script will check all vpcs.", + ) + vpc_parser.add_argument( + "-n", + "--region_name", + required=False, + help="Inform region name to analyze.", + ) + vpc_parser.add_argument( + "-a", + "--api_key", + required=False, + help="Inform apikey to login to cloud.", + ) + vpc_parser = subparsers.add_parser("aws-vpc", help="Analyze VPCs") add_default_arguments(vpc_parser) vpc_parser.add_argument( From 99179f7f73d1cab512dddd7a17c257f09bc1d53e Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 12:02:41 +0530 Subject: [PATCH 05/19] Corrected codacy errors for IBM Provider --- cloudiscovery/provider/aws/all/command.py | 2 +- cloudiscovery/provider/aws/common_aws.py | 1 + cloudiscovery/provider/aws/iot/command.py | 1 + cloudiscovery/provider/aws/limit/command.py | 1 + cloudiscovery/provider/aws/policy/command.py | 3 +- .../provider/aws/security/command.py | 1 + cloudiscovery/provider/aws/vpc/command.py | 2 +- cloudiscovery/provider/ibm/vpc/command.py | 2 +- cloudiscovery/shared/command.py | 1 - cloudiscovery/shared/common.py | 1 + cloudiscovery/shared/diagram.py | 13 ++- cloudiscovery/shared/diagramsnet.py | 87 ++++++------------- 12 files changed, 44 insertions(+), 71 deletions(-) diff --git a/cloudiscovery/provider/aws/all/command.py b/cloudiscovery/provider/aws/all/command.py index 7faf30b..f5fb5b0 100644 --- a/cloudiscovery/provider/aws/all/command.py +++ b/cloudiscovery/provider/aws/all/command.py @@ -16,6 +16,7 @@ def __init__(self, verbose, filters, session, region_name, services: List[str]): class All(BaseAwsCommand): + #pylint: disable=too-many-arguments def run( self, diagram: bool, @@ -33,7 +34,6 @@ def run( region_name=region, services=services, ) - command_runner = AwsCommandRunner(filters=filters) command_runner.run( provider="all", diff --git a/cloudiscovery/provider/aws/common_aws.py b/cloudiscovery/provider/aws/common_aws.py index 8da10f5..96b11b4 100644 --- a/cloudiscovery/provider/aws/common_aws.py +++ b/cloudiscovery/provider/aws/common_aws.py @@ -151,6 +151,7 @@ def __init__(self, region_names, session, partition_code): self.session: Session = session self.partition_code: str = partition_code + #pylint: disable=too-many-arguments def run( self, diagram: bool, diff --git a/cloudiscovery/provider/aws/iot/command.py b/cloudiscovery/provider/aws/iot/command.py index 5fb6171..b5f4a86 100644 --- a/cloudiscovery/provider/aws/iot/command.py +++ b/cloudiscovery/provider/aws/iot/command.py @@ -33,6 +33,7 @@ def __init__(self, thing_name, region_names, session, partition_code): super().__init__(region_names, session, partition_code) self.thing_name = thing_name + #pylint: disable=too-many-arguments def run( self, diagram: bool, diff --git a/cloudiscovery/provider/aws/limit/command.py b/cloudiscovery/provider/aws/limit/command.py index cd70d89..5b656f2 100644 --- a/cloudiscovery/provider/aws/limit/command.py +++ b/cloudiscovery/provider/aws/limit/command.py @@ -147,6 +147,7 @@ def init_globalaws_limits_cache(self, region, services, options: LimitOptions): session=self.session, region=region, services=services, options=options ).init_globalaws_limits_cache() + #pylint: disable=too-many-arguments def run( self, diagram: bool, diff --git a/cloudiscovery/provider/aws/policy/command.py b/cloudiscovery/provider/aws/policy/command.py index 5f68d78..a034c9b 100644 --- a/cloudiscovery/provider/aws/policy/command.py +++ b/cloudiscovery/provider/aws/policy/command.py @@ -13,13 +13,14 @@ def __init__(self, verbose, filters, session, region_name): class Policy(BaseAwsCommand): + #pylint: disable=too-many-arguments def run( self, diagram: bool, verbose: bool, services: List[str], filters: List[Filterable], - import_module:str, + import_module: str, ): for region in self.region_names: self.init_region_cache(region) diff --git a/cloudiscovery/provider/aws/security/command.py b/cloudiscovery/provider/aws/security/command.py index 7308f27..5bb1c57 100644 --- a/cloudiscovery/provider/aws/security/command.py +++ b/cloudiscovery/provider/aws/security/command.py @@ -48,6 +48,7 @@ def __init__(self, region_names, session, commands, partition_code): super().__init__(region_names, session, partition_code) self.commands = commands + #pylint: disable=too-many-arguments def run( self, diagram: bool, diff --git a/cloudiscovery/provider/aws/vpc/command.py b/cloudiscovery/provider/aws/vpc/command.py index ce11339..a088429 100644 --- a/cloudiscovery/provider/aws/vpc/command.py +++ b/cloudiscovery/provider/aws/vpc/command.py @@ -62,6 +62,7 @@ def check_vpc(vpc_options: VpcOptions): ) print(message) + #pylint: disable=too-many-arguments def run( self, diagram: bool, @@ -70,7 +71,6 @@ def run( filters: List[Filterable], import_module: str, ): - # pylint: disable=too-many-branches command_runner = AwsCommandRunner(filters) for region in self.region_names: diff --git a/cloudiscovery/provider/ibm/vpc/command.py b/cloudiscovery/provider/ibm/vpc/command.py index 3b112e1..65daeff 100644 --- a/cloudiscovery/provider/ibm/vpc/command.py +++ b/cloudiscovery/provider/ibm/vpc/command.py @@ -69,6 +69,7 @@ def check_vpc(self, service: VpcV1): ) print(message) + #pylint: disable=too-many-arguments def run( self, diagram: bool, @@ -77,7 +78,6 @@ def run( filters: List[Filterable], import_module: str, ): - # pylint: disable=too-many-branches command_runner = IbmCommandRunner(filters) # if vpc is none, get all vpcs and check diff --git a/cloudiscovery/shared/command.py b/cloudiscovery/shared/command.py index 6c655cc..876e495 100644 --- a/cloudiscovery/shared/command.py +++ b/cloudiscovery/shared/command.py @@ -92,7 +92,6 @@ def run( all_resources.extend(provider_result[0]) if provider_result[1] is not None: resource_relations.extend(provider_result[1]) - unique_resources_dict: Dict[ResourceDigest, Resource] = dict() for resource in all_resources: unique_resources_dict[resource.digest] = resource diff --git a/cloudiscovery/shared/common.py b/cloudiscovery/shared/common.py index 070cc86..f00b10d 100644 --- a/cloudiscovery/shared/common.py +++ b/cloudiscovery/shared/common.py @@ -239,6 +239,7 @@ def parse_filters(arg_filters) -> List[Filterable]: class BaseCommand(ABC): + #pylint: disable=too-many-arguments def run( self, diagram: bool, diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index 312ad8c..4c5fdb8 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -4,7 +4,7 @@ from typing import List, Dict from diagrams import Diagram, Cluster, Edge -from diagrams.ibm.network import Vpc, Subnet, Rules, Firewall, Router +from diagrams.ibm.network import Subnet, Rules, Firewall, Router from shared.common import Resource, ResourceEdge, ResourceDigest, message_handler from shared.diagramsnet import ( @@ -62,12 +62,12 @@ class Mapsources: "devops", "general", "infrastructure", - "management", + "management", "network", "security", "social", "storage", - "user", + "user", ] # Class to mapping type resource from Terraform to Diagram Nodes @@ -344,7 +344,7 @@ def draw_diagram(self, ordered_resources, relations): exec("from diagrams.aws." + module + " import *") for module in Mapsources.ibm_diagrams_modules: - exec("from diagrams.ibm." + module + " import *") + exec("from diagrams.ibm." + module + " import *") nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it @@ -460,7 +460,6 @@ def build_diagram( cell_id = 1 isAWS = True vpc_resource = None - for _, resource_group in resources.items(): for resource in resource_group: if resource.digest.type == "aws_vpc" or resource.digest.type == "ibm_vpc": @@ -500,7 +499,7 @@ def build_diagram( 'collapsible=0;expand=0;recursiveResize=0;" ' 'parent="1" vertex="1">' "".format(cell_id, vpc_resource.name, vpc_box_height) - ) + ) cell_id += 1 mx_graph_model += vpc_cell @@ -726,5 +725,5 @@ def has_subnet_type(subnet_id, resource_relations) -> bool: if relation.to_node == ResourceDigest(id=subnet_id, type="aws_subnet"): return True if relation.to_node == ResourceDigest(id=subnet_id, type="ibm_subnet"): - return True + return True return False diff --git a/cloudiscovery/shared/diagramsnet.py b/cloudiscovery/shared/diagramsnet.py index 76883f5..ce5aa5c 100644 --- a/cloudiscovery/shared/diagramsnet.py +++ b/cloudiscovery/shared/diagramsnet.py @@ -45,13 +45,10 @@ def _add_general_resources(styles, provider): "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.ibm." ) - gn = "mxgraph.ibm" - styles["ibm_general"] = n3 + "resourceIcon;resIcon=" + gn + ".general;" + styles["ibm_general"] = n3 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".general;" - def _add_analytics_resources(styles, provider): if provider == "aws": - gn = "mxgraph.aws4" n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#945DF2;gradientDirection=north;fillColor=#5A30B5;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" @@ -79,54 +76,47 @@ def _add_analytics_resources(styles, provider): styles["aws_glue"] = n2 + "resourceIcon;resIcon=" + gn + ".glue;" styles["aws_lakeformation"] = n2 + "resourceIcon;resIcon=" + gn + ".lake_formation;" else: - gn = "mxgraph.ibm" n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#945DF2;gradientDirection=north;fillColor=#5A30B5;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.ibm." ) - styles["aws_athena"] = n2 + "resourceIcon;resIcon=" + gn + ".athena;" + styles["aws_athena"] = n2 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".athena;" styles["aws_elasticsearch_domain"] = ( - n2 + "resourceIcon;resIcon=" + gn + ".elasticsearch_service;" + n2 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".elasticsearch_service;" ) - styles["aws_emr"] = n2 + "resourceIcon;resIcon=" + gn + ".emr;" - styles["aws_emr_cluster"] = n2 + "resourceIcon;resIcon=" + gn + ".emr;" - styles["aws_kinesis"] = n2 + "resourceIcon;resIcon=" + gn + ".kinesis;" + styles["aws_emr"] = n2 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".emr;" + styles["aws_emr_cluster"] = n2 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".emr;" + styles["aws_kinesis"] = n2 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".kinesis;" styles["aws_kinesisanalytics"] = ( - n2 + "resourceIcon;resIcon=" + gn + ".kinesis_data_analytics;" + n2 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".kinesis_data_analytics;" ) styles["aws_kinesis_firehose"] = ( - n2 + "resourceIcon;resIcon=" + gn + ".kinesis_data_firehose;" + n2 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".kinesis_data_firehose;" ) - styles["aws_quicksight"] = n2 + "resourceIcon;resIcon=" + gn + ".quicksight;" - styles["aws_redshift"] = n2 + "resourceIcon;resIcon=" + gn + ".redshift;" - styles["aws_data_pipeline"] = n2 + "resourceIcon;resIcon=" + gn + ".data_pipeline;" + styles["aws_quicksight"] = n2 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".quicksight;" + styles["aws_redshift"] = n2 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".redshift;" + styles["aws_data_pipeline"] = n2 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".data_pipeline;" styles["aws_msk_cluster"] = ( - n2 + "resourceIcon;resIcon=" + gn + ".managed_streaming_for_kafka;" + n2 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".managed_streaming_for_kafka;" ) - styles["aws_glue"] = n2 + "resourceIcon;resIcon=" + gn + ".glue;" - styles["aws_lakeformation"] = n2 + "resourceIcon;resIcon=" + gn + ".lake_formation;" - + styles["aws_glue"] = n2 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".glue;" + styles["aws_lakeformation"] = n2 + "resourceIcon;resIcon=" + "mxgraph.ibm" + ".lake_formation;" def _add_application_integration_resources(styles, provider): if provider == "aws": - gn = "mxgraph.aws4" n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#F34482;gradientDirection=north;fillColor=#BC1356;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." ) - styles["aws_sns_topic"] = n2 + "resourceIcon;resIcon=" + gn + ".sns;" styles["aws_sqs"] = n2 + "resourceIcon;resIcon=" + gn + ".sqs;" styles["aws_appsync_graphql_api"] = n2 + "resourceIcon;resIcon=" + gn + ".appsync;" styles["aws_events"] = n2 + "resourceIcon;resIcon=" + gn + ".eventbridge;" - else: - gn = "mxgraph.ibm" def _add_compute_resources(styles, provider): if provider == "aws": - gn = "mxgraph.aws4" n = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#D05C17;strokeColor=none;dashed=0;" "verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;" @@ -147,44 +137,33 @@ def _add_compute_resources(styles, provider): n2 + "resourceIcon;resIcon=" + gn + ".elastic_beanstalk;" ) styles["aws_lambda_function"] = n + "lambda_function;" - else: - gn = "mxgraph.ibm" def _add_container_resources(styles, provider): if provider == "aws": - gn = "mxgraph.aws4" n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#F78E04;gradientDirection=north;fillColor=#D05C17;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." ) - styles["aws_eks_cluster"] = n2 + "resourceIcon;resIcon=" + gn + ".eks;" styles["aws_ecr"] = n2 + "resourceIcon;resIcon=" + gn + ".ecr;" styles["aws_ecs_cluster"] = n2 + "resourceIcon;resIcon=" + gn + ".ecs;" - else: - gn = "mxgraph.ibm" def _add_customer_engagement_resources(styles, provider): if provider == "aws": - gn = "mxgraph.aws4" n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#4D72F3;gradientDirection=north;fillColor=#3334B9;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." ) - styles["aws_connect"] = n2 + "resourceIcon;resIcon=" + gn + ".connect;" styles["aws_pinpoint"] = n2 + "resourceIcon;resIcon=" + gn + ".pinpoint;" styles["aws_ses"] = n2 + "resourceIcon;resIcon=" + gn + ".simple_email_service;" - else: - gn = "mxgraph.ibm" def _add_database_resources(styles, provider): if provider == "aws": - gn = "mxgraph.aws4" n = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#3334B9;strokeColor=none;dashed=0;" "verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;" @@ -195,7 +174,6 @@ def _add_database_resources(styles, provider): "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." ) - styles["aws_docdb_cluster"] = ( n2 + "resourceIcon;resIcon=" + gn + ".documentdb_with_mongodb_compatibility;" ) @@ -208,31 +186,24 @@ def _add_database_resources(styles, provider): styles["aws_db_instance"] = n + "rds_instance;" styles["aws_dax"] = n + "dynamodb_dax;" - else: - gn = "mxgraph.ibm" + def _add_ml_resources(styles, provider): if provider == "aws": - gn = "mxgraph.aws4" n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#4AB29A;gradientDirection=north;fillColor=#116D5B;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." ) - styles["aws_sagemaker"] = n2 + "resourceIcon;resIcon=" + gn + ".sagemaker;" - else: - gn = "mxgraph.ibm" def _add_management_governance_resources(styles, provider): if provider == "aws": - gn = "mxgraph.aws4" n2 = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=#F34482;gradientDirection=north;fillColor=#BC1356;" "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." ) - styles["aws_cloudwatch"] = n2 + "resourceIcon;resIcon=" + gn + ".cloudwatch_2;" styles["aws_autoscaling_group"] = ( n2 + "resourceIcon;resIcon=" + gn + ".autoscaling;" @@ -241,13 +212,9 @@ def _add_management_governance_resources(styles, provider): styles["aws_cloudformation"] = ( n2 + "resourceIcon;resIcon=" + gn + ".cloudformation;" ) - else: - gn = "mxgraph.ibm" - def _add_network_resources(styles, provider): if provider == "aws": - gn = "mxgraph.aws4" n = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#5A30B5;strokeColor=none;dashed=0;" "verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;" @@ -258,7 +225,6 @@ def _add_network_resources(styles, provider): "strokeColor=#ffffff;dashed=0;verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;" "fontSize=12;fontStyle=0;aspect=fixed;shape=mxgraph.aws4." ) - styles["aws_api_gateway_rest_api"] = ( n2 + "resourceIcon;resIcon=" + gn + ".api_gateway;" ) @@ -274,7 +240,6 @@ def _add_network_resources(styles, provider): styles["aws_global_accelerator"] = ( n2 + "resourceIcon;resIcon=" + gn + ".global_accelerator;" ) - styles["aws_route_table"] = n + "route_table;" styles["aws_vpc_endpoint_gateway"] = n + "gateway;" styles["aws_internet_gateway"] = n + "internet_gateway;" @@ -284,20 +249,26 @@ def _add_network_resources(styles, provider): styles["aws_vpn_connection"] = n + "vpn_connection;" styles["aws_vpn_gateway"] = n + "vpn_gateway;" else: - gn = "mxgraph.ibm" - n = ( + styles["ibm_route_table"] = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#5A30B5;strokeColor=none;dashed=0;" "verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;" - "pointerEvents=1;shape=mxgraph.aws4." + "pointerEvents=1;shape=mxgraph.aws4.route_table;" ) - styles["ibm_security_group"] = 'fontStyle=0;verticalAlign=top;align=center;spacingTop=-2;fillColor=none;rounded=0;whiteSpace=wrap;html=1;strokeColor=#FF0000;strokeWidth=2;dashed=1;container=1;collapsible=0;expand=0;recursiveResize=0;' - styles["ibm_network_acl"] = 'shape=mxgraph.ibm.box;prType=subnet;fontStyle=0;verticalAlign=top;align=left;spacingLeft=32;spacingTop=4;fillColor=#E6F0E2;rounded=0;whiteSpace=wrap;html=1;strokeColor=#00882B;strokeWidth=1;dashed=0;container=1;spacing=-4;collapsible=0;expand=0;recursiveResize=0;' - styles["ibm_route_table"] = n + "route_table;" + styles["ibm_security_group"] = 'fontStyle=0;verticalAlign=top;align=center;spacingTop=-2;fillColor=none;' + styles["ibm_security_group"] = styles["ibm_security_group"] + 'rounded=0;whiteSpace=wrap;html=1;' + styles["ibm_security_group"] = styles["ibm_security_group"] + 'strokeColor=#FF0000;strokeWidth=2;' + styles["ibm_security_group"] = styles["ibm_security_group"] + 'dashed=1;container=1;collapsible=0;' + styles["ibm_security_group"] = styles["ibm_security_group"] + 'expand=0;recursiveResize=0;' + styles["ibm_network_acl"] = 'shape=mxgraph.ibm.box;prType=subnet;fontStyle=0;verticalAlign=top;' + styles["ibm_network_acl"] = styles["ibm_network_acl"] + 'align=left;spacingLeft=32;spacingTop=4;' + styles["ibm_network_acl"] = styles["ibm_network_acl"] + 'fillColor=#E6F0E2;rounded=0;whiteSpace=wrap;' + styles["ibm_network_acl"] = styles["ibm_network_acl"] + 'html=1;strokeColor=#00882B;strokeWidth=1;dashed=0;' + styles["ibm_network_acl"] = styles["ibm_network_acl"] + 'container=1;spacing=-4;collapsible=0;expand=0;' + styles["ibm_network_acl"] = styles["ibm_network_acl"] + 'recursiveResize=0;' def _add_storage_resources(styles, provider): if provider == "aws": - gn = "mxgraph.aws4" n = ( "outlineConnect=0;fontColor=#232F3E;gradientColor=none;fillColor=#277116;strokeColor=none;dashed=0;" "verticalLabelPosition=bottom;verticalAlign=top;align=center;html=1;fontSize=12;fontStyle=0;aspect=fixed;" @@ -315,8 +286,6 @@ def _add_storage_resources(styles, provider): styles["aws_fsx"] = n2 + "resourceIcon;resIcon=" + gn + ".fsx;" styles["aws_s3"] = n + "bucket;" - else: - gn = "mxgraph.ibm" def build_styles(provider): From 743ed90e9147bc6e50ae95886821829309e26081 Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 12:20:20 +0530 Subject: [PATCH 06/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 7 +++---- cloudiscovery/shared/diagramsnet.py | 6 +++--- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index 4c5fdb8..0e6efc7 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -4,7 +4,6 @@ from typing import List, Dict from diagrams import Diagram, Cluster, Edge -from diagrams.ibm.network import Subnet, Rules, Firewall, Router from shared.common import Resource, ResourceEdge, ResourceDigest, message_handler from shared.diagramsnet import ( @@ -341,10 +340,10 @@ def draw_diagram(self, ordered_resources, relations): # Import all AWS nodes for module in Mapsources.diagrams_modules: - exec("from diagrams.aws." + module + " import *") + exec("from diagrams.aws." + module + " import *") #pylint: disable=exec-used for module in Mapsources.ibm_diagrams_modules: - exec("from diagrams.ibm." + module + " import *") + exec("from diagrams.ibm." + module + " import *") #pylint: disable=exec-used nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it @@ -644,7 +643,7 @@ def build_diagram( Mapsources.resource_styles[resource.digest.type] if resource.digest.type in Mapsources.resource_styles else Mapsources.resource_styles["ibm_general"] - ) + ) cell = CELL_TEMPLATE.format_map( { "CELL_IDX": resource.digest.to_string(), diff --git a/cloudiscovery/shared/diagramsnet.py b/cloudiscovery/shared/diagramsnet.py index ce5aa5c..c01b115 100644 --- a/cloudiscovery/shared/diagramsnet.py +++ b/cloudiscovery/shared/diagramsnet.py @@ -260,9 +260,9 @@ def _add_network_resources(styles, provider): styles["ibm_security_group"] = styles["ibm_security_group"] + 'dashed=1;container=1;collapsible=0;' styles["ibm_security_group"] = styles["ibm_security_group"] + 'expand=0;recursiveResize=0;' styles["ibm_network_acl"] = 'shape=mxgraph.ibm.box;prType=subnet;fontStyle=0;verticalAlign=top;' - styles["ibm_network_acl"] = styles["ibm_network_acl"] + 'align=left;spacingLeft=32;spacingTop=4;' - styles["ibm_network_acl"] = styles["ibm_network_acl"] + 'fillColor=#E6F0E2;rounded=0;whiteSpace=wrap;' - styles["ibm_network_acl"] = styles["ibm_network_acl"] + 'html=1;strokeColor=#00882B;strokeWidth=1;dashed=0;' + styles["ibm_network_acl"] = styles["ibm_network_acl"] + 'align=left;spacingLeft=32;spacingTop=4;' + styles["ibm_network_acl"] = styles["ibm_network_acl"] + 'fillColor=#E6F0E2;rounded=0;whiteSpace=wrap;' + styles["ibm_network_acl"] = styles["ibm_network_acl"] + 'html=1;strokeColor=#00882B;strokeWidth=1;dashed=0;' styles["ibm_network_acl"] = styles["ibm_network_acl"] + 'container=1;spacing=-4;collapsible=0;expand=0;' styles["ibm_network_acl"] = styles["ibm_network_acl"] + 'recursiveResize=0;' From 2f661a8110421b5d41dd259f9f8182cd0f3031d5 Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 12:28:26 +0530 Subject: [PATCH 07/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index 0e6efc7..01168f6 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -340,10 +340,12 @@ def draw_diagram(self, ordered_resources, relations): # Import all AWS nodes for module in Mapsources.diagrams_modules: - exec("from diagrams.aws." + module + " import *") #pylint: disable=exec-used + # pylint: disable=exec-used + exec("from diagrams.aws." + module + " import *") # pylint: disable=exec-used for module in Mapsources.ibm_diagrams_modules: - exec("from diagrams.ibm." + module + " import *") #pylint: disable=exec-used + # pylint: disable=exec-used + exec("from diagrams.ibm." + module + " import *") # pylint: disable=exec-used nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it From eeb52f0e0e3f05d5e08f1bdaf9775e48d6819395 Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 12:39:20 +0530 Subject: [PATCH 08/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index 01168f6..87d84d2 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -335,6 +335,7 @@ def generate_diagram( message_handler("\n\nPNG diagram generated", "HEADER") message_handler("Check your diagram: " + output_filename + ".png", "OKBLUE") + # pylint: disable=exec-used def draw_diagram(self, ordered_resources, relations): already_drawn_elements = {} From 40618f6814fc9f5821fed0767050e36a034a68c0 Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 15:19:12 +0530 Subject: [PATCH 09/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index 87d84d2..d80e791 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -341,12 +341,12 @@ def draw_diagram(self, ordered_resources, relations): # Import all AWS nodes for module in Mapsources.diagrams_modules: - # pylint: disable=exec-used - exec("from diagrams.aws." + module + " import *") # pylint: disable=exec-used + # pylint: disable=W0122 + exec("from diagrams.aws." + module + " import *") # pylint: disable=W0122 for module in Mapsources.ibm_diagrams_modules: - # pylint: disable=exec-used - exec("from diagrams.ibm." + module + " import *") # pylint: disable=exec-used + # pylint: disable=W0122 + exec("from diagrams.ibm." + module + " import *") # pylint: disable=W0122 nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it From f30d87ef6128ba321f8e747632bb1ccf6cab9b3f Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 15:24:54 +0530 Subject: [PATCH 10/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index d80e791..87d84d2 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -341,12 +341,12 @@ def draw_diagram(self, ordered_resources, relations): # Import all AWS nodes for module in Mapsources.diagrams_modules: - # pylint: disable=W0122 - exec("from diagrams.aws." + module + " import *") # pylint: disable=W0122 + # pylint: disable=exec-used + exec("from diagrams.aws." + module + " import *") # pylint: disable=exec-used for module in Mapsources.ibm_diagrams_modules: - # pylint: disable=W0122 - exec("from diagrams.ibm." + module + " import *") # pylint: disable=W0122 + # pylint: disable=exec-used + exec("from diagrams.ibm." + module + " import *") # pylint: disable=exec-used nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it From f267751906e305b685d834aeee27d0a9f8a6286e Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 15:56:55 +0530 Subject: [PATCH 11/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index 87d84d2..02f54b9 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -342,11 +342,11 @@ def draw_diagram(self, ordered_resources, relations): # Import all AWS nodes for module in Mapsources.diagrams_modules: # pylint: disable=exec-used - exec("from diagrams.aws." + module + " import *") # pylint: disable=exec-used + exec("from diagrams.aws." + module + " import *") for module in Mapsources.ibm_diagrams_modules: # pylint: disable=exec-used - exec("from diagrams.ibm." + module + " import *") # pylint: disable=exec-used + exec("from diagrams.ibm." + module + " import *") nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it From f8ec574e3fba335c0bdd69297b1b9cdbe1ed7782 Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 16:09:03 +0530 Subject: [PATCH 12/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index 02f54b9..09b64d0 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -345,6 +345,7 @@ def draw_diagram(self, ordered_resources, relations): exec("from diagrams.aws." + module + " import *") for module in Mapsources.ibm_diagrams_modules: + # pylint: disable=exec-used # pylint: disable=exec-used exec("from diagrams.ibm." + module + " import *") From 2128bbd118723035cd210c5cd12fc3cc79469212 Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 16:12:59 +0530 Subject: [PATCH 13/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index 09b64d0..695245c 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -339,16 +339,16 @@ def generate_diagram( def draw_diagram(self, ordered_resources, relations): already_drawn_elements = {} + # Import all IBM nodes + for module in Mapsources.ibm_diagrams_modules: + # pylint: disable=exec-used + exec("from diagrams.ibm." + module + " import *") + # Import all AWS nodes for module in Mapsources.diagrams_modules: # pylint: disable=exec-used exec("from diagrams.aws." + module + " import *") - for module in Mapsources.ibm_diagrams_modules: - # pylint: disable=exec-used - # pylint: disable=exec-used - exec("from diagrams.ibm." + module + " import *") - nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it for group_name in ordered_resources: From 407536ae32183489a68f2377eca7e5d3aa32543e Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 16:18:47 +0530 Subject: [PATCH 14/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index 695245c..e5f3d90 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -338,17 +338,14 @@ def generate_diagram( # pylint: disable=exec-used def draw_diagram(self, ordered_resources, relations): already_drawn_elements = {} - - # Import all IBM nodes - for module in Mapsources.ibm_diagrams_modules: - # pylint: disable=exec-used - exec("from diagrams.ibm." + module + " import *") - # Import all AWS nodes for module in Mapsources.diagrams_modules: # pylint: disable=exec-used exec("from diagrams.aws." + module + " import *") - + # Import all IBM nodes + for module in Mapsources.ibm_diagrams_modules: + # pylint: disable=exec-used + exec("from diagrams.ibm." + module + " import *") nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it for group_name in ordered_resources: From 4a15076850d80f586a9994362999b70c94157124 Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 16:23:40 +0530 Subject: [PATCH 15/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index e5f3d90..1d82b9a 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -342,10 +342,12 @@ def draw_diagram(self, ordered_resources, relations): for module in Mapsources.diagrams_modules: # pylint: disable=exec-used exec("from diagrams.aws." + module + " import *") + """ # Import all IBM nodes for module in Mapsources.ibm_diagrams_modules: # pylint: disable=exec-used exec("from diagrams.ibm." + module + " import *") + """ nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it for group_name in ordered_resources: From 35ba8c40f95d0cebf55fd62c7d116bd678b63917 Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 16:28:02 +0530 Subject: [PATCH 16/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index 1d82b9a..a008a48 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -347,7 +347,7 @@ def draw_diagram(self, ordered_resources, relations): for module in Mapsources.ibm_diagrams_modules: # pylint: disable=exec-used exec("from diagrams.ibm." + module + " import *") - """ + """ nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it for group_name in ordered_resources: From 10dcf4699b814465791ca9c02d3f5579d0008b81 Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 16:48:59 +0530 Subject: [PATCH 17/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index a008a48..e5f3d90 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -342,12 +342,10 @@ def draw_diagram(self, ordered_resources, relations): for module in Mapsources.diagrams_modules: # pylint: disable=exec-used exec("from diagrams.aws." + module + " import *") - """ # Import all IBM nodes for module in Mapsources.ibm_diagrams_modules: # pylint: disable=exec-used exec("from diagrams.ibm." + module + " import *") - """ nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it for group_name in ordered_resources: From b77a21872c6f02ba48f1b25a05d9dffc5d9e3813 Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 16:52:07 +0530 Subject: [PATCH 18/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index e5f3d90..5f3266c 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -344,8 +344,7 @@ def draw_diagram(self, ordered_resources, relations): exec("from diagrams.aws." + module + " import *") # Import all IBM nodes for module in Mapsources.ibm_diagrams_modules: - # pylint: disable=exec-used - exec("from diagrams.ibm." + module + " import *") + exec("from diagrams.ibm." + module + " import *") # pylint: disable=exec-used nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it for group_name in ordered_resources: From 015a5312eae186e7fd0d52de88588db992b3d7e8 Mon Sep 17 00:00:00 2001 From: Malar Kandasamy Date: Mon, 7 Jun 2021 17:02:10 +0530 Subject: [PATCH 19/19] codacy errors corrected --- cloudiscovery/shared/diagram.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/cloudiscovery/shared/diagram.py b/cloudiscovery/shared/diagram.py index 5f3266c..b5a5300 100644 --- a/cloudiscovery/shared/diagram.py +++ b/cloudiscovery/shared/diagram.py @@ -23,7 +23,8 @@ class Mapsources: # diagrams modules that store classes that represent diagram elements provider = "" - diagrams_modules = [ + modules_dict = {} + modules_dict["aws"] = [ "analytics", "ar", "blockchain", @@ -52,7 +53,7 @@ class Mapsources: "storage", ] - ibm_diagrams_modules = [ + modules_dict['ibm'] = [ "analytics", "applications", "blockchain", @@ -338,13 +339,11 @@ def generate_diagram( # pylint: disable=exec-used def draw_diagram(self, ordered_resources, relations): already_drawn_elements = {} - # Import all AWS nodes - for module in Mapsources.diagrams_modules: - # pylint: disable=exec-used - exec("from diagrams.aws." + module + " import *") - # Import all IBM nodes - for module in Mapsources.ibm_diagrams_modules: - exec("from diagrams.ibm." + module + " import *") # pylint: disable=exec-used + # Import all AWS, IBM nodes + for key, val in Mapsources.modules_dict: + for module in val: + # pylint: disable=exec-used + exec("from diagrams." + key + "." + module + " import *") nodes: Dict[ResourceDigest, any] = {} # Iterate resources to draw it for group_name in ordered_resources: