feat(script): new script to fill positive_expected_result and fixing minibug on similarityID generation in scan commands

.github/scripts/generate-positive-expected-results/models.py

@@ -0,0 +1,93 @@
+import os
+import re
+from dataclasses import dataclass
+
+
+FIELD_ORDER = [
+    "queryName", "severity", "line", "fileName",
+    "resourceType", "resourceName", "searchKey", "searchValue",
+    "expectedValue", "actualValue", "issueType", "similarityID", "search_line",
+]
+
+KICS_RESULT_CODES = {0, 1, 20, 30, 40, 50, 60}
+
+
+def natural_sort_key(s: str):
+    """'positive2.tf' → ['positive', 2, '.tf'] so numeric parts sort numerically."""
+    return [int(c) if c.isdigit() else c for c in re.split(r'(\d+)', s)]
+
+
+@dataclass
+class PositiveTest:
+    """A positive test file to scan."""
+
+    label: str
+    scan_path: str
+    group: str  # "test" for loose files, "test/<dir>" for subdirectory files
+
+
+@dataclass
+class ScanFailure:
+    """A scan that failed with an unexpected return code."""
+
+    scan_path: str
+    payload_path: str
+    return_code: int
+
+
+@dataclass
+class ExpectedResultEntry:
+    """A single expected vulnerability finding."""
+
+    queryName: str = ""
+    severity: str = ""
+    line: int = 0
+    fileName: str = ""
+    resourceType: str = ""
+    resourceName: str = ""
+    searchKey: str = ""
+    searchValue: str = ""
+    expectedValue: str = ""
+    actualValue: str = ""
+    issueType: str = ""
+    similarityID: str = ""
+    search_line: int = -1
+
+    @classmethod
+    def from_kics_result(cls, query_name: str, severity: str, file_entry: dict) -> "ExpectedResultEntry":
+        """Build an entry from a KICS scan result file_entry."""
+        return cls(
+            queryName=query_name,
+            severity=severity,
+            line=file_entry.get("line", 0),
+            fileName=os.path.basename(file_entry.get("file_name", "")),
+            resourceType=file_entry.get("resource_type", ""),
+            resourceName=file_entry.get("resource_name", ""),
+            searchKey=file_entry.get("search_key", ""),
+            searchValue=file_entry.get("search_value", ""),
+            expectedValue=file_entry.get("expected_value", ""),
+            actualValue=file_entry.get("actual_value", ""),
+            issueType=file_entry.get("issue_type", ""),
+            similarityID=file_entry.get("similarity_id", ""),
+            search_line=file_entry.get("search_line", -1),
+        )
+
+    def to_ordered_dict(self) -> dict:
+        """Return a dict with keys in FIELD_ORDER."""
+        return {k: getattr(self, k) for k in FIELD_ORDER}
+
+    def sort_key(self) -> tuple:
+        return (
+            natural_sort_key(self.fileName),
+            self.line,
+            self.searchKey,
+            self.searchValue,
+            self.resourceType,
+            self.resourceName,
+            self.queryName,
+            self.expectedValue,
+            self.actualValue,
+            self.issueType,
+            self.similarityID,
+            self.search_line,
+        )

assets/queries/ansible/aws/alb_listening_on_http/test/positive_expected_result.json

@@ -2,11 +2,31 @@
   {
     "queryName": "ALB Listening on HTTP",
     "severity": "MEDIUM",
-    "line": 11
+    "line": 11,
+    "fileName": "positive.yaml",
+    "resourceType": "community.aws.elb_application_lb",
+    "resourceName": "my_elb_application",
+    "searchKey": "name={{my_elb_application}}.{{community.aws.elb_application_lb}}.listeners.Protocol=HTTP",
+    "searchValue": "",
+    "expectedValue": "'aws_elb_application_lb' Protocol should be 'HTTP'",
+    "actualValue": "'aws_elb_application_lb' Protocol it's not 'HTTP'",
+    "issueType": "IncorrectValue",
+    "similarityID": "607f2f1c612a246aadce48454d34acfb08c461d2bcf9955e6c83e4c1c623a394",
+    "search_line": -1
   },
   {
     "queryName": "ALB Listening on HTTP",
     "severity": "MEDIUM",
-    "line": 29
+    "line": 29,
+    "fileName": "positive.yaml",
+    "resourceType": "community.aws.elb_application_lb",
+    "resourceName": "my_elb_application2",
+    "searchKey": "name={{my_elb_application2}}.{{community.aws.elb_application_lb}}.listeners",
+    "searchValue": "",
+    "expectedValue": "'aws_elb_application_lb' Protocol should be 'HTTP'",
+    "actualValue": "'aws_elb_application_lb' Protocol is missing",
+    "issueType": "MissingAttribute",
+    "similarityID": "b8bad394a7838d266d3b23cbd20fffa0b9695e9b9ce815983d45b422a420d195",
+    "search_line": -1
   }
-]
+]

assets/queries/ansible/aws/ami_not_encrypted/test/positive_expected_result.json

@@ -2,11 +2,31 @@
   {
     "queryName": "AMI Not Encrypted",
     "severity": "MEDIUM",
-    "line": 6
+    "line": 6,
+    "fileName": "positive.yaml",
+    "resourceType": "amazon.aws.ec2_ami",
+    "resourceName": "Basic AMI Creation",
+    "searchKey": "name={{Basic AMI Creation}}.{{amazon.aws.ec2_ami}}.device_mapping.encrypted",
+    "searchValue": "",
+    "expectedValue": "ec2_ami.device_mapping.encrypted should be set to true",
+    "actualValue": "ec2_ami.device_mapping.encrypted is set to false",
+    "issueType": "IncorrectValue",
+    "similarityID": "1019507ea6b5dc1e86ed66df617c66e37d9820bf3f232b628ff0202ed08f649e",
+    "search_line": -1
   },
   {
     "queryName": "AMI Not Encrypted",
     "severity": "MEDIUM",
-    "line": 13
+    "line": 13,
+    "fileName": "positive.yaml",
+    "resourceType": "amazon.aws.ec2_ami",
+    "resourceName": "Basic AMI Creation2",
+    "searchKey": "name={{Basic AMI Creation2}}.{{amazon.aws.ec2_ami}}",
+    "searchValue": "",
+    "expectedValue": "ec2_ami.device_mapping.device_name.encrypted should be set to true",
+    "actualValue": "ec2_ami.device_mapping.device_name.encrypted is undefined",
+    "issueType": "MissingAttribute",
+    "similarityID": "9512ff434c3adcfecc7caf5b9717c1e1896b41010d2f428ce4ef7bb62e49929c",
+    "search_line": -1
   }
-]
+]

assets/queries/ansible/aws/ami_shared_with_multiple_accounts/test/positive_expected_result.json

@@ -2,11 +2,31 @@
   {
     "queryName": "AMI Shared With Multiple Accounts",
     "severity": "MEDIUM",
-    "line": 5
+    "line": 5,
+    "fileName": "positive.yaml",
+    "resourceType": "amazon.aws.ec2_ami",
+    "resourceName": "Update AMI Launch Permissions, making it public",
+    "searchKey": "name={{Update AMI Launch Permissions, making it public}}.{{amazon.aws.ec2_ami}}.launch_permissions",
+    "searchValue": "",
+    "expectedValue": "ec2_ami.launch_permissions just allows one user to launch the AMI",
+    "actualValue": "ec2_ami.launch_permissions allows more than one user to launch the AMI",
+    "issueType": "IncorrectValue",
+    "similarityID": "c9fb9becbae73675895b861b79d0a5ab275dd56551b224e5502c6e1b53b0f039",
+    "search_line": -1
   },
   {
     "queryName": "AMI Shared With Multiple Accounts",
     "severity": "MEDIUM",
-    "line": 11
+    "line": 11,
+    "fileName": "positive.yaml",
+    "resourceType": "amazon.aws.ec2_ami",
+    "resourceName": "Allow AMI to be launched by another account",
+    "searchKey": "name={{Allow AMI to be launched by another account}}.{{amazon.aws.ec2_ami}}.launch_permissions",
+    "searchValue": "",
+    "expectedValue": "ec2_ami.launch_permissions just allows one user to launch the AMI",
+    "actualValue": "ec2_ami.launch_permissions allows more than one user to launch the AMI",
+    "issueType": "IncorrectValue",
+    "similarityID": "d9d6be03b8285c258824251592a50a5025987ae73334efa87c313b7dfa49c547",
+    "search_line": -1
   }
 ]

assets/queries/ansible/aws/api_gateway_endpoint_config_is_not_private/test/positive_expected_result.json

@@ -2,6 +2,16 @@
   {
     "queryName": "API Gateway Endpoint Config is Not Private",
     "severity": "MEDIUM",
-    "line": 8
+    "line": 8,
+    "fileName": "positive.yaml",
+    "resourceType": "community.aws.aws_api_gateway",
+    "resourceName": "Setup AWS API Gateway setup on AWS and deploy API definition",
+    "searchKey": "name={{Setup AWS API Gateway setup on AWS and deploy API definition}}.{{community.aws.aws_api_gateway}}.endpoint_type",
+    "searchValue": "",
+    "expectedValue": "'aws_api_gateway.endpoint_type' should be set to 'PRIVATE'",
+    "actualValue": "'aws_api_gateway.endpoint_type' is not 'PRIVATE'",
+    "issueType": "IncorrectValue",
+    "similarityID": "afbad0b8491f6423e427a5b02ade6b0dda8368c355eb2377d9b774291cf0b4e7",
+    "search_line": -1
   }
 ]

assets/queries/ansible/aws/api_gateway_with_cloudwatch_logging_disabled/test/positive_expected_result.json

@@ -2,6 +2,16 @@
   {
     "queryName": "API Gateway With CloudWatch Logging Disabled",
     "severity": "MEDIUM",
-    "line": 3
+    "line": 3,
+    "fileName": "positive.yaml",
+    "resourceType": "community.aws.cloudwatchlogs_log_group",
+    "resourceName": "Setup AWS API Gateway setup on AWS cloudwatchlogs",
+    "searchKey": "name={{Setup AWS API Gateway setup on AWS cloudwatchlogs}}.{{community.aws.cloudwatchlogs_log_group}}",
+    "searchValue": "",
+    "expectedValue": "cloudwatchlogs_log_grouptracing_enabled should contain log_group_name",
+    "actualValue": "cloudwatchlogs_log_group does not contain log_group_name defined",
+    "issueType": "MissingAttribute",
+    "similarityID": "d9c365cf6b6e094fce902d9ef3572fc723f092e8ca343c0d36fe5a41c7bb63c6",
+    "search_line": -1
   }
 ]

assets/queries/ansible/aws/api_gateway_without_configured_authorizer/test/positive_expected_result.json

@@ -3,24 +3,60 @@
     "queryName": "API Gateway Without Configured Authorizer",
     "severity": "MEDIUM",
     "line": 3,
-    "fileName": "positive1.yaml"
+    "fileName": "positive1.yaml",
+    "resourceType": "community.aws.aws_api_gateway",
+    "resourceName": "Setup AWS API Gateway setup on AWS and deploy API definition",
+    "searchKey": "name={{Setup AWS API Gateway setup on AWS and deploy API definition}}.{{community.aws.aws_api_gateway}}.swagger_dict",
+    "searchValue": "",
+    "expectedValue": "'community.aws.aws_api_gateway.swagger_dict' should have an authorizer set",
+    "actualValue": "'community.aws.aws_api_gateway.swagger_dict' does not have a authorizer set",
+    "issueType": "IncorrectValue",
+    "similarityID": "fe3e377c6f66b5ec4b204a63bd1d92d5bd0940f7709d64d1c8020c8a22d12b3c",
+    "search_line": -1
   },
   {
     "queryName": "API Gateway Without Configured Authorizer",
     "severity": "MEDIUM",
     "line": 2,
-    "fileName": "positive2.yaml"
+    "fileName": "positive2.yaml",
+    "resourceType": "aws_api_gateway",
+    "resourceName": "Setup AWS API Gateway setup on AWS and deploy API definition2",
+    "searchKey": "name={{Setup AWS API Gateway setup on AWS and deploy API definition2}}.{{aws_api_gateway}}",
+    "searchValue": "",
+    "expectedValue": "'aws_api_gateway' should have swagger_file, swagger_text or swagger_dict set",
+    "actualValue": "'aws_api_gateway' does not have swagger_file, swagger_text or swagger_dict set",
+    "issueType": "IncorrectValue",
+    "similarityID": "ddfb7af6321147fc9725e8e859bb0e19cf96060b80fc3977302df93344a243ec",
+    "search_line": -1
   },
   {
     "queryName": "API Gateway Without Configured Authorizer",
     "severity": "MEDIUM",
     "line": 3,
-    "fileName": "positive3.yaml"
+    "fileName": "positive3.yaml",
+    "resourceType": "aws_api_gateway",
+    "resourceName": "Setup AWS API Gateway setup on AWS and deploy API 222",
+    "searchKey": "name={{Setup AWS API Gateway setup on AWS and deploy API 222}}.{{aws_api_gateway}}.swagger_file",
+    "searchValue": "",
+    "expectedValue": "'aws_api_gateway.swagger_file' should have an authorizer set",
+    "actualValue": "'aws_api_gateway.swagger_file' does not have a authorizer set",
+    "issueType": "IncorrectValue",
+    "similarityID": "54d917d994910ab583e416d8ed3ffbfbc5c3b76e4e389d4ec001c942a5895542",
+    "search_line": 0
   },
   {
     "queryName": "API Gateway Without Configured Authorizer",
     "severity": "MEDIUM",
     "line": 3,
-    "fileName": "positive4.yaml"
+    "fileName": "positive4.yaml",
+    "resourceType": "aws_api_gateway",
+    "resourceName": "Setup AWS API Gateway setup on AWS and deploy API 222",
+    "searchKey": "name={{Setup AWS API Gateway setup on AWS and deploy API 222}}.{{aws_api_gateway}}.swagger_text",
+    "searchValue": "",
+    "expectedValue": "'aws_api_gateway.swagger_text' should have an authorizer set",
+    "actualValue": "'aws_api_gateway.swagger_text' does not have a authorizer set",
+    "issueType": "IncorrectValue",
+    "similarityID": "7eeb99b352c87ebbcec5de88dfe2e950b218a1f367b6215a0c9f7f7b5297bc54",
+    "search_line": -1
   }
 ]

assets/queries/ansible/aws/api_gateway_without_ssl_certificate/test/positive_expected_result.json

@@ -1,25 +1,62 @@
 [
-	{
-		"queryName": "API Gateway Without SSL Certificate",
-		"severity": "MEDIUM",
-		"line": 6
-	},
-
-	{
-		"queryName": "API Gateway Without SSL Certificate",
-		"severity": "MEDIUM",
-		"line": 8
-	},
-
-	{
-		"queryName": "API Gateway Without SSL Certificate",
-		"severity": "MEDIUM",
-		"line": 21
-	},
-
-	{
-		"queryName": "API Gateway Without SSL Certificate",
-		"severity": "MEDIUM",
-		"line": 23
-	}
+  {
+    "queryName": "API Gateway Without SSL Certificate",
+    "severity": "MEDIUM",
+    "line": 6,
+    "fileName": "positive.yaml",
+    "resourceType": "aws_api_gateway",
+    "resourceName": "update API",
+    "searchKey": "name={{update API}}.{{aws_api_gateway}}.validate_certs",
+    "searchValue": "",
+    "expectedValue": "aws_api_gateway.validate_certs should be set to yes",
+    "actualValue": "aws_api_gateway.validate_certs is not set to yes",
+    "issueType": "IncorrectValue",
+    "similarityID": "5c46a089b29a823bedcbf51c4e833b03ae397256d5ad889cf25e60064dd20f39",
+    "search_line": -1
+  },
+  {
+    "queryName": "API Gateway Without SSL Certificate",
+    "severity": "MEDIUM",
+    "line": 8,
+    "fileName": "positive.yaml",
+    "resourceType": "aws_api_gateway",
+    "resourceName": "update API v1",
+    "searchKey": "name={{update API v1}}.{{aws_api_gateway}}",
+    "searchValue": "",
+    "expectedValue": "aws_api_gateway.validate_certs should be set",
+    "actualValue": "aws_api_gateway.validate_certs is undefined",
+    "issueType": "MissingAttribute",
+    "similarityID": "cbf1cde1a0cc1fe648d3c6d3d6e55ccb4d76e087abdceb87744f7737ac8a55f2",
+    "search_line": -1
+  },
+  {
+    "queryName": "API Gateway Without SSL Certificate",
+    "severity": "MEDIUM",
+    "line": 21,
+    "fileName": "positive.yaml",
+    "resourceType": "community.aws.aws_api_gateway",
+    "resourceName": "Setup AWS API Gateway setup on AWS and deploy API definition",
+    "searchKey": "name={{Setup AWS API Gateway setup on AWS and deploy API definition}}.{{community.aws.aws_api_gateway}}.validate_certs",
+    "searchValue": "",
+    "expectedValue": "aws_api_gateway.validate_certs should be set to yes",
+    "actualValue": "aws_api_gateway.validate_certs is not set to yes",
+    "issueType": "IncorrectValue",
+    "similarityID": "2e77f7861dc2f396095e45ac172a705a566d59798cbdc5f752be8f1df09aa8f3",
+    "search_line": -1
+  },
+  {
+    "queryName": "API Gateway Without SSL Certificate",
+    "severity": "MEDIUM",
+    "line": 23,
+    "fileName": "positive.yaml",
+    "resourceType": "community.aws.aws_api_gateway",
+    "resourceName": "Setup AWS API Gateway setup on AWS and deploy API definition v1",
+    "searchKey": "name={{Setup AWS API Gateway setup on AWS and deploy API definition v1}}.{{community.aws.aws_api_gateway}}",
+    "searchValue": "",
+    "expectedValue": "aws_api_gateway.validate_certs should be set",
+    "actualValue": "aws_api_gateway.validate_certs is undefined",
+    "issueType": "MissingAttribute",
+    "similarityID": "94d19a8449a34828a18c4ae4488edf623f806c7b1bcfa67b27ec4651c1a14fd6",
+    "search_line": -1
+  }
 ]