Add and Update SQS Versions of PE Report Scans

backend/Dockerfile.pe

@@ -28,7 +28,7 @@ RUN ./aws/install
 # Sync the latest from cf-staging branch
 RUN git clone -b crossfeed-SQS https://github.com/cisagov/ATC-Framework.git && \
     cd ATC-Framework && \
-    git checkout 563f59e8b67ca153ab5564d697433ca0d8db451a && \
+    git checkout 4dc934cb07d50111a80459206f7bf27389e08e9e && \
     pip install .
 
 RUN python -m spacy download en_core_web_lg

backend/serverless.yml

@@ -181,49 +181,66 @@ resources:
             - ${self:provider.stage}
             - integration
   Resources:
-    ShodanQueue:
+    ASMSyncQueue:
+      Type: AWS::SQS::Queue
+      Properties:
+        QueueName: ${self:provider.stage}-asmSync-queue
+        VisibilityTimeout: 18000  # 5 hours
+        MaximumMessageSize: 262144  # 256 KB
+        MessageRetentionPeriod: 604800  # 7 days
+    CybersixgillAlertsQueue:
       Type: AWS::SQS::Queue
       Condition: IsDMZ
       Properties:
-        QueueName: ${self:provider.stage}-shodan-queue
+        QueueName: ${self:provider.stage}-cybersixgill-alerts-queue
         VisibilityTimeout: 18000  # 5 hours
         MaximumMessageSize: 262144  # 256 KB
         MessageRetentionPeriod: 604800  # 7 days
-    DnstwistQueue:
+    CybersixgillCredentialsQueue:
       Type: AWS::SQS::Queue
       Condition: IsDMZ
       Properties:
-        QueueName: ${self:provider.stage}-dnstwist-queue
+        QueueName: ${self:provider.stage}-cybersixgill-credentials-queue
         VisibilityTimeout: 18000  # 5 hours
         MaximumMessageSize: 262144  # 256 KB
         MessageRetentionPeriod: 604800  # 7 days
-    IntelxQueue:
+    CybersixgillMentionsQueue:
       Type: AWS::SQS::Queue
       Condition: IsDMZ
       Properties:
-        QueueName: ${self:provider.stage}-intelx-queue
+        QueueName: ${self:provider.stage}-cybersixgill-mentions-queue
         VisibilityTimeout: 18000  # 5 hours
         MaximumMessageSize: 262144  # 256 KB
         MessageRetentionPeriod: 604800  # 7 days
-    CybersixgillQueue:
+    CybersixgillTopcvesQueue:
       Type: AWS::SQS::Queue
       Condition: IsDMZ
       Properties:
-        QueueName: ${self:provider.stage}-cybersixgill-queue
+        QueueName: ${self:provider.stage}-cybersixgill-topcves-queue
         VisibilityTimeout: 18000  # 5 hours
         MaximumMessageSize: 262144  # 256 KB
         MessageRetentionPeriod: 604800  # 7 days
-    XpanseQueue:
+    DnsmonitorQueue:
       Type: AWS::SQS::Queue
+      Condition: IsDMZ
       Properties:
-        QueueName: ${self:provider.stage}-xpanse-queue
+        QueueName: ${self:provider.stage}-dnsmonitor-queue
         VisibilityTimeout: 18000  # 5 hours
         MaximumMessageSize: 262144  # 256 KB
         MessageRetentionPeriod: 604800  # 7 days
-    ASMSyncQueue:
+    DnstwistQueue:
       Type: AWS::SQS::Queue
+      Condition: IsDMZ
       Properties:
-        QueueName: ${self:provider.stage}-asmSync-queue
+        QueueName: ${self:provider.stage}-dnstwist-queue
+        VisibilityTimeout: 18000  # 5 hours
+        MaximumMessageSize: 262144  # 256 KB
+        MessageRetentionPeriod: 604800  # 7 days
+    IntelxQueue:
+      Type: AWS::SQS::Queue
+      Condition: IsDMZ
+      Properties:
+        QueueName: ${self:provider.stage}-intelx-queue
         VisibilityTimeout: 18000  # 5 hours
         MaximumMessageSize: 262144  # 256 KB
         MessageRetentionPeriod: 604800  # 7 days
@@ -234,6 +251,21 @@ resources:
         VisibilityTimeout: 18000  # 5 hours
         MaximumMessageSize: 262144  # 256 KB
         MessageRetentionPeriod: 604800  # 7 days
+    ShodanQueue:
+      Type: AWS::SQS::Queue
+      Condition: IsDMZ
+      Properties:
+        QueueName: ${self:provider.stage}-shodan-queue
+        VisibilityTimeout: 18000  # 5 hours
+        MaximumMessageSize: 262144  # 256 KB
+        MessageRetentionPeriod: 604800  # 7 days
+    XpanseQueue:
+      Type: AWS::SQS::Queue
+      Properties:
+        QueueName: ${self:provider.stage}-xpanse-queue
+        VisibilityTimeout: 18000  # 5 hours
+        MaximumMessageSize: 262144  # 256 KB
+        MessageRetentionPeriod: 604800  # 7 days
 
 functions:
   - ${file(./src/tasks/functions.yml)}

backend/src/xfd_django/xfd_api/tasks/scanExecution.py

@@ -20,13 +20,17 @@
 # Initialize AWS clients
 
 SCAN_LIST = [
+    "asmSync",
+    "cybersixgill-alerts",
+    "cybersixgill-credentials",
+    "cybersixgill-mentions",
+    "cybersixgill-topcves",
+    "dnsmonitor",
     "dnstwist",
     "intelx",
-    "cybersixgill",
+    "qualys",
     "shodan",
     "xpanse",
-    "asmSync",
-    "qualys",
 ]
 QUEUE_URL = os.getenv("QUEUE_URL")
 
@@ -51,7 +55,7 @@ def start_desired_tasks(scan_type, desired_count, shodan_api_keys=None):
     shodan_api_keys = shodan_api_keys or []
     queue_url = "{}{}-queue".format(QUEUE_URL, scan_type)
 
-    batch_size = 1 if scan_type == "shodan" else 10
+    batch_size = 1 if scan_type in ["shodan", "asmSync"] else 10
     remaining_count = desired_count
 
     while remaining_count > 0:
@@ -151,7 +155,7 @@ def handler(event, context):
             print("scanType must be provided.")
             return {"statusCode": 400, "body": "Failed: no scanType provided."}
 
-        if scan_type == "shodan":
+        if scan_type in ["shodan", "asmSync"]:
             api_key_list = event.get("apiKeyList", "")
             shodan_api_keys = (
                 [key.strip() for key in api_key_list.split(",")] if api_key_list else []

backend/worker/pe-worker-entry.sh

@@ -52,21 +52,29 @@ while true; do
     ORG=$(echo "$MESSAGE" | jq -r '.Messages[0].Body | fromjson | .org')
   fi
 
-  if [[ "$SERVICE_TYPE" = *"shodan"* ]]; then
-    COMMAND="pe-source shodan --soc_med_included --org=$ORG"
+  # Run command for the specified script
+  if [[ "$SERVICE_TYPE" = *"asmSync"* ]]; then
+    COMMAND="pe-asm-sync asm-sqs --orgs=$ORG"
+  elif [[ "$SERVICE_TYPE" = *"cybersixgill-alerts"* ]]; then
+    COMMAND="pe-source cybersixgill --cybersix-methods=alerts --soc_med_included --orgs=$ORG"
+  elif [[ "$SERVICE_TYPE" = *"cybersixgill-credentials"* ]]; then
+    COMMAND="pe-source cybersixgill --cybersix-methods=credentials --soc_med_included --orgs=$ORG"
+  elif [[ "$SERVICE_TYPE" = *"cybersixgill-mentions"* ]]; then
+    COMMAND="pe-source cybersixgill --cybersix-methods=mentions --soc_med_included --orgs=$ORG"
+  elif [[ "$SERVICE_TYPE" = *"cybersixgill-topcves"* ]]; then
+    COMMAND="pe-source cybersixgill --cybersix-methods=topCVEs --soc_med_included"
+  elif [[ "$SERVICE_TYPE" = *"dnsmonitor"* ]]; then
+    COMMAND="pe-source dnsmonitor --orgs=$ORG"
   elif [[ "$SERVICE_TYPE" = *"dnstwist"* ]]; then
-    COMMAND="pe-source dnstwist --org=$ORG"
+    COMMAND="pe-source dnstwist --orgs=$ORG"
   elif [[ "$SERVICE_TYPE" = *"intelx"* ]]; then
-    COMMAND="pe-source intelx --org=$ORG --soc_med_included"
-  elif [[ "$SERVICE_TYPE" = *"cybersixgill"* ]]; then
-    COMMAND="pe-source cybersixgill --org=$ORG --soc_med_included"
-  elif [[ "$SERVICE_TYPE" = *"xpanse"* ]]; then
-    COMMAND="pe-source xpanse --org='$ORG'"
-  elif [[ "$SERVICE_TYPE" = *"asmSync"* ]]; then
-    COMMAND="pe-asm-sync asm-sqs --org='$ORG'"
+    COMMAND="pe-source intelx --orgs=$ORG"
   elif [[ "$SERVICE_TYPE" = *"qualys"* ]]; then
     COMMAND="pe-source was-report-pull --org='$ORG' && pe-source was-findings-sync --org='$ORG'"
-
+  elif [[ "$SERVICE_TYPE" = *"shodan"* ]]; then
+    COMMAND="pe-source shodan --soc_med_included --orgs=$ORG"
+  elif [[ "$SERVICE_TYPE" = *"xpanse"* ]]; then
+    COMMAND="pe-source xpanse --org='$ORG'"
   else
     echo "Unsupported SERVICE_TYPE: $SERVICE_TYPE"
     break