[sophos] Generate processor tags and normalize error handler

packages/sophos/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.15.4"
+  changes:
+    - description: Generate processor tags and normalize error handler.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15714
 - version: "3.15.3"
   changes:
     - description: Fix add_locale time zone handling

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/default.yml

@@ -36,10 +36,12 @@ processors:
       tag: grok_syslog_header
       on_failure:
         - append:
+            tag: append_error_message_0e0667da
             field: error.message
             value: |-
               Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
         - set:
+            tag: set_event_kind_795f04d8
             field: event.kind
             value: pipeline_error
   - lowercase:
@@ -50,6 +52,7 @@ processors:
       tag: set_host_name
   # Process syslog parameters
   - script:
+      tag: script_36f87397
       description: Translate log.syslog.priority to log.syslog.severity.code and log.syslog.facility.code
       lang: painless
       source: |
@@ -62,6 +65,7 @@ processors:
           ctx.log.syslog['facility'] = facility;
         }
   - script:
+      tag: script_75865bea
       description: Translate log.syslog.facility.code to log.syslog.facility.name
       lang: painless
       params:
@@ -95,6 +99,7 @@ processors:
         }
         ctx.log.syslog.facility.name = params[(ctx.log.syslog.facility.code).toString()];
   - script:
+      tag: script_ac58ee9e
       description: Translate log.syslog.severity.code to log.syslog.severity.name
       lang: painless
       params:
@@ -132,10 +137,12 @@ processors:
       tag: date_set_timestamp
       on_failure:
         - append:
+            tag: append_error_message_6ccd9b54
             field: error.message
             value: |-
               Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
         - set:
+            tag: set_event_kind_43729006
             field: event.kind
             value: pipeline_error
   # Choose the pipeline based on the log source
@@ -222,8 +229,11 @@ processors:
 on_failure:
   - append:
       field: error.message
-      value: |-
-        Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'
   - set:
       field: event.kind
       value: pipeline_error

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dhcp.yml

@@ -3,7 +3,7 @@ description: Pipeline for Sophos UTM DHCP logs
 processors:
   - append:
       field: event.type
-      value: 
+      value:
         - connection
         - protocol
       allow_duplicates: false
@@ -161,8 +161,11 @@ processors:
 on_failure:
   - append:
       field: error.message
-      value: |-
-        Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'
   - set:
       field: event.kind
       value: pipeline_error

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/dns.yml

@@ -3,7 +3,7 @@ description: Pipeline for Sophos UTM DNS logs
 processors:
   - append:
       field: event.type
-      value: 
+      value:
         - connection
         - protocol
       allow_duplicates: false
@@ -30,7 +30,7 @@ processors:
       patterns:
         - '^%{GREEDYDATA:message}$'
       tag: grok_dns_default
-  
+
   # Lowercase fields
   - lowercase:
       field: event.action
@@ -89,8 +89,11 @@ processors:
 on_failure:
   - append:
       field: error.message
-      value: |-
-        Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'
   - set:
       field: event.kind
       value: pipeline_error

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/http.yml

@@ -121,9 +121,11 @@ processors:
       ignore_missing: true
       tag: rename_user_agent
   - user_agent:
+      tag: user_agent_user_agent_original_b5325863
       field: user_agent.original
       ignore_missing: true
   - uri_parts:
+      tag: uri_parts_sophos_utm_url_to_url_f29fd9d9
       field: sophos.utm.url
       target_field: url
       remove_if_successful: true
@@ -195,6 +197,7 @@ processors:
 
   # Converts all kebab-case key names to snake_case
   - foreach:
+      tag: foreach_sophos_utm_94c822ba
       field: sophos.utm
       processor:
         gsub:
@@ -313,8 +316,11 @@ processors:
 on_failure:
   - append:
       field: error.message
-      value: |-
-        Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'
   - set:
       field: event.kind
       value: pipeline_error

packages/sophos/data_stream/utm/elasticsearch/ingest_pipeline/packetfilter.yml

@@ -20,14 +20,14 @@ processors:
   - append:
       if: ctx.event?.action == 'accept'
       field: event.type
-      value: 
+      value:
         - allowed
       allow_duplicates: false
       tag: append_type_allowed
   - append:
       if: ctx.event?.action == 'drop'
       field: event.type
-      value: 
+      value:
         - denied
       allow_duplicates: false
       tag: append_type_denied
@@ -268,8 +268,11 @@ processors:
 on_failure:
   - append:
       field: error.message
-      value: |-
-        Processor "{{{ _ingest.on_failure_processor_type }}}" with tag "{{{ _ingest.on_failure_processor_tag }}}" in pipeline "{{{ _ingest.on_failure_pipeline }}}" failed with message "{{{ _ingest.on_failure_message }}}"
+      value: >-
+        Processor '{{{ _ingest.on_failure_processor_type }}}'
+        {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+        {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+        failed with message '{{{ _ingest.on_failure_message }}}'
   - set:
       field: event.kind
       value: pipeline_error

packages/sophos/data_stream/xg/_dev/test/pipeline/test-time-zone.json

@@ -1,10 +1,10 @@
 {
-  "events": [
-    {
-      "message": "<30>device=\"SFW\" date=2020-05-18 time=14:38:48 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=041101618035 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"Allowed\" status=\"\" priority=Information fw_rule_id=0 user_name=\"\" av_policy_name=\"None\" from_email_address=\"[email protected]\" to_email_address=\"[email protected]\" email_subject=\"*ALERT* Sophos XG Firewall\" mailid=\"qkW2Y6-LxBk6U-vH-1590055245\" mailsize=19728 spamaction=\"QUEUED\" reason=\"Email has been accepted by Device and queued for scanning.\" src_domainname=\"elasticuser.com\" dst_domainname=\"\" src_ip=\"\" src_country_code=\"\" dst_ip=\"\" dst_country_code=\"\" protocol=\"TCP\" src_port=0 dst_port=0 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Other\"\n",
-      "event": {
-        "timezone": "-05:00"
-      }
-    }
-  ]
+    "events": [
+        {
+            "message": "<30>device=\"SFW\" date=2020-05-18 time=14:38:48 timezone=\"CEST\" device_name=\"XG230\" device_id=1234567890123456 log_id=041101618035 log_type=\"Anti-Spam\" log_component=\"SMTP\" log_subtype=\"Allowed\" status=\"\" priority=Information fw_rule_id=0 user_name=\"\" av_policy_name=\"None\" from_email_address=\"[email protected]\" to_email_address=\"[email protected]\" email_subject=\"*ALERT* Sophos XG Firewall\" mailid=\"qkW2Y6-LxBk6U-vH-1590055245\" mailsize=19728 spamaction=\"QUEUED\" reason=\"Email has been accepted by Device and queued for scanning.\" src_domainname=\"elasticuser.com\" dst_domainname=\"\" src_ip=\"\" src_country_code=\"\" dst_ip=\"\" dst_country_code=\"\" protocol=\"TCP\" src_port=0 dst_port=0 sent_bytes=0 recv_bytes=0 quarantine_reason=\"Other\"\n",
+            "event": {
+                "timezone": "-05:00"
+            }
+        }
+    ]
 }

packages/sophos/data_stream/xg/elasticsearch/ingest_pipeline/antispam.yml

@@ -5,38 +5,47 @@ processors:
 ## ECS Event Mapping ##
 #######################
 - set:
+    tag: set_event_kind_de80643c
     field: event.kind
     value: event
 - set:
+    tag: set_event_action_1bb29990
     field: event.action
     value: "{{{sophos.xg.log_subtype}}}"
     ignore_empty_value: true
 - set:
+    tag: set_event_outcome_1abe2ae5
     field: event.outcome
     value: success
     ignore_empty_value: true
 - set:
+    tag: set_event_kind_23ecd1e1
     field: event.kind
     value: alert
     if: '["13001", "13002", "13004", "13005", "13006", "13009", "13012", "13014", "14001", "14002", "15001", "15002"].contains(ctx.event?.code)'
 - append:
+    tag: append_event_category_70da92c5
     field: event.category
     value: malware
     if: '["13001", "13002", "13004", "13005", "13006", "13009", "13014", "14001", "14002", "15001", "15002"].contains(ctx.event?.code)'
 - append:
+    tag: append_event_category_c687ba22
     field: event.category
     value: intrusion_detection
     if: "ctx.event?.code == '13012'"
 - append:
+    tag: append_event_category_7afdca3c
     field: event.category
     value: network
 - append:
+    tag: append_event_type_fbb0b513
     field: event.type
     value:
      - allowed
      - connection
     if: '["13003", "13007", "13008", "13010", "13013", "14003", "15003", "18035"].contains(ctx.event?.code)'
 - append:
+    tag: append_event_type_8062f8c6
     field: event.type
     value:
       - info
@@ -48,11 +57,13 @@ processors:
 ## ECS Destination Mapping
 ####################################
 - rename:
+    tag: rename_sophos_xg_dst_ip_to_destination_ip_17b8be30
     field: sophos.xg.dst_ip
     target_field: destination.ip
     ignore_missing: true
     if: "ctx.sophos?.xg?.dst_ip != null"
 - convert:
+    tag: convert_sophos_xg_dst_port_to_destination_port_cea07508
     field: sophos.xg.dst_port
     target_field: destination.port
     type: long
@@ -64,17 +75,20 @@ processors:
 ## ECS Source Mapping
 ###############################
 - rename:
+    tag: rename_sophos_xg_src_ip_to_source_ip_3b2e0aa1
     field: sophos.xg.src_ip
     target_field: source.ip
     ignore_missing: true
 - convert:
+    tag: convert_sophos_xg_src_port_to_source_port_c97a8dfb
     field: sophos.xg.src_port
     target_field: source.port
     type: long
     ignore_failure: true
     ignore_missing: true
     if: "ctx.sophos?.xg?.src_port != null"
 - rename:
+     tag: rename_sophos_xg_src_domainname_to_source_domain_0625141c
      field: sophos.xg.src_domainname
      target_field: source.domain
      ignore_missing: true
@@ -83,26 +97,32 @@ processors:
 ## ECS Email Mapping ##
 #######################
 - rename:
+      tag: rename_sophos_xg_from_email_address_to_source_user_email_97624a63
       field: sophos.xg.from_email_address
       target_field: source.user.email
       ignore_missing: true
 - rename:
+      tag: rename_sophos_xg_to_email_address_to_destination_user_email_62e2176b
       field: sophos.xg.to_email_address
       target_field: destination.user.email
       ignore_missing: true
 - append:
+    tag: append_email_from_address_620b8c30
     field: email.from.address
     value: "{{{source.user.email}}}"
     if: "ctx?.source?.user?.email != null"
 - append:
+    tag: append_email_to_address_65b7cb39
     field: email.to.address
     value: "{{{destination.user.email}}}"
     if: "ctx?.destination?.user?.email != null"
 - set:
+    tag: set_email_subject_15d4cf78
     field: email.subject
     copy_from: sophos.xg.email_subject
     if: "ctx?.sophos.xg?.email_subject != null"
 - set:
+    tag: set_email_subject_2f9d7e59
     field: email.subject
     copy_from: sophos.xg.subject
     if: "ctx?.sophos.xg?.subject != null && ctx.email?.subject == null"
@@ -111,10 +131,12 @@ processors:
 ## ECS Network Mapping
 ######################
 - rename:
+    tag: rename_sophos_xg_protocol_to_network_transport_9c443cec
     field: sophos.xg.protocol
     target_field: network.transport
     ignore_missing: true
 - lowercase:
+    tag: lowercase_sophos_xg_log_component_to_network_protocol_7530f65c
     field: sophos.xg.log_component
     target_field: network.protocol
     ignore_missing: true
@@ -123,6 +145,7 @@ processors:
 ## Cleanup ##
 #############
 - remove:
+    tag: remove_2e5bd66a
     field:
     - sophos.xg.dst_port
     - sophos.xg.src_port
@@ -135,4 +158,8 @@ on_failure:
     value: pipeline_error
 - append:
     field: error.message
-    value: '{{{ _ingest.on_failure_message }}}'
+    value: >-
+      Processor '{{{ _ingest.on_failure_processor_type }}}'
+      {{#_ingest.on_failure_processor_tag}}with tag '{{{ _ingest.on_failure_processor_tag }}}'
+      {{/_ingest.on_failure_processor_tag}}in pipeline '{{{ _ingest.pipeline }}}'
+      failed with message '{{{ _ingest.on_failure_message }}}'