fix-parsing-error-due-to-duplicate-fields Don't remove Mesages or Folders field if they have not been renamed

Author: StacieClark-Elastic

packages/o365/changelog.yml

@@ -3,14 +3,14 @@
   changes:
     - description: >-
         Fix flattening errors in `Action` List items due to duplicate `QueryTime` fields by removing duplicate field.
-      type: bugfix
+      type: enhancement
       link: https://github.com/elastic/integrations/pull/15699
     - description: >- 
         Fixes undefined errors by adding fields `ActorInfoString`, `OperationCount`, `TokenObjectId`, `TokenTenantId`
       type: bugfix
       link: https://github.com/elastic/integrations/pull/15699
     - description: >-
-        Fixes errors due to SizInBytes fields in `Messages` and `Folders` structures previously imported as long 
+        Fixes errors due to SizeInBytes fields in `Messages` and `Folders` structures previously imported as long 
         and then being sent as floats. Moves the fields to explicitly defined fields `ExchangeAggregatedMessages` and 
         `ExchangeAggregatedFolders`and explicitly converts SizeInBytes to long for record type 50: `ExchangeItemAggregated`.
       type: bugfix

packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml

@@ -132,16 +132,15 @@ processors:
         if (!(ctx.o365audit.Actions instanceof List)) {
           ctx.o365audit.Actions = [ctx.o365audit.Actions];
         }
-        /*
-        * Actions contains both a human readable `QueryTime` using AM/PM and an ISO8601 format `QueryTime`
-        * We remove the AM/PM containing `QueryTime` to avoid duplicate field errors on flattening.
-        */
-        def regex = /,"QueryTime":"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M"|"QueryTime":"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M",/;
+      
+        // Actions contains both a human readable `QueryTime` using AM/PM and an ISO8601 format `QueryTime`
+        // We remove the AM/PM containing `QueryTime` to avoid duplicate field errors on flattening.
+        def queryTimePattern = /,"QueryTime":"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M"|"QueryTime":"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M",/;
         for (def e: ctx.o365audit.Actions) {
           if (e instanceof Map) {
             actions.add(e);
           } else if (e instanceof String) {
-            ctx._tmp.action_strings.add(regex.matcher(e).replaceAll(''));
+            ctx._tmp.action_strings.add(queryTimePattern.matcher(e).replaceAll(''));
           }
         }
         if (actions.length == ctx.o365audit.Actions.length) {
@@ -1801,70 +1800,61 @@ processors:
   - append:
       field: event.type
       value: access
-      if: ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50"
+      if: ctx.o365audit?.RecordType == "50"
   - append:
       field: event.category
       value: email
-      if: ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50"
+      if: ctx.o365audit?.RecordType == "50"
   - rename:
       field: o365audit.Messages
       target_field: o365audit.ExchangeAggregatedMessages
       tag: rename_messages_exchange
-      description: 'Move generic Messages field to the ExchangeAggregatedMessages field type'
-      if: ctx.o365audit?.Messages != null && ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50"
+      description: 'move generic Messages field to the ExchangeAggregatedMessages field type'
+      if: ctx.o365audit?.Messages != null && ctx.o365audit.RecordType == "50"
   - script:
       tag: convert_exchange_message_size_to_long
       if: ctx.o365audit?.ExchangeAggregatedMessages != null
       lang: painless
       source: |
         for (def i = 0; i < ctx.o365audit.ExchangeAggregatedMessages.length; i++) {
-          if (ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems != null) {
-            for (def j = 0; j < ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems.length; j++) {
-              def size = ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes;
-              if (size instanceof String) {
-                ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes = Long.parseLong(size);
-              } else {
-                ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes = (long)size;
-              }
+          if (ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems == null) {
+             continue;
+          }
+          for (def j = 0; j < ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems.length; j++) {
+            def size = ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes;
+            if (size instanceof String) {
+              ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes = Long.parseLong(size);
+            } else {
+              ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes = (long)size;
             }
           }
         }
 
-  - remove:
-      field: o365audit.Messages
-      tag: remove_messages_field
-      if: ctx.o365audit?.Messages != null
-      description: 'remove o365audit.Messages if we have not explicitly renamed them based on record type'
-
   - rename:
       field: o365audit.Folders
       target_field: o365audit.ExchangeAggregatedFolders
       tag: rename_folders_exchange
-      description: 'Move generic Folders field to the O365 ExchangeAggregatedFolders field type'
-      if: ctx.o365audit?.Folders != null && ctx.o365audit?.RecordType != null && ctx.o365audit?.RecordType == "50"
+      description: 'move generic Folders field to the O365 ExchangeAggregatedFolders field type'
+      if: ctx.o365audit?.Folders != null && ctx.o365audit.RecordType == "50"
   - script:
       tag: convert_exchange_folder_size_to_long
       if: ctx.o365audit?.ExchangeAggregatedFolders != null
       lang: painless
       source: |
         for (def i = 0; i < ctx.o365audit.ExchangeAggregatedFolders.length; i++) {
-          if (ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems != null) {
-            for (def j = 0; j < ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems.length; j++) {
-              def size = ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes;
-              if (size instanceof String) {
-                ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes = Long.parseLong(size);
-              } else {
-                ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes = (long)size;
-              }
+          if (ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems == null) {
+              continue;
+          }
+          for (def j = 0; j < ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems.length; j++) {
+            def size = ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes;
+            if (size instanceof String) {
+              ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes = Long.parseLong(size);
+            } else {
+              ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes = (long)size;
             }
           }
         }
 
-  - remove:
-      field: o365audit.Folders
-      tag: remove_folders_field
-      if: ctx.o365audit?.Folders != null
-      description: 'Remove o365audit.Folders if we have not explicitly renamed them based on record type'
   - script:
       description: Handle _tmp.entities.ThreatDetectionMethods containing list of lists.
       lang: painless