[o365] Fixed parsing and indexing errors

packages/o365/changelog.yml

@@ -1,4 +1,20 @@
 # newer versions go on top
+- version: "2.32.0"
+  changes:
+    - description: >-
+        Fix flattening errors in `Action` List items due to duplicate `QueryTime` fields by removing duplicate field.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15699
+    - description: >- 
+        Fixes undefined errors by adding fields `ActorInfoString`, `OperationCount`, `TokenObjectId`, `TokenTenantId`
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15699
+    - description: >-
+        Fixes errors due to SizeInBytes fields in `Messages` and `Folders` structures previously imported as long 
+        and then being sent as floats. Moves the fields to explicitly defined fields `ExchangeAggregatedMessages` and 
+        `ExchangeAggregatedFolders`and explicitly converts SizeInBytes to long for record type 50: `ExchangeItemAggregated`.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/15699
 - version: "2.31.0"
   changes:
     - description: Improve documentation.

packages/o365/data_stream/audit/elasticsearch/ingest_pipeline/default.yml

@@ -132,11 +132,15 @@ processors:
         if (!(ctx.o365audit.Actions instanceof List)) {
           ctx.o365audit.Actions = [ctx.o365audit.Actions];
         }
+
+        // Actions contains both a human readable `QueryTime` using AM/PM and an ISO8601 format `QueryTime`
+        // We remove the AM/PM containing `QueryTime` to avoid duplicate field errors on flattening.
+        def queryTimePattern = /,"QueryTime":"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M"|"QueryTime":"[0-9\/]+\s[0-9]+:[0-9]+:[0-9]+\s[AP]M",/;
         for (def e: ctx.o365audit.Actions) {
           if (e instanceof Map) {
             actions.add(e);
           } else if (e instanceof String) {
-            ctx._tmp.action_strings.add(e);
+            ctx._tmp.action_strings.add(queryTimePattern.matcher(e).replaceAll(''));
           }
         }
         if (actions.length == ctx.o365audit.Actions.length) {
@@ -672,11 +676,11 @@ processors:
       target_field: file.extension
       ignore_missing: true
       if: ctx.event?.code != null && ["SharePointFileOperation", "SharePointSharingOperation"].contains(ctx.event.code)
-  - append: 
+  - append:
       field: event.category
       value: file
       if: 'ctx.event?.action != null && ["FileAccessed", "FileDeleted", "FileDownloaded", "FileModified", "FileMoved", "FileRenamed", "FileRestored", "FileUploaded", "FolderCopied", "FolderCreated", "FolderDeleted", "FolderModified", "FolderMoved", "FolderRenamed", "FolderRestored"].contains(ctx.event?.action)'
-  - append: 
+  - append:
       field: event.category
       value: configuration
       if: ctx.event?.action == "ComplianceSettingChanged"
@@ -1398,6 +1402,26 @@ processors:
         } else {
           ctx.o365audit.YammerNetworkId = ctx.o365audit.YammerNetworkId.toString();
         }
+  - script:
+      tag: convert_runningtime
+      description: Ensure that RunningTime is not rendered with e-notation or other numeric
+      if: ctx.o365audit?.RunningTime != null
+      source: |-
+        if (ctx.o365audit.RunningTime instanceof double) {
+          ctx.o365audit.RunningTime = ((long)ctx.o365audit.RunningTime).toString();
+        } else {
+          ctx.o365audit.RunningTime = ctx.o365audit.RunningTime.toString();
+        }
+  - script:
+      tag: convert_operationcount
+      description: Ensure that OperationCount is not rendered with e-notation or other numeric
+      if: ctx.o365audit?.OperationCount != null
+      source: |-
+        if (ctx.o365audit.OperationCount instanceof Number) {
+          ctx.o365audit.OperationCount = ((long)ctx.o365audit.OperationCount).toString();
+        } else {
+          ctx.o365audit.OperationCount = ctx.o365audit.OperationCount.toString();
+        }
   - append:
       field: email.message_id
       value: "{{{o365audit.InternetMessageId}}}"
@@ -1446,6 +1470,7 @@ processors:
       field: o365audit.EndTimeUtc
       target_field: o365audit.EndTimeUtc
       tag: date_EndTimeUtc
+      timezone: "UTC"
       formats:
         - ISO8601
       if: ctx.o365audit?.EndTimeUtc != null
@@ -1770,6 +1795,66 @@ processors:
       copy_from: o365audit.ApplicationDisplayName
       tag: set_application_name
       ignore_empty_value: true
+
+  # ExchangeItemAggregated Schema
+  - append:
+      field: event.type
+      value: access
+      if: ctx.o365audit?.RecordType == "50"
+  - append:
+      field: event.category
+      value: email
+      if: ctx.o365audit?.RecordType == "50"
+  - rename:
+      field: o365audit.Messages
+      target_field: o365audit.ExchangeAggregatedMessages
+      tag: rename_messages_exchange
+      description: 'move generic Messages field to the ExchangeAggregatedMessages field type'
+      if: ctx.o365audit?.Messages != null && ctx.o365audit.RecordType == "50"
+  - script:
+      tag: convert_exchange_message_size_to_long
+      if: ctx.o365audit?.ExchangeAggregatedMessages != null
+      lang: painless
+      source: |
+        for (def i = 0; i < ctx.o365audit.ExchangeAggregatedMessages.length; i++) {
+          if (ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems == null) {
+             continue;
+          }
+          for (def j = 0; j < ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems.length; j++) {
+            def size = ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes;
+            if (size instanceof String) {
+              ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes = Long.parseLong(size);
+            } else {
+              ctx.o365audit.ExchangeAggregatedMessages[i].MessageItems[j].SizeInBytes = (long)size;
+            }
+          }
+        }
+
+  - rename:
+      field: o365audit.Folders
+      target_field: o365audit.ExchangeAggregatedFolders
+      tag: rename_folders_exchange
+      description: 'move generic Folders field to the O365 ExchangeAggregatedFolders field type'
+      if: ctx.o365audit?.Folders != null && ctx.o365audit.RecordType == "50"
+  - script:
+      tag: convert_exchange_folder_size_to_long
+      if: ctx.o365audit?.ExchangeAggregatedFolders != null
+      lang: painless
+      source: |
+        for (def i = 0; i < ctx.o365audit.ExchangeAggregatedFolders.length; i++) {
+          if (ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems == null) {
+              continue;
+          }
+          for (def j = 0; j < ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems.length; j++) {
+            def size = ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes;
+            if (size instanceof String) {
+              ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes = Long.parseLong(size);
+            } else {
+              ctx.o365audit.ExchangeAggregatedFolders[i].FolderItems[j].SizeInBytes = (long)size;
+            }
+          }
+        }
+
   - script:
       description: Handle _tmp.entities.ThreatDetectionMethods containing list of lists.
       lang: painless

packages/o365/data_stream/audit/fields/fields.yml

@@ -16,6 +16,8 @@
           type: keyword
     - name: ActorContextId
       type: keyword
+    - name: ActorInfoString
+      type: keyword
     - name: ActorIpAddress
       type: keyword
     - name: ActorUserId
@@ -275,6 +277,52 @@
       # not expressible here; object_type_mapping_type cannot be 'boolean'.
       object_type: keyword
       object_type_mapping_type: '*'
+    - name: ExchangeAggregatedFolders
+      type: nested
+      description: List of folders
+      fields:
+        - name: Path
+          type: keyword
+          description: Path of the folder
+        - name: Id
+          type: keyword
+          description: Folder ID
+        - name: FolderItems
+          type: nested
+          description: Items in the folder
+          fields:
+            - name: SizeInBytes
+              type: long
+              description: Size of the item in bytes
+            - name: Id
+              type: keyword
+              description: Item ID
+            - name: ImmutableId
+              type: keyword
+              description: Immutable ID of the item
+            - name: InternetMessageId
+              type: keyword
+              description: Internet message ID
+    - name: ExchangeAggregatedMessages
+      type: nested
+      description: List of messages
+      fields:
+        - name: Path
+          type: keyword
+          description: Path of the message
+        - name: Id
+          type: keyword
+          description: Message ID
+        - name: MessageItems
+          type: nested
+          description: Items in the message
+          fields:
+            - name: SizeInBytes
+              type: long
+              description: Size of the message item in bytes
+            - name: Id
+              type: keyword
+              description: Message item ID
     - name: ExchangeMetaData
       type: group
       fields:
@@ -415,6 +463,8 @@
       type: keyword
     - name: Operation
       type: keyword
+    - name: OperationCount
+      type: keyword
     - name: OperationId
       type: keyword
     - name: OperationProperties
@@ -604,6 +654,10 @@
       type: keyword
     - name: ThreatDetectionMethods
       type: keyword
+    - name: TokenObjectId
+      type: keyword
+    - name: TokenTenantId
+      type: keyword
     - name: Timestamp
       type: keyword
     - name: UniqueSharingId

packages/o365/docs/README.md

@@ -237,6 +237,7 @@ An example event for `audit` looks as following:
 | o365.audit.Actor.ID |  | keyword |
 | o365.audit.Actor.Type |  | keyword |
 | o365.audit.ActorContextId |  | keyword |
+| o365.audit.ActorInfoString |  | keyword |
 | o365.audit.ActorIpAddress |  | keyword |
 | o365.audit.ActorUserId |  | keyword |
 | o365.audit.ActorYammerUserId |  | keyword |
@@ -356,6 +357,16 @@ An example event for `audit` looks as following:
 | o365.audit.EventDeepLink |  | keyword |
 | o365.audit.EventSource |  | keyword |
 | o365.audit.ExceptionInfo.\* |  | object |
+| o365.audit.ExchangeAggregatedFolders.FolderItems.Id | Item ID | keyword |
+| o365.audit.ExchangeAggregatedFolders.FolderItems.ImmutableId | Immutable ID of the item | keyword |
+| o365.audit.ExchangeAggregatedFolders.FolderItems.InternetMessageId | Internet message ID | keyword |
+| o365.audit.ExchangeAggregatedFolders.FolderItems.SizeInBytes | Size of the item in bytes | long |
+| o365.audit.ExchangeAggregatedFolders.Id | Folder ID | keyword |
+| o365.audit.ExchangeAggregatedFolders.Path | Path of the folder | keyword |
+| o365.audit.ExchangeAggregatedMessages.Id | Message ID | keyword |
+| o365.audit.ExchangeAggregatedMessages.MessageItems.Id | Message item ID | keyword |
+| o365.audit.ExchangeAggregatedMessages.MessageItems.SizeInBytes | Size of the message item in bytes | long |
+| o365.audit.ExchangeAggregatedMessages.Path | Path of the message | keyword |
 | o365.audit.ExchangeMetaData.\* |  | long |
 | o365.audit.ExchangeMetaData.CC |  | keyword |
 | o365.audit.ExchangeMetaData.MessageID |  | keyword |
@@ -417,6 +428,7 @@ An example event for `audit` looks as following:
 | o365.audit.ObjectId |  | keyword |
 | o365.audit.ObjectType |  | keyword |
 | o365.audit.Operation |  | keyword |
+| o365.audit.OperationCount |  | keyword |
 | o365.audit.OperationId |  | keyword |
 | o365.audit.OperationProperties |  | object |
 | o365.audit.OrganizationId |  | keyword |
@@ -501,6 +513,8 @@ An example event for `audit` looks as following:
 | o365.audit.TeamName |  | keyword |
 | o365.audit.ThreatDetectionMethods |  | keyword |
 | o365.audit.Timestamp |  | keyword |
+| o365.audit.TokenObjectId |  | keyword |
+| o365.audit.TokenTenantId |  | keyword |
 | o365.audit.UniqueSharingId |  | keyword |
 | o365.audit.UserAgent |  | keyword |
 | o365.audit.UserId |  | keyword |

packages/o365/manifest.yml

@@ -1,6 +1,6 @@
 name: o365
 title: Microsoft Office 365
-version: "2.31.0"
+version: "2.32.0"
 description: Collect logs from Microsoft Office 365 with Elastic Agent.
 type: integration
 format_version: "3.2.3"