diff --git a/baseline-results.csv b/baseline-results.csv new file mode 100644 index 0000000..47acd84 --- /dev/null +++ b/baseline-results.csv @@ -0,0 +1,16 @@ +command,mean,stddev,median,user,system,min,max +01-passthrough-tsv,0.66335433384,0.01682146201750171,0.66601856204,0.5140556199999999,0.14985426,0.64621943104,0.68657031004 +02-passthrough-json,1.32866054724,0.03433318243449869,1.32225606504,1.01681782,0.31346366,1.2809032550400001,1.36213361704 +03-line-filter-tsv,1.9981326990400003,0.022830074964112598,1.99742874204,1.84471542,0.15384405999999998,1.97727274404,2.03358676504 +04-line-filter-json,4.60715976144,0.1819041054682861,4.61132453404,4.05177522,0.55677266,4.35345054604,4.788800295040001 +05-at-column-filter-tsv,1.7714589674400003,0.00904163024753627,1.77316088004,1.56909762,0.20349725999999996,1.75959619604,1.78390190004 +06-at-column-filter-json,9.176219561439998,0.12270907337465913,9.21773630004,8.385497619999999,0.77066646,8.969152809039999,9.293088575039999 +07-atat-column-filter-tsv,2.11805135724,0.023085245503879803,2.12826743904,1.92706882,0.19230966,2.08036520804,2.13831083404 +08-atat-column-filter-json,9.328013240239999,0.24676947367829533,9.414504064039999,8.55142022,0.77987606,8.905050726039999,9.53294901204 +09-mutator-tsv,6.018914668906667,0.1338733140062458,6.08180925124,5.800859113333334,0.21936123333333332,5.86517564524,6.10975911024 +10-mutator-json,22.98265302388667,0.3160655408217702,22.84062220122,22.511812719999998,0.4732879266666667,22.76251922922,23.34481764122 +11-mutator-filter-tsv,3.8698376845533335,0.06932762152645182,3.8913622022200003,3.6719947199999994,0.19787892666666665,3.79230087622,3.9258499752200002 +12-mutator-filter-json,18.705930759886666,0.1697635546411535,18.78274739422,18.00068872,0.70838526,18.51133643822,18.82370844722 +13-format-zeek-to-json,24.628451007886667,0.15874012321473524,24.56034414322,24.373273386666668,0.25756992666666667,24.515129008219997,24.809879872219998 +14-format-json-to-zeek,9.503400084219999,0.2628510924230409,9.58666781522,8.847611053333331,0.65929826,9.20900039722,9.71453204022 + diff --git a/bench.lisp b/bench.lisp new file mode 100644 index 0000000..e16adb3 --- /dev/null +++ b/bench.lisp @@ -0,0 +1,149 @@ +(asdf:load-system :cleek) + +(in-package :cleek) + +(require :sb-sprof) + +(defparameter *bench-zeek-input* + (asdf:system-relative-pathname "cleek" "data/test-input/homenet-uncompressed.zeek.log")) + +(defparameter *bench-json-input* + (asdf:system-relative-pathname "cleek" "data/test-input/homenet-uncompressed.json.log")) + +;;; --- Timing infrastructure (mirrors perf.lisp:my-time) --- + +(defmacro bench-time (&body body) + "Execute BODY and return (values result seconds bytes-consed)." + `(let ((start-bytes (sb-ext:get-bytes-consed)) + (start-time (get-internal-real-time))) + (let ((result (progn ,@body))) + (let ((elapsed-seconds (/ (- (get-internal-real-time) start-time) + (float internal-time-units-per-second 1.0d0))) + (bytes (- (sb-ext:get-bytes-consed) start-bytes))) + (values result elapsed-seconds bytes))))) + +(defun run-bench (name n thunk) + "Run THUNK N times, report timing as CSV row to *standard-output*." + (sb-ext:gc :full t) + (multiple-value-bind (_result elapsed bytes) + (bench-time (dotimes (_ n) (funcall thunk))) + (declare (ignore _result)) + (format t "~a,~a,~,6f,~d,~,6f~%" + name n elapsed bytes (/ elapsed n)))) + +;;; --- Function-level benchmarks --- + +(defun bench-ensure-row-strings (&optional (n 3)) + (with-zeek-log (log *bench-zeek-input* '(:proto)) + (run-bench "ensure-row-strings" n + (lambda () + (loop while (zeek-line log) + do (setf (zeek-status log) :unparsed) + (ensure-row-strings log) + (next-record log)))))) + +(defun bench-parse-zeek-type (&optional (n 100000)) + (run-bench "parse-zeek-type" n + (lambda () + (parse-zeek-type "1623187704.078114" :time) + (parse-zeek-type "48610" :port) + (parse-zeek-type "140.249.20.119" :addr) + (parse-zeek-type "tcp" :enum) + (parse-zeek-type "12345" :count)))) + +(defun bench-ensure-map-zeek (&optional (n 3)) + (with-zeek-log (log *bench-zeek-input* '(:proto)) + (run-bench "ensure-map-zeek" n + (lambda () + (loop while (zeek-line log) + do (setf (zeek-status log) :unparsed) + (clrhash (zeek-map log)) + (ensure-map log) + (next-record log)))))) + +(defun bench-ensure-map-json (&optional (n 3)) + (with-zeek-log (log *bench-json-input* '(:proto)) + (run-bench "ensure-map-json" n + (lambda () + (loop while (zeek-line log) + do (setf (zeek-status log) :unparsed) + (clrhash (zeek-map log)) + (ensure-map log) + (next-record log)))))) + +(defun bench-write-zeek-log-line (&optional (n 3)) + (with-zeek-log (log *bench-zeek-input* '(:proto)) + (with-open-file (out "/dev/null" :direction :output :if-exists :supersede) + (run-bench "write-zeek-log-line" n + (lambda () + (loop while (zeek-line log) + do (write-zeek-log-line log out :zeek) + (next-record log))))))) + +(defun bench-next-record (&optional (n 3)) + (with-zeek-log (log *bench-zeek-input*) + (run-bench "next-record" n + (lambda () + (loop while (zeek-line log) do (next-record log)))))) + +(defun run-all-function-benchmarks (&optional (output-path "bench-results.csv")) + "Run all function-level benchmarks and write CSV to OUTPUT-PATH." + (with-open-file (*standard-output* output-path :direction :output :if-exists :supersede) + (format t "benchmark,iterations,total_seconds,bytes_consed,seconds_per_iteration~%") + (bench-ensure-row-strings) + (bench-parse-zeek-type) + (bench-ensure-map-zeek) + (bench-ensure-map-json) + (bench-write-zeek-log-line) + (bench-next-record)) + (format *error-output* "Function benchmarks written to ~a~%" output-path)) + +;;; --- sb-sprof profiling --- + +(defun profile-function (name thunk) + "Profile THUNK with sb-sprof. NAME is for display only." + (format t "~%=== Profiling: ~a ===~%" name) + (sb-sprof:with-profiling (:report :graph :sample-interval 0.001) + (funcall thunk))) + +(defun profile-passthrough-zeek () + (profile-function "passthrough-zeek" + (lambda () (cat-logs-string #P"/dev/null" :zeek nil nil *bench-zeek-input*)))) + +(defun profile-passthrough-json () + (profile-function "passthrough-json" + (lambda () (cat-logs-string #P"/dev/null" :json nil nil *bench-json-input*)))) + +(defun profile-filter-zeek () + (profile-function "filter-@@-zeek" + (lambda () (cat-logs-string #P"/dev/null" :zeek nil + "(and (plusp @@orig_bytes) (plusp @@resp_bytes))" + *bench-zeek-input*)))) + +(defun profile-filter-json () + (profile-function "filter-@@-json" + (lambda () (cat-logs-string #P"/dev/null" :json nil + "(and (plusp @@orig_bytes) (plusp @@resp_bytes))" + *bench-json-input*)))) + +(defun profile-mutator-zeek () + (profile-function "mutator-zeek" + (lambda () (cat-logs-string #P"/dev/null" :zeek + "(anonip! @id.orig_h @id.resp_h)" nil + *bench-zeek-input*)))) + +(defun profile-format-conversion () + (profile-function "zeek->json" + (lambda () (cat-logs-string #P"/dev/null" :json nil nil *bench-zeek-input*)))) + +(defun run-all-profiles () + "Run sb-sprof profiling on key scenarios." + (profile-passthrough-zeek) + (sb-ext:gc :full t) + (profile-passthrough-json) + (sb-ext:gc :full t) + (profile-filter-zeek) + (sb-ext:gc :full t) + (profile-mutator-zeek) + (sb-ext:gc :full t) + (profile-format-conversion)) diff --git a/benchmark.sh b/benchmark.sh new file mode 100755 index 0000000..473df08 --- /dev/null +++ b/benchmark.sh @@ -0,0 +1,116 @@ +#!/bin/bash +# A/B benchmark suite for comparing two cleek binaries using hyperfine. +# +# Usage: bash benchmark.sh [CLEEK_A] [CLEEK_B] +# +# Defaults: +# CLEEK_A = bin/cleek +# CLEEK_B = bin/cleek.old +# +# Prerequisites: +# - hyperfine installed (https://github.com/sharkdp/hyperfine) +# - Both binaries built +# - data/test-input/homenet-uncompressed.{zeek,json}.log present + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CLEEK_A="${1:-${SCRIPT_DIR}/bin/cleek}" +CLEEK_B="${2:-${SCRIPT_DIR}/bin/cleek.old}" +ZEEK_INPUT="${SCRIPT_DIR}/data/test-input/homenet-uncompressed.zeek.log" +JSON_INPUT="${SCRIPT_DIR}/data/test-input/homenet-uncompressed.json.log" + +# --- Preflight checks --- +if ! command -v hyperfine &>/dev/null; then + echo "ERROR: hyperfine not found. Install from https://github.com/sharkdp/hyperfine" >&2 + exit 1 +fi + +for bin in "${CLEEK_A}" "${CLEEK_B}"; do + if [[ ! -x "${bin}" ]]; then + echo "ERROR: ${bin} not found or not executable." >&2 + exit 1 + fi +done + +for f in "${ZEEK_INPUT}" "${JSON_INPUT}"; do + if [[ ! -f "${f}" ]]; then + echo "ERROR: Input file not found: ${f}" >&2 + exit 1 + fi +done + +LABEL_A="$(basename "${CLEEK_A}")" +LABEL_B="$(basename "${CLEEK_B}")" + +# Disambiguate labels when basenames collide (e.g. both are "cleek") +if [[ "${LABEL_A}" == "${LABEL_B}" ]]; then + LABEL_A="${CLEEK_A}" + LABEL_B="${CLEEK_B}" +fi + +echo "=== cleek A/B Benchmark Suite ===" +echo "Binary A: ${CLEEK_A}" +echo "Binary B: ${CLEEK_B}" +echo "Zeek input: ${ZEEK_INPUT}" +echo "JSON input: ${JSON_INPUT}" +echo "" + +run_bench() { + local name="$1" + local args="$2" + + echo "--- ${name} ---" + hyperfine \ + --warmup 1 \ + --min-runs 5 \ + --command-name "${LABEL_A}" \ + "${CLEEK_A} ${args}" \ + --command-name "${LABEL_B}" \ + "${CLEEK_B} ${args}" + echo "" +} + +run_bench "01-passthrough-tsv" \ + "${ZEEK_INPUT} > /dev/null" + +run_bench "02-passthrough-json" \ + "${JSON_INPUT} > /dev/null" + +run_bench "03-line-filter-tsv" \ + "-x '(~ \"tcp\" LINE)' ${ZEEK_INPUT} > /dev/null" + +run_bench "04-line-filter-json" \ + "-x '(~ \"tcp\" LINE)' ${JSON_INPUT} > /dev/null" + +run_bench "05-at-column-filter-tsv" \ + "-x '(string= @proto \"tcp\")' ${ZEEK_INPUT} > /dev/null" + +run_bench "06-at-column-filter-json" \ + "-x '(string= @proto \"tcp\")' ${JSON_INPUT} > /dev/null" + +run_bench "07-atat-column-filter-tsv" \ + "-x '(and (plusp @@orig_bytes) (plusp @@resp_bytes))' ${ZEEK_INPUT} > /dev/null" + +run_bench "08-atat-column-filter-json" \ + "-x '(and (plusp @@orig_bytes) (plusp @@resp_bytes))' ${JSON_INPUT} > /dev/null" + +run_bench "09-mutator-tsv" \ + "-m '(anonip! @id.orig_h @id.resp_h)' ${ZEEK_INPUT} > /dev/null" + +run_bench "10-mutator-json" \ + "-m '(anonip! @id.orig_h @id.resp_h)' ${JSON_INPUT} > /dev/null" + +run_bench "11-mutator-filter-tsv" \ + "-m '(setf @total_bytes (+ @@orig_bytes @@resp_bytes))' -x '(plusp @total_bytes)' ${ZEEK_INPUT} > /dev/null" + +run_bench "12-mutator-filter-json" \ + "-m '(setf @total_bytes (+ @@orig_bytes @@resp_bytes))' -x '(plusp @total_bytes)' ${JSON_INPUT} > /dev/null" + +run_bench "13-format-zeek-to-json" \ + "-f json ${ZEEK_INPUT} > /dev/null" + +run_bench "14-format-json-to-zeek" \ + "-f zeek ${JSON_INPUT} > /dev/null" + +echo "=== Benchmark complete ===" diff --git a/helpers.lisp b/helpers.lisp index 80fb7c5..11deb3b 100644 --- a/helpers.lisp +++ b/helpers.lisp @@ -90,25 +90,38 @@ (:method ((field t)) (sha256-string (format nil "~a" field)))) +;; TODO: You should rewrite this so you can provide a seed so the same binary doesn't always generate the same +;; permutation (but a single run would). (let* ((v6-permutors (loop repeat 16 collect (ax:shuffle (coerce (loop for x upto 255 collect x) 'vector)))) (v4-permutors (nthcdr 12 v6-permutors)) (v4-string-permutors (loop for p in v4-permutors collect (map 'vector #'write-to-string p)))) - (defgeneric anonip (ip) - (:documentation "Anonymize an IP address by permuting each byte with a fixed set of permutations for each byte.") - (:method ((ip string)) + (defgeneric anonip (ip &optional num-octets) + (:documentation "Anonymize an IP address by permuting each byte with a fixed set of permutations for each byte. Optionally, only anonymize the first NUM-OCTETS starting with the most significant. Defaults to anonymizing all of them.") + (:method ((ip string) &optional num-octets) + (unless num-octets + (setf num-octets (if (na::ipv4-str? ip) 4 16))) (if (na::ipv4-str? ip) (let ((quads (split-sequence #\. ip))) - (setf (first quads) (aref (first v4-string-permutors) (parse-integer (first quads))) - (second quads) (aref (second v4-string-permutors) (parse-integer (second quads))) - (third quads) (aref (third v4-string-permutors) (parse-integer (third quads))) - (fourth quads) (aref (fourth v4-string-permutors) (parse-integer (fourth quads)))) + (setf (first quads) (if (>= num-octets 1) + (aref (first v4-string-permutors) (parse-integer (first quads))) + (first quads)) + (second quads) (if (>= num-octets 2) + (aref (second v4-string-permutors) (parse-integer (second quads))) + (second quads)) + (third quads) (if (>= num-octets 3) + (aref (third v4-string-permutors) (parse-integer (third quads))) + (third quads)) + (fourth quads) (if (>= num-octets 4) + (aref (fourth v4-string-permutors) (parse-integer (fourth quads))) + (fourth quads))) (str:join "." quads)) - (str:downcase (na:str (anonip (na:make-ip-address ip)))))) - (:method ((ip na::ip-address)) + (str:downcase (na:str (anonip (na:make-ip-address ip) num-octets))))) + (:method ((ip na::ip-address) &optional num-octets) (let ((version (na:version ip)) (ip (na:make-ip-address (na:int ip)))) (loop for offset from (if (= version 4) 24 120) downto 0 by 8 for permutor in (if (= version 4) v4-permutors v6-permutors) + repeat (or num-octets 16) ; default is always sufficient for IPv6/IPv4. do (setf (ldb (byte 8 offset) (slot-value ip 'netaddr::int)) (aref permutor (ldb (byte 8 offset) (slot-value ip 'netaddr::int))))) (setf (slot-value ip 'netaddr:str) (na::ip-int-to-str (na:int ip) version)) @@ -127,6 +140,12 @@ (defalias private? #'na:private? "Alias for NETADDR:PRIVATE? which returns T if the IP address is privately routable. Requires a NETADDR::IP-LIKE (so fully parse with @@).") (defalias reserved? #'na:reserved? "Alias for NETADDR:RESERVED? which returns T if the IP address is reserved. Requires a NETADDR::IP-LIKE (so fully parse with @@).") +(defun routes (ip-like) + (cond ((private? ip-like) "private") + ((public? ip-like) "public") + ((reserved? ip-like) "reserved") + (t "other"))) + ;; is there a reasonable way to anonymize domains? (defmacro ~ (regex field) diff --git a/io.lisp b/io.lisp index 80dfadd..43d51c4 100644 --- a/io.lisp +++ b/io.lisp @@ -41,22 +41,64 @@ (:json (gethash field (zeek-map zeek-log))))) (defun get-value (zeek-log field &optional fully-parsed?) + (declare (optimize (speed 3) (debug 0) (space 0) (compilation-speed 0)) + (type zeek zeek-log) + (type keyword field)) (ecase (zeek-format zeek-log) - (:zeek (let ((val (if (and (zeek-created-fields zeek-log) (not (has-field? zeek-log field))) - (gethash field (zeek-map zeek-log)) - (aref (zeek-row-strings zeek-log) (field->idx zeek-log field))))) - (if fully-parsed? - (parse-zeek-type val (field->type zeek-log field) t) - val))) + (:zeek (if (and (zeek-created-fields zeek-log) (not (has-field? zeek-log field))) + ;; Created fields: read from zeek-map (unchanged) + (let ((val (gethash field (zeek-map zeek-log)))) + (if fully-parsed? + (parse-zeek-type val (field->type zeek-log field) t) + val)) + ;; Regular fields: lazy materialization from offsets + (let* ((idx (the fixnum (field->idx zeek-log field))) + (cached (svref (zeek-row-strings zeek-log) idx))) + (if cached + (if fully-parsed? + (parse-zeek-type cached (field->type zeek-log field) t) + cached) + (let* ((offsets (zeek-offsets zeek-log)) + (line (zeek-line zeek-log)) + (start (if (zerop idx) + (aref offsets idx) + (1+ (aref offsets idx)))) + (end (aref offsets (1+ idx))) + (type (field->type zeek-log field))) + (declare (type (simple-array fixnum (*)) offsets) + (type simple-string line) + (type fixnum start end)) + ;; For numeric types with fully-parsed?, parse directly + ;; from line with :start/:end — avoids subseq allocation. + (if (and fully-parsed? + ;; Not unset ("-") or empty "(empty)" + (/= (- end start) 1) + (member type '(:double :interval :count :int :port) :test #'eq)) + (case type + ((:double :interval) + (fast-parse-double line :start start :end end)) + ((:count :int :port) + (parse-integer line :start start :end end))) + ;; Non-numeric or needs subseq: materialize and cache + (let ((val (subseq line start end))) + (setf (svref (zeek-row-strings zeek-log) idx) val) + (if fully-parsed? + (parse-zeek-type val type t) + val)))))))) (:json (if fully-parsed? (zeekify-json-type (gethash field (zeek-map zeek-log)) (field->type zeek-log field)) (gethash field (zeek-map zeek-log)))))) (defun (setf get-value) (new-value zeek-log field &optional fully-parsed?) - (declare (ignore fully-parsed?)) + (declare (optimize (speed 3) (debug 0) (space 0) (compilation-speed 0)) + (ignore fully-parsed?) + (type zeek zeek-log) + (type keyword field)) (if (has-field? zeek-log field) (prog1 (ecase (zeek-format zeek-log) - (:zeek (setf (aref (zeek-row-strings zeek-log) (field->idx zeek-log field)) new-value)) + (:zeek (setf (aref (the simple-vector (zeek-row-strings zeek-log)) + (the fixnum (field->idx zeek-log field))) + new-value)) (:json (setf (gethash field (zeek-map zeek-log)) new-value))) (setf (zeek-modified? zeek-log) t)) (prog1 (ecase (zeek-format zeek-log) @@ -75,6 +117,12 @@ (ecase (zeek-format zeek-log) (:zeek (parse-zeek-header zeek-log)) (:json (next-record zeek-log) (infer-log-path-fields-types zeek-log))) + ;; Pre-allocate offsets and row-strings vectors now that field count is known. + (when (zeek-fields zeek-log) + (setf (zeek-offsets zeek-log) + (make-array (1+ (length (zeek-fields zeek-log))) :element-type 'fixnum) + (zeek-row-strings zeek-log) + (make-array (length (zeek-fields zeek-log)) :initial-element nil))) (when (zeek-accessed-columns zeek-log) (ensure-fields->idx zeek-log) (ecase (zeek-format zeek-log) @@ -84,6 +132,8 @@ (defun next-record (zeek-log) ;; TODO: Add condition handling for when the header differs. + (declare (optimize (speed 3) (debug 0) (space 0) (compilation-speed 0)) + (type zeek zeek-log)) (progn (setf (zeek-line zeek-log) (read-line (zeek-stream zeek-log) nil) (zeek-status zeek-log) :unparsed (zeek-modified? zeek-log) nil) @@ -139,6 +189,50 @@ (mapcar (lambda (s) (close s :abort ,abort?)) ,streams) (close (zeek-stream ,log) :abort ,abort?)))))) +(defun %split-tab-to-vector (line) + "Split LINE on tabs into simple-vector of strings. +Single-pass tab splitter: collect fields via POSITION, then copy to vector." + (declare (optimize (speed 3) (safety 1)) + (type simple-string line)) + (let ((fields nil) + (nfields 0) + (start 0)) + (declare (type fixnum nfields start)) + (loop + (let ((tab-pos (position #\Tab line :start start))) + (push (subseq line start (or tab-pos (length line))) fields) + (incf nfields) + (unless tab-pos (return)) + (setf start (the fixnum (1+ tab-pos))))) + (let ((result (make-array nfields))) + (loop for i from (1- nfields) downto 0 + for f in fields + do (setf (svref result i) f)) + result))) + +(defun compute-offsets (line offsets) + "Scan LINE for tabs, store field boundaries in OFFSETS (pre-allocated). + OFFSETS[0] = 0 (start of field 0). + OFFSETS[i] for i=1..nfields-1 = tab position (exclusive end of field i-1). + OFFSETS[nfields] = (length line) (exclusive end of last field). + Field 0: (subseq line 0 (aref offsets 1)). + Field i>0: (subseq line (1+ (aref offsets i)) (aref offsets (1+ i)))." + (declare (optimize (speed 3) (safety 1)) + (type simple-string line) + (type (simple-array fixnum (*)) offsets)) + (let ((nfields (1- (length offsets)))) + (declare (type fixnum nfields)) + (setf (aref offsets 0) 0) + (loop with start fixnum = 0 + for i fixnum from 1 below nfields + for tab-pos = (position #\Tab line :start start) + while tab-pos + do (setf (aref offsets i) (the fixnum tab-pos) + start (the fixnum (1+ tab-pos))) + finally (loop for j fixnum from i to nfields + do (setf (aref offsets j) (length line)))) + (setf (aref offsets nfields) (length line)))) + ;; TODO: if FIELDS is non-NIL, only parse those fields. ;; just make ROW-STRINGS only as long as the number of fields you have ;; then ENSURE-ROW will just work. @@ -147,11 +241,16 @@ ;; for the delimiters and it would only be noticeably faster if there aren't a ;; lot of fields and they're early on in the line. (defun ensure-row-strings (zeek-log &optional fields) + (declare (optimize (speed 3) (debug 0) (space 0) (compilation-speed 0)) + (type zeek zeek-log)) (when (eq :unparsed (zeek-status zeek-log)) (if fields (error "Parsing of individual fields not implemented.") - (setf (zeek-row-strings zeek-log) (coerce (split-sequence #\Tab (zeek-line zeek-log)) 'vector) - (zeek-status zeek-log) :row-strings)))) + (let ((line (zeek-line zeek-log))) + (declare (type simple-string line)) + (compute-offsets line (zeek-offsets zeek-log)) + (fill (zeek-row-strings zeek-log) nil) + (setf (zeek-status zeek-log) :row-strings))))) ;; TODO: if FIELDS is non-NIL, only parse those fields. ;; TODO: you should do a quick check to see if only parsing the needed fields @@ -161,11 +260,17 @@ (ensure-row-strings zeek-log fields) (if fields (error "Parsing of individual fields not implemented.") - (setf (zeek-row zeek-log) - (coerce (loop for type in (zeek-types zeek-log) - for field across (zeek-row-strings zeek-log) - collect (parse-zeek-type field type)) 'vector) - (zeek-status zeek-log) :row)))) + (let ((offsets (zeek-offsets zeek-log)) + (line (zeek-line zeek-log))) + (setf (zeek-row zeek-log) + (coerce (loop for type in (zeek-types zeek-log) + for idx from 0 + for field = (or (svref (zeek-row-strings zeek-log) idx) + (subseq line + (if (zerop idx) (aref offsets idx) (1+ (aref offsets idx))) + (aref offsets (1+ idx)))) + collect (parse-zeek-type field type)) 'vector) + (zeek-status zeek-log) :row))))) (defun ensure-fields->idx (zeek-log) (when (zerop (hash-table-count (zeek-field->idx zeek-log))) @@ -176,9 +281,15 @@ (gethash field (zeek-field->type zeek-log)) type)))) (defun field->idx (zeek-log field) - (gethash field (zeek-field->idx zeek-log))) + (declare (optimize (speed 3) (debug 0) (space 0) (compilation-speed 0)) + (type zeek zeek-log) + (type keyword field)) + (the (or null fixnum) (gethash field (zeek-field->idx zeek-log)))) (defun field->type (zeek-log field) + (declare (optimize (speed 3) (debug 0) (space 0) (compilation-speed 0)) + (type zeek zeek-log) + (type keyword field)) (gethash field (zeek-field->type zeek-log))) ;; TODO: i probably don't need this anymore. just something that @@ -187,13 +298,26 @@ (when (member (zeek-status zeek-log) '(:unparsed :row-strings)) (ecase (zeek-format zeek-log) (:zeek - (loop for field in (if (eq :unparsed (zeek-status zeek-log)) - (split-sequence *zeek-field-separator* (zeek-line zeek-log)) - (coerce (zeek-row-strings zeek-log) 'list)) - for name in (zeek-fields zeek-log) - for type in (zeek-types zeek-log) - do (setf (gethash name (zeek-map zeek-log)) (parse-zeek-type field type) - (zeek-status zeek-log) :zeek-map))) + (if (eq :unparsed (zeek-status zeek-log)) + ;; Unparsed: split directly from line (no offsets computed yet) + (loop for field in (split-sequence *zeek-field-separator* (zeek-line zeek-log)) + for name in (zeek-fields zeek-log) + for type in (zeek-types zeek-log) + do (setf (gethash name (zeek-map zeek-log)) (parse-zeek-type field type) + (zeek-status zeek-log) :zeek-map)) + ;; Row-strings: use cached values (may include mutations), fallback to offsets + (let ((offsets (zeek-offsets zeek-log)) + (line (zeek-line zeek-log)) + (row-strings (zeek-row-strings zeek-log))) + (loop for name in (zeek-fields zeek-log) + for type in (zeek-types zeek-log) + for idx from 0 + for field = (or (svref row-strings idx) + (subseq line + (if (zerop idx) (aref offsets idx) (1+ (aref offsets idx))) + (aref offsets (1+ idx)))) + do (setf (gethash name (zeek-map zeek-log)) (parse-zeek-type field type) + (zeek-status zeek-log) :zeek-map))))) (:json (restart-case (progn (clrhash (zeek-map zeek-log)) (setf (zeek-map zeek-log) (jzon:parse (zeek-line zeek-log) :key-fn #'string->keyword) @@ -246,6 +370,9 @@ (format stream "~a" (generate-zeek-header zeek-log))))))) (defun write-zeek-log-line (zeek-log stream format) + (declare (optimize (speed 3) (debug 0) (space 0) (compilation-speed 0)) + (type zeek zeek-log) + (type keyword format)) (let ((same-format? (eq format (zeek-format zeek-log)))) (cond ((and (not (zeek-modified? zeek-log)) same-format?) (write-line (zeek-line zeek-log) stream)) @@ -265,11 +392,22 @@ (:json (jzon:stringify (if (eq (zeek-status zeek-log) :zeek-map) (jsonify-zeek-map (zeek-map zeek-log)) (zeek-map zeek-log)) :stream stream) (terpri stream)) - (:zeek (ecase (zeek-status zeek-log) - (:row-strings (loop for i from 1 for field across (zeek-row-strings zeek-log) - do (princ field stream) - when (< i (length (zeek-row-strings zeek-log))) - do (princ *zeek-field-separator* stream)) + (:zeek (ecase (zeek-status zeek-log) + (:row-strings + (let ((row (zeek-row-strings zeek-log)) + (offsets (zeek-offsets zeek-log)) + (line (zeek-line zeek-log)) + (nfields (length (zeek-row-strings zeek-log)))) + (loop for i from 0 below nfields + when (plusp i) do (write-char *zeek-field-separator* stream) + do (let ((val (svref row i))) + (if val + (write-string val stream) + (write-string line stream + :start (if (zerop i) + (aref offsets i) + (1+ (aref offsets i))) + :end (aref offsets (1+ i))))))) (when (zeek-created-fields zeek-log) (princ *zeek-field-separator* stream) (loop for i from 1 for created-field in (zeek-created-fields zeek-log) diff --git a/main.lisp b/main.lisp index 491dfad..e937a57 100644 --- a/main.lisp +++ b/main.lisp @@ -34,7 +34,9 @@ (defun ensure-fully-parsed-non-nil-func (full-columns) (when full-columns (let ((filters (cons 'and (mapcar (lambda (c) `(get-value log ,(or-nickname c) t)) full-columns)))) - (values (compile nil `(lambda (log) (declare (ignorable log)) + (values (compile nil `(lambda (log) (declare (optimize (speed 3) (debug 0) (space 0) (compilation-speed 0)) + (ignorable log) + (type zeek log)) ;; duplication of filter RESTART-CASE is a bit annoying. (restart-case ,filters (drop-line () :report (lambda (stream) @@ -43,6 +45,41 @@ filters)))) (defun cat-logs-string (output-file output-format mutator-expr filter-expr &rest input-files) + ;; Fast path: no filter, no mutator, same format → simple line copy + (when (and (null mutator-expr) (null filter-expr) + (not *debug-compiled-functions*) + (eq output-format :input-format)) + (with-open-file (out output-file :direction :output :if-exists :supersede) + (when (zerop (length input-files)) + (push "/dev/stdin" input-files)) + (let ((first-file t) + (detected-format nil)) + (dolist (file input-files) + (with-open-file (in file) + (when (and (null detected-format) (listen in)) + (setf detected-format (infer-format in))) + (if (eq detected-format :json) + ;; JSON: just copy all lines + (loop for line = (read-line in nil) + while line + do (write-line line out)) + ;; Zeek: first file writes header, subsequent files skip headers + (loop for line = (read-line in nil) + while line + do (cond ((str:starts-with? "#close" line) + ;; Skip #close markers from individual files + nil) + ((and (not first-file) (char= #\# (char line 0))) + ;; Skip header lines from subsequent files + nil) + (t (write-line line out))))) + (setf first-file nil))) + (when (eq detected-format :zeek) + (format out (format nil "#close~a~~a~%" *zeek-field-separator*) + (timestamp-to-zeek-open-close-string (local-time:now)))))) + (return-from cat-logs-string)) + + ;; Normal path (existing code continues) (multiple-value-bind (mutator-func mutator-columns mutator-full-columns mutators) (compile-runtime-mutators mutator-expr) (multiple-value-bind (filter-func filter-columns filter-full-columns filters) (compile-runtime-filters filter-expr) (multiple-value-bind (ensure-truthy-parse-func ensure-filters) @@ -174,7 +211,9 @@ (let ((raw-filters (with-input-from-string (in s) (read in nil)))) (multiple-value-bind (filters columns full-columns) (expand-columns raw-filters) - (values (compile nil `(lambda (log) (declare (ignorable log)) + (values (compile nil `(lambda (log) (declare (optimize (speed 3) (debug 0) (space 0) (compilation-speed 0)) + (ignorable log) + (type zeek log)) (restart-case ,(macroexpand-1 filters) (drop-line () :report (lambda (stream) (format stream "DROP-LINE: \"~a\"" (zeek-line log))) nil) @@ -222,7 +261,10 @@ Primarily used anonymize IPs and hash fields with ANONIP and HASH." (let* ((raw-mutators (with-input-from-string (in s) (read-all-progn in)))) (multiple-value-bind (mutators columns full-columns) (expand-columns (update-setters raw-mutators)) - (values (compile nil `(lambda (log) (declare (ignorable log)) ,(macroexpand-1 mutators))) + (values (compile nil `(lambda (log) (declare (optimize (speed 3) (debug 0) (space 0) (compilation-speed 0)) + (ignorable log) + (type zeek log)) + ,(macroexpand-1 mutators))) columns full-columns mutators)))) @@ -265,7 +307,13 @@ Primarily used anonymize IPs and hash fields with ANONIP and HASH." :short-name #\d :long-name "debug-compiled-functions" :initial-value nil - :key :debug-compiled-functions))) + :key :debug-compiled-functions) + (clingon:make-option :boolean/true + :description "Non-interactive mode (drop all lines with errors)" + :short-name #\n + :long-name "non-interactive-mode" + :initial-value nil + :key :non-interactive-mode))) (defun cat/handler (cmd) (in-package :cleek) @@ -276,9 +324,14 @@ Primarily used anonymize IPs and hash fields with ANONIP and HASH." (format (string->keyword (clingon:getopt cmd :format))) (filter-expr (clingon:getopt cmd :filter-expr)) (mutator-expr (clingon:getopt cmd :mutator-expr)) - (*debug-compiled-functions* (clingon:getopt cmd :debug-compiled-functions))) + (*debug-compiled-functions* (clingon:getopt cmd :debug-compiled-functions)) + (non-interactive-mode? (clingon:getopt cmd :non-interactive-mode))) (init-common-filters-and-mutators) - (handler-bind ((error (lambda (condition) (invoke-debugger condition)))) + (handler-bind ((error (lambda (condition) (if non-interactive-mode? + (if (find-restart 'drop-line condition) + (invoke-restart 'drop-line) + (format *standard-error* "Failed to proceed automatically. Exiting...~%")) + (invoke-debugger condition))))) (handler-case (apply #'cat-logs-string output-file format mutator-expr filter-expr args) ;; Still doesn't build on ECL due to not being able to find ASDF/SYSTEM::FIND-SYSTEM even if i add "asdf" to the @@ -298,7 +351,7 @@ Primarily used anonymize IPs and hash fields with ANONIP and HASH." (defun cat/command () (clingon:make-command :name "cleek" - :version "0.14.1" + :version "0.17.0" :usage "[ZEEK-LOG]..." :description "Concatenate, filter, and convert Zeek logs" :handler #'cat/handler diff --git a/types.lisp b/types.lisp index 6bae029..401df1d 100644 --- a/types.lisp +++ b/types.lisp @@ -9,6 +9,7 @@ line types fields + offsets row-strings row (field->idx (make-hash-table) :type hash-table) @@ -28,6 +29,55 @@ (na:enable-ip-syntax) (cl-interpol:enable-interpol-syntax) +(declaim (ftype (function (simple-string &key (:start fixnum) (:end fixnum)) double-float) fast-parse-double)) +(defun fast-parse-double (string &key (start 0) (end (length string))) + "Parse STRING within [START, END) to double-float. Handles [-]digits[.digits][e|E[-|+]digits]. +Avoids the full CL reader overhead of READ-FROM-STRING by doing direct +character-level parsing with rational arithmetic for exact rounding." + (declare (optimize (speed 3) (safety 0)) + (type simple-string string) + (type fixnum start end)) + (let ((pos start) + (neg nil) + (significand 0) + (n-frac 0) + (exp-part 0) + (exp-neg nil)) + (declare (fixnum pos n-frac exp-part)) + ;; Sign + (when (and (< pos end) (char= (schar string pos) #\-)) + (setf neg t) + (incf pos)) + ;; Integer part + (loop while (and (< pos end) (digit-char-p (schar string pos))) + do (setf significand (+ (* significand 10) (- (char-code (schar string pos)) 48))) + (incf pos)) + ;; Fractional part — keep accumulating into significand, count digits + (when (and (< pos end) (char= (schar string pos) #\.)) + (incf pos) + (loop while (and (< pos end) (digit-char-p (schar string pos))) + do (setf significand (+ (* significand 10) (- (char-code (schar string pos)) 48))) + (incf n-frac) + (incf pos))) + ;; Exponent + (when (and (< pos end) (or (char= (schar string pos) #\e) (char= (schar string pos) #\E))) + (incf pos) + (when (and (< pos end) (or (char= (schar string pos) #\-) (char= (schar string pos) #\+))) + (when (char= (schar string pos) #\-) + (setf exp-neg t)) + (incf pos)) + (loop while (and (< pos end) (digit-char-p (schar string pos))) + do (setf exp-part (+ (* exp-part 10) (- (char-code (schar string pos)) 48))) + (incf pos))) + ;; Combine using rational arithmetic: significand * 10^(exp - n-frac) + ;; This ensures exact rounding identical to READ-FROM-STRING. + (let* ((effective-exp (- (if exp-neg (- exp-part) exp-part) n-frac)) + (rational-val (if (>= effective-exp 0) + (* significand (expt 10 effective-exp)) + (/ significand (expt 10 (- effective-exp))))) + (result (coerce rational-val 'double-float))) + (if neg (- result) result)))) + (defvar *zeek-set-separator* #\,) (defvar *zeek-unset-field* #\-) (defvar *zeek-empty-field* "(empty)") @@ -57,19 +107,44 @@ (local-time:unix-to-timestamp secs :nsec (* (expt 10 6) (truncate nsecs))))) ;; type conversions between zeek, JSON, and lisp. -(defparameter *zeek-primitive-type-parsers* - `((:bool . ,(lambda (x) (if (string= "T" x) t nil))) - (:count . ,#'parse-integer) - (:int . ,#'parse-integer) - (:double . ,#'read-from-string) - (:time . ,#'zeek-ts-string-to-timestamp) - (:interval . ,#'read-from-string) - (:string . ,#'identity) ; or #'string maybe? - (:port . ,#'parse-integer) - (:addr . ,#'na:make-ip-address) - (:subnet . ,#'na:make-ip-network) - (:enum . ,#'identity) ; just keep it as a string i suppose - )) +;; Hash table for O(1) type→parser dispatch (replaces alist *zeek-primitive-type-parsers*). +;; Includes both primitive types and compound types (set[...], vector[...]). +;; Compound types are lazily cached on first access. +(defparameter *zeek-type-parsers* (make-hash-table :test 'eq)) + +(defun %init-zeek-type-parsers () + "Populate *zeek-type-parsers* with primitive type parsers." + (clrhash *zeek-type-parsers*) + (loop for (type . parser) in + `((:bool . ,(lambda (x) (if (string= "T" x) t nil))) + (:count . ,#'parse-integer) + (:int . ,#'parse-integer) + (:double . ,#'fast-parse-double) + (:time . ,#'zeek-ts-string-to-timestamp) + (:interval . ,#'fast-parse-double) + (:string . ,#'identity) + (:port . ,#'parse-integer) + (:addr . ,#'na:make-ip-address) + (:subnet . ,#'na:make-ip-network) + (:enum . ,#'identity)) + do (setf (gethash type *zeek-type-parsers*) parser))) + +(%init-zeek-type-parsers) + +(defun %get-compound-parser (type) + "For compound types like :SET[ADDR] or :VECTOR[COUNT], extract the element +type, look up its parser, and return a closure that splits on separator and +maps the element parser. Caches the result in *zeek-type-parsers*." + (let* ((type-string (keyword->string type))) + (cl-ppcre:register-groups-bind (nil primitive-type-str) + ("(set|vector)\\[(.*?)\\]" type-string) + (let ((element-parser (gethash (string->keyword primitive-type-str) *zeek-type-parsers*))) + (when element-parser + (let ((parser (lambda (field) + (map 'vector (lambda (f) (funcall element-parser f)) + (split-sequence *zeek-set-separator* field))))) + (setf (gethash type *zeek-type-parsers*) parser) + parser)))))) ;; TODO: think about this more. For port, 0 makes sense since it's reserved and should never actually be used. but for ;; something like RCODE, which is a count, 0 is a valid value. I suppose we're typically only doing math on these things @@ -80,30 +155,39 @@ ((:double :interval :time) 0.0d0) (t 'cl:null))) +(defun %make-type-hash-table (entries) + "Create eq hash-table from alist entries." + (let ((ht (make-hash-table :test 'eq))) + (loop for (type . fn) in entries do (setf (gethash type ht) fn)) + ht)) + (defparameter *zeek-jsonify* - `((:bool . ,(lambda (x) (if x "T" "F"))) - (:time . ,#'timestamp-to-zeek-ts) - (:addr . ,(lambda (x) (str:downcase (na:str x)))) - (:subnet . ,(lambda (x) (str:downcase (na:str x)))))) + (%make-type-hash-table + `((:bool . ,(lambda (x) (if x "T" "F"))) + (:time . ,#'timestamp-to-zeek-ts) + (:addr . ,(lambda (x) (str:downcase (na:str x)))) + (:subnet . ,(lambda (x) (str:downcase (na:str x))))))) (defparameter *json-zeekify* - `((:bool . ,(lambda (x) (string= x "T"))) - (:time . ,#'double-to-timestamp) - (:addr . ,#'na:make-ip-address) - (:subnet . ,#'na:make-ip-network))) + (%make-type-hash-table + `((:bool . ,(lambda (x) (string= x "T"))) + (:time . ,#'double-to-timestamp) + (:addr . ,#'na:make-ip-address) + (:subnet . ,#'na:make-ip-network)))) (defparameter *zeek-stringify* - `((:bool . ,(lambda (x) (if x "T" "F"))) - (:count . ,#'write-to-string) - (:int . ,#'write-to-string) - (:double . ,(lambda (x) (format nil "~,6f" x))) - (:time . ,#'timestamp-to-zeek-ts-string) - (:interval . ,(lambda (x) (format nil "~,6f" x))) - (:string . ,#'identity) - (:port . ,#'string) - (:addr . ,(lambda (x) (str:downcase (na:str x)))) - (:subnet . ,(lambda (x) (str:downcase (na:str x)))) - (:enum . ,#'identity))) + (%make-type-hash-table + `((:bool . ,(lambda (x) (if x "T" "F"))) + (:count . ,#'write-to-string) + (:int . ,#'write-to-string) + (:double . ,(lambda (x) (format nil "~,6f" x))) + (:time . ,#'timestamp-to-zeek-ts-string) + (:interval . ,(lambda (x) (format nil "~,6f" x))) + (:string . ,#'identity) + (:port . ,#'string) + (:addr . ,(lambda (x) (str:downcase (na:str x)))) + (:subnet . ,(lambda (x) (str:downcase (na:str x)))) + (:enum . ,#'identity)))) ;; TODO: make these learnable, i.e., provide a bunch of zeek logs, output a file of these defparameter calls and load ;; the file if it's present. as long as you fully search the data structure it's fine to have duplicate "keys" in the @@ -137,9 +221,12 @@ )) (defparameter *field->type* - (remove-duplicates (loop for (nil . fields) in *path->fields* - for (nil . types) in *path->types* append - (loop for field in fields for type in types collect (cons field type))) :test #'equal)) + (let ((ht (make-hash-table :test 'eq))) + (loop for (nil . fields) in *path->fields* + for (nil . types) in *path->types* + do (loop for field in fields for type in types + do (setf (gethash field ht) type))) + ht)) (defun infer-log-path-fields-types (zeek-log) (unless (zeek-path zeek-log) @@ -163,74 +250,81 @@ ;; newly created field) but what if we need to filter fields that are going to be fully parsed and used as part of a ;; mutator? hmm. multiple rounds? interleaving sounds annoying though. (defun parse-zeek-type (field type &optional unset-is-nil?) - (let ((type-string (keyword->string type))) - ;; unset should probably be 'null and empty should probably be #() (since '() is eq to nil) - (cond ((string= field (string *zeek-unset-field*)) (if unset-is-nil? nil 'cl::null)) - ((string= field (string *zeek-empty-field*)) - (if (eq type :string) "" #())) - ((or (str:starts-with? "set" type-string) - (str:starts-with? "vector" type-string)) - (cl-ppcre:register-groups-bind (nil primitive-type) - ("(set|vector)\\[(.*?)\\]" type-string) - (map 'vector (lambda (f) (parse-zeek-type f (string->keyword primitive-type))) - (split-sequence *zeek-set-separator* field)))) - (t (funcall (ax:assoc-value *zeek-primitive-type-parsers* type) field))))) + (declare (optimize (speed 3) (debug 0) (space 0) (compilation-speed 0)) + (type string field) + (type keyword type)) + ;; unset should probably be 'null and empty should probably be #() (since '() is eq to nil) + (cond ((string= field (string *zeek-unset-field*)) (if unset-is-nil? nil 'cl::null)) + ((string= field (string *zeek-empty-field*)) + (if (eq type :string) "" #())) + (t (let ((parser (or (gethash type *zeek-type-parsers*) + (%get-compound-parser type)))) + (if parser + (funcall parser field) + (error "Unknown Zeek type: ~a" type)))))) (defun unparse-zeek-type (field type) - (let ((type-string (keyword->string type))) - (cond ((eq 'cl:null field) "-") ;; - ((stringp field) - field) - ((or (str:starts-with? "set" type-string) - (str:starts-with? "vector" type-string)) - (cl-ppcre:register-groups-bind (nil primitive-type) - ("(set|vector)\\[(.*?)\\]" type-string) - (str:join *zeek-set-separator* - (map 'list (lambda (f) (unparse-zeek-type f (string->keyword primitive-type))) - field)))) - (t (funcall (ax:assoc-value *zeek-stringify* type) field))))) + (cond ((eq 'cl:null field) "-") + ((stringp field) + field) + (t (multiple-value-bind (func found) (gethash type *zeek-stringify*) + (if found + (funcall func field) + ;; compound type — extract element type once + (let ((type-string (keyword->string type))) + (cl-ppcre:register-groups-bind (nil primitive-type) + ("(set|vector)\\[(.*?)\\]" type-string) + (str:join *zeek-set-separator* + (map 'list (lambda (f) (unparse-zeek-type f (string->keyword primitive-type))) + field))))))))) (defun jsonify-zeek-type (field type) - (let ((type-string (keyword->string type))) - (cond ((stringp field) - field) - ((or (str:starts-with? "set" type-string) - (str:starts-with? "vector" type-string)) - (cl-ppcre:register-groups-bind (nil primitive-type) - ("(set|vector)\\[(.*?)\\]" type-string) - (map 'vector (lambda (f) (jsonify-zeek-type f (string->keyword primitive-type))) - field))) - (t (ax:if-let ((func (ax:assoc-value *zeek-jsonify* type))) - (funcall func field) - field))))) + (cond ((stringp field) + field) + (t (multiple-value-bind (func found) (gethash type *zeek-jsonify*) + (if found + (funcall func field) + ;; might be compound or just pass-through + (let ((type-string (keyword->string type))) + (if (or (str:starts-with? "set" type-string) + (str:starts-with? "vector" type-string)) + (cl-ppcre:register-groups-bind (nil primitive-type) + ("(set|vector)\\[(.*?)\\]" type-string) + (map 'vector (lambda (f) (jsonify-zeek-type f (string->keyword primitive-type))) + field)) + field))))))) (defun jsonify-zeek-map (zeek-map) (loop for field being the hash-key of zeek-map - do (ax:when-let ((type (ax:assoc-value *field->type* field))) - ;; Zeek format uses "-" to indicate unset while JSON simply has the key not present in the output. We use - ;; CL:NULL in Zeek formatted logs to differentiate this from the string "-", so we remove these from the - ;; map when JSONifying a :ZEEK-MAP typed ZEEK-MAP. - (if (eq 'cl:null (gethash field zeek-map)) - (remhash field zeek-map) - (setf (gethash field zeek-map) (jsonify-zeek-type (gethash field zeek-map) type))))) + do (multiple-value-bind (type found) (gethash field *field->type*) + (when found + ;; Zeek format uses "-" to indicate unset while JSON simply has the key not present in the output. We use + ;; CL:NULL in Zeek formatted logs to differentiate this from the string "-", so we remove these from the + ;; map when JSONifying a :ZEEK-MAP typed ZEEK-MAP. + (if (eq 'cl:null (gethash field zeek-map)) + (remhash field zeek-map) + (setf (gethash field zeek-map) (jsonify-zeek-type (gethash field zeek-map) type)))))) zeek-map) (defun zeekify-json-type (field type) - (let ((type-string (keyword->string type))) - (cond ((or (str:starts-with? "set" type-string) - (str:starts-with? "vector" type-string)) - (cl-ppcre:register-groups-bind (nil primitive-type) - ("(set|vector)\\[(.*?)\\]" type-string) - (map 'vector (lambda (f) (zeekify-json-type f (string->keyword primitive-type))) - field))) - (t (ax:if-let ((func (ax:assoc-value *json-zeekify* type))) - (funcall func field) - field))))) + (multiple-value-bind (func found) (gethash type *json-zeekify*) + (if found + (funcall func field) + ;; might be compound or just pass-through + (let ((type-string (keyword->string type))) + (if (or (str:starts-with? "set" type-string) + (str:starts-with? "vector" type-string)) + (cl-ppcre:register-groups-bind (nil primitive-type) + ("(set|vector)\\[(.*?)\\]" type-string) + (map 'vector (lambda (f) (zeekify-json-type f (string->keyword primitive-type))) + field)) + field))))) (defun zeekify-json-map (json-map) (loop for field being the hash-key of json-map - do (ax:when-let ((type (ax:assoc-value *field->type* field))) - (setf (gethash field json-map) (zeekify-json-type (gethash field json-map) type))))) + do (multiple-value-bind (type found) (gethash field *field->type*) + (when found + (setf (gethash field json-map) (zeekify-json-type (gethash field json-map) type)))))) (defun stringify-json-type-to-zeek-string (field type) (etypecase field