Conversation
Signed-off-by: Matthew Rafuse <15745691+mattrafuse@users.noreply.github.com>
* Create and add CaseLog ODM to Case ODM, create_case service * Audit True * Minor fixes * Consolidate updating, without ODM Helper
* tasks ui * pr feedback
* Tests for Cases * Add hiding of cases, update tests * blah
Contributor
Contributor
Howler Evidence Plugin - Coverage Results
Diff CoverageDiff: origin/main...HEAD, staged and unstaged changes
Summary
Full Coverage ReportExpand |
-passing-brightgreen){kind=icon}
Contributor
Howler Sentinel Plugin - Coverage Results
Diff CoverageDiff: origin/main...HEAD, staged and unstaged changes
Summary
plugins/sentinel/sentinel/routes/ingest.pyLines 257-266 257 item_type="hit",
258 item_value=child_id,
259 item_path=child_label,
260 )
! 261 except Exception:
! 262 logger.exception("Failed to add child hit %s to case", child_id)
263
264 datastore().case.commit()
265
266 datastore().hit.commit()Lines 279-285 279 "organization": bundle_hit["organization"]["name"],
280 }
281 return created(response_body)
282 except HowlerException as e:
! 283 logger.exception("Failed to create incident")
! 284 return internal_error(err=f"Failed to create incident: {str(e)}")Full Coverage ReportExpand |
Contributor
Howler API - Coverage Results
Diff CoverageDiff: origin/main...HEAD, staged and unstaged changes
Summary
api/howler/actions/add_to_bundle.pyLines 53-61 53 ## Check hit limit against the query before searching
54 if user:
55 limit_error = check_hit_limit(query, user, MAX_HITS_BASIC, MAX_HITS_ADVANCED)
56 if limit_error:
! 57 return [limit_error]
58
59 matching_hits = ds.hit.search(query, rows=MAX_HITS_ADVANCED)["items"]
60
61 if not matching_hits:Lines 66-74 66 "title": "No Matching Hits",
67 "message": "There were no hits matching this query.",
68 }
69 )
! 70 return report
71
72 added = []
73 skipped = []
74 for hit in matching_hits:Lines 80-89 80 item_value=hit.howler.id,
81 item_path=child_label,
82 )
83 added.append(hit.howler.id)
! 84 except Exception:
! 85 skipped.append(hit.howler.id)
86
87 if skipped:
88 report.append(
89 {Lines 103-111 103 "message": "The specified bundle has had all matching hits added.",
104 }
105 )
106
! 107 except NotFoundException as e:
108 report.append(
109 {
110 "query": query,
111 "outcome": "error",api/howler/actions/add_to_case.pyLines 81-92 81 )
82 added.append(hit.howler.id)
83 except InvalidDataException as e:
84 skipped.append(f"{hit.howler.id}: {e}")
! 85 except NotFoundException as e:
! 86 skipped.append(f"{hit.howler.id}: {e}")
! 87 except Exception as e:
! 88 skipped.append(f"{hit.howler.id}: {e}")
89
90 if added:
91 report.append(
92 {api/howler/actions/remove_from_bundle.pyLines 53-61 53 ## Check hit limit against the query before searching
54 if user:
55 limit_error = check_hit_limit(query, user, MAX_HITS_BASIC, MAX_HITS_ADVANCED)
56 if limit_error:
! 57 return [limit_error]
58
59 matching_hits = ds.hit.search(query, rows=MAX_HITS_ADVANCED)["items"]
60
61 if not matching_hits:Lines 66-79 66 "title": "No Matching Hits",
67 "message": "There were no hits matching this query.",
68 }
69 )
! 70 return report
71
72 ## Get the case to check which hits are actually in it
73 case = ds.case.get(case_id)
74 if case is None:
! 75 report.append(
76 {
77 "query": query,
78 "outcome": "error",
79 "title": "Case Not Found",Lines 79-87 79 "title": "Case Not Found",
80 "message": f"Associated case {case_id} no longer exists.",
81 }
82 )
! 83 return report
84
85 case_item_values = {item.value for item in case.items}
86 values_to_remove = [h.howler.id for h in matching_hits if h.howler.id in case_item_values]
87 skipped_ids = [h.howler.id for h in matching_hits if h.howler.id not in case_item_values]Lines 96-104 96 }
97 )
98
99 if not values_to_remove:
! 100 report.append(
101 {
102 "query": query,
103 "outcome": "skipped",
104 "title": "No Matching Hits",Lines 117-126 117 "message": f"Matching hits removed from bundle with id {bundle_id}",
118 }
119 )
120
! 121 except NotFoundException as e:
! 122 report.append(
123 {
124 "query": query,
125 "outcome": "error",
126 "title": "Failed to Execute",Lines 126-134 126 "title": "Failed to Execute",
127 "message": str(e),
128 }
129 )
! 130 except Exception as e:
131 report.append(
132 {
133 "query": query,
134 "outcome": "error",api/howler/api/socket.pyLines 63-71 63
64 Result Example:
65 ["user1", "user2"]
66 """
! 67 return ok(viewer_service.get_viewers(entity_id))
68
69
70 @tracer.start_as_current_span(f"{__name__}.connect")
71 @socket_api.route("/connect", websocket=True) ## type: ignoreLines 96-109 96 logger.debug("Sending action: %s", data)
97 ws.send(ws_response("action", data))
98
99 def send_case(data: dict[str, Any]):
! 100 logger.debug("Sending case update: %s", data.get("case", {}).get("case_id", "unknown"))
! 101 ws.send(ws_response("cases", data))
102
103 def send_viewers_update(data: dict[str, Any]):
! 104 logger.debug("Sending viewers update: %s", data.get("id", "unknown"))
! 105 ws.send(ws_response("viewers_update", data))
106
107 try:
108 event_service.on("hits", send_hit)
109 event_service.on("broadcast", send_broadcast)api/howler/api/v1/action.pyLines 238-246 238 execute_req = request.json
239 if not isinstance(execute_req, dict):
240 return bad_request(err="Incorrect data structure!")
241
! 242 action = datastore().action.get(id)
243 if not action:
244 return not_found(err="The specified action does not exist")
245
246 reports: dict[str, list[dict]] = {}api/howler/api/v1/analytic.pyLines 493-501 493 if not existing_analytic:
494 return not_found(err="This analytic does not exist")
495
496 if not user:
! 497 return forbidden(err="User was not found.")
498
499 try:
500 user.favourite_analytics.append(id)Lines 528-536 528 if not storage.analytic.exists(id):
529 return not_found(err="This analytic does not exist")
530
531 if not user:
! 532 return forbidden(err="User was not found.")
533
534 try:
535 user.favourite_analytics = list(filter(lambda f: f != id, user.favourite_analytics))api/howler/api/v2/case.pyLines 50-62 50
51 try:
52 return created(case_service.create_case(case_data, user.uname))
53 except InvalidDataException as e:
! 54 return bad_request(err=str(e))
55 except ResourceExists as e:
56 return bad_request(err=str(e))
! 57 except HowlerException as e:
! 58 return bad_request(err=str(e))
59
60
61 @generate_swagger_docs()
62 @case_api.route("/<id>", methods=["GET"])api/howler/api/v2/ingest.pyLines 170-178 170 break
171
172 existing = [record_id for record_id in remaining if ds[index].exists(record_id)]
173 if not existing:
! 174 continue
175
176 for record_id in existing:
177 ds[index].delete(record_id)Lines 307-316 307
308 new_record, new_version = ds[index].get(id, as_obj=False, version=True)
309
310 return ok(new_record), new_version
! 311 except HowlerValueError as e:
! 312 return bad_request(err=e.message)
313
314
315 @generate_swagger_docs()
316 @ingest_api.route("/<indexes>/update", methods=["PUT"])Lines 366-372 366
367 return ok({"success": all(ds[index].update_by_query(query, operations) for index in indexes.split(","))})
368 except (HowlerValueError, KeyError, DataStoreException) as e:
369 return bad_request(err=str(e))
! 370 except Exception as e:
! 371 return internal_error(err=str(e))api/howler/api/v2/search.pyLines 200-210 200 for explanation in result["explanations"]:
201 del explanation["index"]
202
203 return ok(result)
! 204 except Exception as e:
! 205 logger.exception("Exception on query explanation")
! 206 return bad_request(err=f"Exception: {e}")
207
208
209 @generate_swagger_docs()
210 @search_api.route("/count/<index>", methods=["GET", "POST"])Lines 321-329 321 if collection is None:
322 return bad_request(err=f"Not a valid index to search in: {index}")
323
324 if has_access_control(index):
! 325 params.update({"access_control": user["access_control"]})
326 for field in fields:
327 facet_result.setdefault(field, {})
328
329 if field not in collection().fields():Lines 332-339 332
333 facet_result[field].update(collection().facet(field, **params))
334
335 return ok(facet_result)
! 336 except (SearchException, BadRequestError) as e:
! 337 logger.error("SearchException: %s", str(e), exc_info=True)
! 338 return bad_request(err=f"SearchException: {e}")api/howler/cronjobs/correlation.pyLines 21-30 21 """Start the correlation worker thread if correlation is enabled."""
22 global _thread
23
24 if not config.system.correlation.enabled:
! 25 logger.info("Correlation worker disabled by configuration")
! 26 return
27
28 if _thread is not None and _thread.is_alive():
29 logger.debug("Correlation worker thread already running")
30 returnapi/howler/helper/discover.pyLines 36-45 36 if resp.ok:
37 data = resp.json()
38 for app in data["applications"]["application"]:
39 try:
! 40 app_url = app["instance"][0]["hostName"]
! 41 if "howler" not in app_url:
42 apps.append(
43 {
44 "alt": app["instance"][0]["metadata"]["alternateText"],
45 "name": app["name"],api/howler/helper/hit.pyLines 228-236 228 Status.IN_PROGRESS,
229 Status.OPEN,
230 ]:
231 actions.append(odm_helper.update("howler.assignment", "unassigned"))
! 232 actions.append(odm_helper.update("howler.status", Status.OPEN))
233 else:
234 raise InvalidDataException("Cannot vote on hit you are assigned to.")
235
236 return actionsapi/howler/helper/search.pyLines 77-85 77 bool: Does the index have access control
78 """
79 if isinstance(index, str):
80 if "," in index:
! 81 index = index.split(",")
82 else:
83 index = [index]
84
85 return any(_index in ACCESS_CONTROLLED_INDICES for _index in index)api/howler/odm/mixins.pyLines 52-60 52 f"Use '{type(obj).__name__}.store' instead."
53 )
54
55 if objtype is None:
! 56 raise HowlerRuntimeError("Cannot resolve owner class for 'store' descriptor.")
57
58 index_name = objtype.__name__.lower()
59 return datastore()[index_name]api/howler/services/bundle_compat_service.pyLines 124-133 124 item_type="hit",
125 item_value=child_id,
126 item_path=child_label,
127 )
! 128 except (InvalidDataException, NotFoundException, DataStoreException) as exc:
! 129 logger.warning("Could not add child hit %s to case: %s", child_id, exc)
130
131 datastore().hit.commit()
132 datastore().case.commit()Lines 132-140 132 datastore().case.commit()
133
134 updated_case: Case | None = datastore().case.get(case.case_id)
135 if updated_case is None:
! 136 raise NotFoundException(f"Case {case.case_id} disappeared after creation")
137
138 return synthesize_bundle_response(updated_case, odm, warnings=warnings)
139 Lines 172-180 172
173 ## Check for duplicates and nested bundles before modifying
174 current_case: Case | None = datastore().case.get(case_id)
175 if current_case is None:
! 176 raise NotFoundException(f"Case {case_id} not found")
177
178 existing_values = {item.value for item in current_case.items}
179
180 for hit_id in hit_ids:Lines 187-196 187
188 for hit_id in hit_ids:
189 child_hit = hit_service.get_hit(hit_id, as_odm=True)
190 if child_hit is None:
! 191 logger.warning("Hit %s does not exist, skipping", hit_id)
! 192 continue
193
194 child_label = f"hits/{child_hit.howler.analytic} ({hit_id})"
195 case_service.append_case_item(
196 case_id,Lines 200-208 200 )
201
202 updated_case: Case | None = datastore().case.get(case_id)
203 if updated_case is None:
! 204 raise NotFoundException(f"Case {case_id} not found")
205
206 return synthesize_bundle_response(updated_case, root_hit)
207 Lines 222-230 222 raise InvalidDataException("The specified hit must be a bundle.")
223
224 case: Case | None = datastore().case.get(case_id)
225 if case is None:
! 226 raise NotFoundException(f"Case {case_id} not found")
227
228 if hit_ids == ["*"]:
229 values_to_remove = [item.value for item in case.items if item.value != bundle_id]
230 else:Lines 239-247 239 case_service.remove_case_items(case_id, values_to_remove)
240
241 updated_case: Case | None = datastore().case.get(case_id)
242 if updated_case is None:
! 243 raise NotFoundException(f"Case {case_id} not found")
244
245 return synthesize_bundle_response(updated_case, root_hit)
246 api/howler/services/case_service.pyLines 48-64 48 datastore().case.save(case.case_id, case)
49 CREATED_CASES.inc()
50
51 for item in items:
! 52 append_case_item(case.case_id, item=CaseItem(item))
53
54 if items:
! 55 updated_case = datastore().case.get(case.case_id)
56
! 57 if not updated_case:
! 58 raise HowlerValueError("Error occurred when creating case")
59
! 60 case = updated_case
61
62 event_service.emit("cases", {"case": case.as_primitives()})
63
64 return caseLines 321-332 321 case CaseItemTypes.TABLE:
322 return append_table(case_id, item)
323 case CaseItemTypes.LEAD:
324 return append_lead(case_id, item)
! 325 case CaseItemTypes.REFERENCE:
! 326 return append_reference(case_id, item)
! 327 case _:
! 328 raise InvalidDataException(f"Unsupported item type: {item_type}")
329
330
331 def append_hit(case_id: str, item: CaseItem) -> Case:
332 """Append a hit item to a case and create a back-reference on the hit.Lines 362-370 362
363 _case.items.append(item)
364
365 if not ds.case.save(_case.case_id, _case):
! 366 raise DataStoreException(f"Failed to save {_case.case_id} with new item {item.value}")
367
368 _add_backreference(hit, _case.case_id)
369
370 _sync_case_metadata(_case.case_id)Lines 409-417 409
410 _case.items.append(item)
411
412 if not ds.case.save(_case.case_id, _case):
! 413 raise DataStoreException(f"Failed to save {_case.case_id} with new item {item.value}")
414
415 _add_backreference(observable, _case.case_id)
416 _sync_case_metadata(case_id)Lines 455-463 455
456 _case.items.append(item)
457
458 if not datastore().case.save(_case.case_id, _case):
! 459 raise DataStoreException(f"Failed to save {_case.case_id} with new item {item.value}")
460
461 event_service.emit("cases", {"case": _case.as_primitives()})
462
463 return _caseLines 520-528 520
521 _case.items.append(item)
522
523 if not datastore().case.save(_case.case_id, _case):
! 524 raise DataStoreException(f"Failed to save {_case.case_id} with new item {item.value}")
525
526 event_service.emit("cases", {"case": _case.as_primitives()})
527
528 return _caseLines 561-569 561 for item in _case.items:
562 if item.type == CaseItemTypes.HIT and item.value:
563 hit = ds.hit.get(item.value)
564 if hit is None:
! 565 continue
566
567 indicators.update(_collect_indicators_from_related(hit.related))
568
569 if hit.howler.outline:Lines 632-640 632 if backing_obj is None:
633 raise InvalidDataException("Cannot remove back reference on a nonexisting object")
634
635 if not case_id:
! 636 raise InvalidDataException("Missing back reference case_id")
637
638 if case_id in backing_obj.howler.related:
639 backing_obj.howler.related.remove(case_id)
640 datastore()[backing_obj.__class__.__name__.lower()].save(backing_obj.howler.id, backing_obj)Lines 689-697 689 for item in items_to_remove:
690 _case.items.remove(item)
691
692 if not ds.case.save(_case.case_id, _case):
! 693 raise DataStoreException("Failed to save case after item removal")
694
695 ## Clean up back-references after the case is safely persisted.
696 for backing_obj in backing_objs:
697 remove_backreference(backing_obj, _case.case_id)api/howler/services/correlation_service.pyLines 120-129 120 except InvalidDataException:
121 logger.debug("Record %s already exists in case %s, skipping", record_id, case_id)
122 except NotFoundException:
123 logger.warning("Case %s or record %s not found during correlation", case_id, record_id)
! 124 except Exception:
! 125 logger.exception("Failed to add record %s to case %s", record_id, case_id)
126
127 return added
128 api/howler/services/docs_service.pyLines 55-64 55
56 if blueprint not in api_blueprints:
57 try:
58 doc = current_app.blueprints[rule.endpoint[: rule.endpoint.rindex(".")]]._doc ## type: ignore[attr-defined]
! 59 except Exception:
! 60 doc = ""
61
62 api_blueprints[blueprint] = doc
63
64 if doc_string:Lines 63-71 63
64 if doc_string:
65 description = dedent(doc_string)
66 else:
! 67 description = "[INCOMPLETE]\n\nTHIS API HAS NOT BEEN DOCUMENTED YET!"
68
69 api_id = rule.endpoint.replace(f"api{version}.", "").replace(".", "_")
70
71 api_list.append(api/howler/services/event_service.pyLines 47-56 47
48 for handler in handlers[event_type]:
49 try:
50 handler(payload)
! 51 except Exception:
! 52 logger.exception("Error in event handler for %s", event_type)
53
54
55 def start_watcher():
56 """Start the Redis pubsub watcher that routes incoming events to local handlers.Lines 88-105 88
89 if DEBUG and not _watcher_started:
90 ## In debug/single-process mode without a watcher, call handlers
91 ## directly for immediate feedback.
! 92 if event in handlers:
! 93 logger.debug("event:%s - emitting data (in-process)", event)
! 94 for handler in handlers[event]:
! 95 handler(data)
96
97 ## Always publish to Redis so other pods receive the event.
98 try:
99 _get_sender().send(event, {"__event__": event, "__payload__": data})
! 100 except Exception:
! 101 logger.exception("Failed to publish event %s to Redis", event)
102
103
104 def on(event: str, handler: Callable):
105 """Add a new listener to the specified eventapi/howler/services/observable_service.pyLines 86-94 86 odm_flatten = odm.flat_fields(show_compound=True)
87 unused_keys = extra_keys(Observable, data)
88
89 if unused_keys and not ignore_extra_values:
! 90 raise HowlerValueError(f"Observable was created with invalid parameters: {', '.join(unused_keys)}")
91 deprecated_keys = set(key for key in odm_flatten.keys() & data.keys() if odm_flatten[key].deprecated)
92
93 warnings = [f"{key} is not currently used by howler." for key in unused_keys]
94 warnings.extend(api/howler/services/search_service.pyLines 210-219 210
211 if next_deep_paging_id is not None and len(response["items"]) < rows:
212 try:
213 client.clear_scroll(scroll_id=next_deep_paging_id)
! 214 except elasticsearch.exceptions.NotFoundError:
! 215 pass
216 next_deep_paging_id = None
217
218 if next_deep_paging_id is not None:
219 response["next_deep_paging_id"] = next_deep_paging_idFull Coverage ReportExpand |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
7 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
No description provided.