[WEB-5071] chore: implement webhook logging to MongoDB and fallback to database

apps/api/plane/bgtasks/cleanup_task.py

@@ -410,7 +410,8 @@ def get_webhook_logs_queryset():
             "response_headers",
             "retry_count",
         )
-        .iterator(chunk_size=BATCH_SIZE)
+        .order_by("created_at")
+        .iterator(chunk_size=100)
     )
 
 

apps/api/plane/bgtasks/webhook_task.py

@@ -48,6 +48,8 @@
 )
 from plane.license.utils.instance_value import get_email_configuration
 from plane.utils.exception_logger import log_exception
+from plane.settings.mongo import MongoConnection
+
 
 SERIALIZER_MAPPER = {
     "project": ProjectSerializer,
@@ -84,6 +86,58 @@ def get_issue_prefetches():
     ]
 
 
+
+def save_webhook_log(
+    webhook: Webhook,
+    request_method: str,
+    request_headers: str,
+    request_body: str,
+    response_status: str,
+    response_headers: str,
+    response_body: str,
+    retry_count: int,
+    event_type: str,
+) -> None:
+
+    # webhook_logs
+    mongo_collection = MongoConnection.get_collection("webhook_logs")
+
+    log_data = {
+        "workspace_id": str(webhook.workspace_id),
+        "webhook": str(webhook.id),
+        "event_type": str(event_type),
+        "request_method": str(request_method),
+        "request_headers": str(request_headers),
+        "request_body": str(request_body),
+        "response_status": str(response_status),
+        "response_headers": str(response_headers),
+        "response_body": str(response_body),
+        "retry_count": retry_count,
+    }
+
+    mongo_save_success = False
+    if mongo_collection is not None:
+        try:
+            # insert the log data into the mongo collection
+            mongo_collection.insert_one(log_data)
+            logger.info("Webhook log saved successfully to mongo")
+            mongo_save_success = True
+        except Exception as e:
+            log_exception(e)
+            logger.error(f"Failed to save webhook log: {e}")
+            mongo_save_success = False
+
+    # if the mongo save is not successful, save the log data into the database
+    if not mongo_save_success:
+        try:
+            # insert the log data into the database
+            WebhookLog.objects.create(**log_data)
+            logger.info("Webhook log saved successfully to database")
+        except Exception as e:
+            log_exception(e)
+            logger.error(f"Failed to save webhook log: {e}")
+
+
 def get_model_data(event: str, event_id: Union[str, List[str]], many: bool = False) -> Dict[str, Any]:
     """
     Retrieve and serialize model data based on the event type.
@@ -273,32 +327,30 @@ def webhook_send_task(
         response = requests.post(webhook.url, headers=headers, json=payload, timeout=30)
 
         # Log the webhook request
-        WebhookLog.objects.create(
-            workspace_id=str(webhook.workspace_id),
-            webhook=str(webhook.id),
-            event_type=str(event),
-            request_method=str(action),
-            request_headers=str(headers),
-            request_body=str(payload),
-            response_status=str(response.status_code),
-            response_headers=str(response.headers),
-            response_body=str(response.text),
-            retry_count=str(self.request.retries),
+        save_webhook_log(
+            webhook=webhook,
+            request_method=action,
+            request_headers=headers,
+            request_body=payload,
+            response_status=response.status_code,
+            response_headers=response.headers,
+            response_body=response.text,
+            retry_count=self.request.retries,
+            event_type=event,
         )
         logger.info(f"Webhook {webhook.id} sent successfully")
     except requests.RequestException as e:
         # Log the failed webhook request
-        WebhookLog.objects.create(
-            workspace_id=str(webhook.workspace_id),
-            webhook=str(webhook.id),
-            event_type=str(event),
-            request_method=str(action),
-            request_headers=str(headers),
-            request_body=str(payload),
+        save_webhook_log(
+            webhook=webhook,
+            request_method=action,
+            request_headers=headers,
+            request_body=payload,
             response_status=500,
             response_headers="",
             response_body=str(e),
-            retry_count=str(self.request.retries),
+            retry_count=self.request.retries,
+            event_type=event,
         )
         logger.error(f"Webhook {webhook.id} failed with error: {e}")
         # Retry logic