🏗️ New logs system management

charts/qiskit-serverless/charts/gateway/values.yaml

@@ -62,7 +62,7 @@ application:
   corsOrigins: "http://localhost"
   dependencies:
       dynamicDependencies: "requirements-dynamic-dependencies.txt"
-  logsMaximumSize: 50 # in MB
+  logsMaximumSize: 52428800 # 50Mb in bytes
 
 cos:
   claimName: gateway-claim

gateway/.gitignore

@@ -137,4 +137,5 @@ GitHub.sublime-settings
 !.vscode/extensions.json
 .history
 
-tests/resources/fake_media/*
+tests/resources/fake_media/**/arguments/*
+!tests/resources/fake_media/**/logs/*.log

gateway/api/access_policies/jobs.py

@@ -68,6 +68,28 @@ def can_read_result(user: type[AbstractUser], job: Job) -> bool:
             )
         return has_access
 
+    @staticmethod
+    def can_read_logs(user: type[AbstractUser], job: Job) -> bool:
+        """
+        Checks if the user has permissions to read the result of a job:
+
+        Args:
+            user: Django user from the request
+            job: Job instance against to check the permission
+
+        Returns:
+            bool: True or False in case the user has permissions
+        """
+
+        has_access = user.id == job.author.id
+        if not has_access:
+            logger.warning(
+                "User [%s] has no access to read the result of the job [%s].",
+                user.username,
+                job.author,
+            )
+        return has_access
+
     @staticmethod
     def can_save_result(user: type[AbstractUser], job: Job) -> bool:
         """

gateway/api/access_policies/providers.py

@@ -29,7 +29,7 @@ def can_access(user, provider: Provider) -> bool:
         """
 
         user_groups = set(user.groups.all())
-        admin_groups = set(provider.admin_groups.all() if provider else [])
+        admin_groups = set(provider.admin_groups.all())
         user_is_admin = bool(user_groups.intersection(admin_groups))
         if not user_is_admin:
             logger.warning(

gateway/api/domain/function/check_logs.py

@@ -1,7 +1,6 @@
 """This class contain methods to manage the logs in the application."""
 
 import logging
-import sys
 from typing import Union
 
 from django.conf import settings
@@ -27,35 +26,35 @@ def check_logs(logs: Union[str, None], job: Job) -> str:
         logs with error message and metadata.
     """
 
-    max_mb = int(settings.FUNCTIONS_LOGS_SIZE_LIMIT)
-    max_bytes = max_mb * 1024**2
-
-    if job.status == Job.FAILED and logs in ["", None]:
+    if job.status == Job.FAILED and not logs:
         logs = f"Job {job.id} failed due to an internal error."
         logger.warning("Job %s failed due to an internal error.", job.id)
 
         return logs
 
-    logs_size = sys.getsizeof(logs)
-    if logs_size == 0:
-        return logs
+    if not logs:
+        return ""
+
+    max_bytes = int(settings.FUNCTIONS_LOGS_SIZE_LIMIT)
+
+    logs_size = len(logs)
 
     if logs_size > max_bytes:
         logger.warning(
             "Job %s is exceeding the maximum size for logs %s MB > %s MB.",
             job.id,
             logs_size,
-            max_mb,
+            max_bytes,
         )
-        ratio = max_bytes / logs_size
-        new_length = max(1, int(len(logs) * ratio))
-
-        # truncate logs depending of the ratio
-        logs = logs[:new_length]
-        logs += (
-            "\nLogs exceeded maximum allowed size ("
-            + str(max_mb)
-            + " MB) and could not be stored."
+
+        # truncate logs discarding older
+        logs = logs[-max_bytes:]
+
+        logs = (
+            "[Logs exceeded maximum allowed size ("
+            + str(max_bytes / (1024**2))
+            + " MB). Logs have been truncated, discarding the oldest entries first.]\n"
+            + logs
         )
 
     return logs

gateway/api/ray.py

@@ -21,9 +21,9 @@
 from opentelemetry import trace
 from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator
 
-from api.services.arguments_storage import ArgumentsStorage
+from api.services.storage import ArgumentsStorage
 from api.models import ComputeResource, Job, JobConfig, DEFAULT_PROGRAM_ENTRYPOINT
-from api.services.file_storage import FileStorage, WorkingDir
+from api.services.storage import FileStorage, WorkingDir
 from api.utils import (
     retry_function,
     decrypt_env_vars,

gateway/api/serializers.py

@@ -11,7 +11,7 @@
 from typing import Tuple, Union
 from django.conf import settings
 from rest_framework import serializers
-from api.services.arguments_storage import ArgumentsStorage
+from api.services.storage import ArgumentsStorage
 
 from api.repositories.functions import FunctionRepository
 from api.repositories.users import UserRepository

gateway/api/services/storage/__init__.py

@@ -0,0 +1,11 @@
+"""
+Qiskit Serverless storage services classes
+
+Storage classes are used to manage the content in Object Storage
+"""
+
+from .arguments_storage import ArgumentsStorage
+from .file_storage import FileStorage
+from .logs_storage import LogsStorage
+from .result_storage import ResultStorage
+from .enums.working_dir import WorkingDir

gateway/api/services/arguments_storage.py → gateway/api/services/storage/arguments_storage.py

@@ -4,7 +4,9 @@
 import logging
 import os
 from typing import Optional
-from django.conf import settings
+
+from api.services.storage.path_builder import PathBuilder
+from api.services.storage.enums.working_dir import WorkingDir
 
 logger = logging.getLogger("gateway")
 
@@ -13,32 +15,32 @@ class ArgumentsStorage:
     """Handles the storage and retrieval of user arguments."""
 
     ARGUMENTS_FILE_EXTENSION = ".json"
+    PATH = "arguments"
     ENCODING = "utf-8"
 
     def __init__(
         self, username: str, function_title: str, provider_name: Optional[str]
     ):
-        # We need to use the same path as the FileStorage here
-        # because it is attached the volume in the docker image
-        if provider_name is None:
-            self.user_arguments_directory = os.path.join(
-                settings.MEDIA_ROOT, username, "arguments"
-            )
-        else:
-            self.user_arguments_directory = os.path.join(
-                settings.MEDIA_ROOT,
-                username,
-                provider_name,
-                function_title,
-                "arguments",
-            )
-
-        os.makedirs(self.user_arguments_directory, exist_ok=True)
+        ### In this case arguments are always stored in user folder
+        self.sub_path = PathBuilder.sub_path(
+            working_dir=WorkingDir.USER_STORAGE,
+            username=username,
+            function_title=function_title,
+            provider_name=provider_name,
+            extra_sub_path=self.PATH,
+        )
+        self.absolute_path = PathBuilder.absolute_path(
+            working_dir=WorkingDir.USER_STORAGE,
+            username=username,
+            function_title=function_title,
+            provider_name=provider_name,
+            extra_sub_path=self.PATH,
+        )
 
     def _get_arguments_path(self, job_id: str) -> str:
         """Construct the full path for a arguments file."""
         return os.path.join(
-            self.user_arguments_directory, f"{job_id}{self.ARGUMENTS_FILE_EXTENSION}"
+            self.absolute_path, f"{job_id}{self.ARGUMENTS_FILE_EXTENSION}"
         )
 
     def get(self, job_id: str) -> Optional[str]:
@@ -56,7 +58,7 @@ def get(self, job_id: str) -> Optional[str]:
             logger.info(
                 "Arguments file for job ID '%s' not found in directory '%s'.",
                 job_id,
-                self.user_arguments_directory,
+                arguments_path,
             )
             return None
 

gateway/api/services/storage/enums/working_dir.py

@@ -0,0 +1,20 @@
+"""
+This class defines WorkingDir enum for Storage services:
+"""
+
+from enum import Enum
+
+
+class WorkingDir(Enum):
+    """
+    This Enum has the values:
+        USER_STORAGE
+        PROVIDER_STORAGE
+
+    Both values are being used to identify in
+        Storages service the path to be used by
+        the user or the provider
+    """
+
+    USER_STORAGE = 1
+    PROVIDER_STORAGE = 2

gateway/api/services/file_storage.py → gateway/api/services/storage/file_storage.py

@@ -5,30 +5,16 @@
 import logging
 import mimetypes
 import os
-from enum import Enum
 from typing import Optional, Tuple
 from wsgiref.util import FileWrapper
 
-from django.conf import settings
 from django.core.files import File
 
+from api.services.storage.path_builder import PathBuilder
+from api.services.storage.enums.working_dir import WorkingDir
 from utils import sanitize_file_path
 
 
-class WorkingDir(Enum):
-    """
-    This Enum has the values:
-        USER_STORAGE
-        PROVIDER_STORAGE
-
-    Both values are being used to identify in
-        FileStorage service the path to be used
-    """
-
-    USER_STORAGE = 1
-    PROVIDER_STORAGE = 2
-
-
 logger = logging.getLogger("gateway")
 
 
@@ -50,83 +36,20 @@ def __init__(
         function_title: str,
         provider_name: Optional[str],
     ) -> None:
-        self.sub_path = None
-        self.absolute_path = None
-        self.username = username
-
-        if working_dir is WorkingDir.USER_STORAGE:
-            self.sub_path = self.__get_user_sub_path(function_title, provider_name)
-        elif working_dir is WorkingDir.PROVIDER_STORAGE:
-            self.sub_path = self.__get_provider_sub_path(function_title, provider_name)
-
-        self.absolute_path = self.__get_absolute_path(self.sub_path)
-
-    def __get_user_sub_path(
-        self, function_title: str, provider_name: Optional[str]
-    ) -> str:
-        """
-        This method returns the sub-path where the user or the function
-        will store files
-
-        Args:
-            function_title (str): in case the function is from a
-                provider it will identify the function folder
-            provider_name (str | None): in case a provider is provided it will
-                identify the folder for the specific function
-
-        Returns:
-            str: storage sub-path.
-                - In case the function is from a provider that sub-path would
-                    be: username/provider_name/function_title
-                - In case the function is from a user that path would
-                    be: username/
-        """
-        if provider_name is None:
-            path = os.path.join(self.username)
-        else:
-            path = os.path.join(self.username, provider_name, function_title)
-
-        return sanitize_file_path(path)
-
-    def __get_provider_sub_path(self, function_title: str, provider_name: str) -> str:
-        """
-        This method returns the provider sub-path where the user
-        or the function will store files
-
-        Args:
-            function_title (str): in case the function is from a provider
-                it will identify the function folder
-            provider_name (str): in case a provider is provided
-                it will identify the folder for the specific function
-
-        Returns:
-            str: storage sub-path following the format provider_name/function_title/
-        """
-        path = os.path.join(provider_name, function_title)
-
-        return sanitize_file_path(path)
-
-    def __get_absolute_path(self, sub_path: str) -> str:
-        """
-        This method returns the absolute path where the user
-        or the function will store files
-
-        Args:
-            sub_path (str): the sub-path that we will use to build
-                the absolute path
-
-        Returns:
-            str: storage path.
-        """
-        path = os.path.join(settings.MEDIA_ROOT, sub_path)
-        sanitized_path = sanitize_file_path(path)
-
-        # Create directory if it doesn't exist
-        if not os.path.exists(sanitized_path):
-            os.makedirs(sanitized_path, exist_ok=True)
-            logger.debug("Path %s was created.", sanitized_path)
-
-        return sanitized_path
+        self.sub_path = PathBuilder.sub_path(
+            working_dir=working_dir,
+            username=username,
+            function_title=function_title,
+            provider_name=provider_name,
+            extra_sub_path=None,
+        )
+        self.absolute_path = PathBuilder.absolute_path(
+            working_dir=working_dir,
+            username=username,
+            function_title=function_title,
+            provider_name=provider_name,
+            extra_sub_path=None,
+        )
 
     def get_files(self) -> list[str]:
         """