From 10a02a7f9b789e9076819733991aded4c482fff7 Mon Sep 17 00:00:00 2001 From: David <9059044+Tansito@users.noreply.github.com> Date: Tue, 14 Oct 2025 11:41:02 -0400 Subject: [PATCH 1/4] Logs Storage service (#1724) * initial version of the log storage * storage refactor to folder * moved enum to be shared by storages * create a path builder * add refactor to file storage * add refactor to logs storage * add refactor to arguments storage * fix lint * moved storages to init * rename method for log path * remove get from static builder --- gateway/api/ray.py | 4 +- gateway/api/serializers.py | 2 +- gateway/api/services/storage/__init__.py | 11 ++ .../{ => storage}/arguments_storage.py | 40 +++-- .../api/services/storage/enums/__init__.py | 0 .../api/services/storage/enums/working_dir.py | 20 +++ .../services/{ => storage}/file_storage.py | 109 ++--------- gateway/api/services/storage/logs_storage.py | 92 ++++++++++ gateway/api/services/storage/path_builder.py | 170 ++++++++++++++++++ .../services/{ => storage}/result_storage.py | 0 gateway/api/use_cases/files/delete.py | 2 +- gateway/api/use_cases/files/download.py | 2 +- gateway/api/use_cases/files/list.py | 2 +- .../api/use_cases/files/provider_delete.py | 2 +- .../api/use_cases/files/provider_download.py | 2 +- gateway/api/use_cases/files/provider_list.py | 2 +- .../api/use_cases/files/provider_upload.py | 2 +- gateway/api/use_cases/files/upload.py | 2 +- gateway/api/use_cases/jobs/retrieve.py | 2 +- gateway/api/use_cases/jobs/save_result.py | 2 +- gateway/tests/api/test_v1_program.py | 2 +- 21 files changed, 344 insertions(+), 126 deletions(-) create mode 100644 gateway/api/services/storage/__init__.py rename gateway/api/services/{ => storage}/arguments_storage.py (72%) create mode 100644 gateway/api/services/storage/enums/__init__.py create mode 100644 gateway/api/services/storage/enums/working_dir.py rename gateway/api/services/{ => storage}/file_storage.py (58%) create mode 100644 gateway/api/services/storage/logs_storage.py create mode 100644 gateway/api/services/storage/path_builder.py rename gateway/api/services/{ => storage}/result_storage.py (100%) diff --git a/gateway/api/ray.py b/gateway/api/ray.py index 3ac8c705c..9fef241f9 100644 --- a/gateway/api/ray.py +++ b/gateway/api/ray.py @@ -21,9 +21,9 @@ from opentelemetry import trace from opentelemetry.trace.propagation.tracecontext import TraceContextTextMapPropagator -from api.services.arguments_storage import ArgumentsStorage +from api.services.storage import ArgumentsStorage from api.models import ComputeResource, Job, JobConfig, DEFAULT_PROGRAM_ENTRYPOINT -from api.services.file_storage import FileStorage, WorkingDir +from api.services.storage import FileStorage, WorkingDir from api.utils import ( retry_function, decrypt_env_vars, diff --git a/gateway/api/serializers.py b/gateway/api/serializers.py index a4e365917..819052c27 100644 --- a/gateway/api/serializers.py +++ b/gateway/api/serializers.py @@ -11,7 +11,7 @@ from typing import Tuple, Union from django.conf import settings from rest_framework import serializers -from api.services.arguments_storage import ArgumentsStorage +from api.services.storage import ArgumentsStorage from api.repositories.functions import FunctionRepository from api.repositories.users import UserRepository diff --git a/gateway/api/services/storage/__init__.py b/gateway/api/services/storage/__init__.py new file mode 100644 index 000000000..f2de530b2 --- /dev/null +++ b/gateway/api/services/storage/__init__.py @@ -0,0 +1,11 @@ +""" +Qiskit Serverless storage services classes + +Storage classes are used to manage the content in Object Storage +""" + +from .arguments_storage import ArgumentsStorage +from .file_storage import FileStorage +from .logs_storage import LogsStorage +from .result_storage import ResultStorage +from .enums.working_dir import WorkingDir diff --git a/gateway/api/services/arguments_storage.py b/gateway/api/services/storage/arguments_storage.py similarity index 72% rename from gateway/api/services/arguments_storage.py rename to gateway/api/services/storage/arguments_storage.py index c37acffc3..cdbc098a9 100644 --- a/gateway/api/services/arguments_storage.py +++ b/gateway/api/services/storage/arguments_storage.py @@ -4,7 +4,9 @@ import logging import os from typing import Optional -from django.conf import settings + +from api.services.storage.path_builder import PathBuilder +from api.services.storage.enums.working_dir import WorkingDir logger = logging.getLogger("gateway") @@ -13,32 +15,32 @@ class ArgumentsStorage: """Handles the storage and retrieval of user arguments.""" ARGUMENTS_FILE_EXTENSION = ".json" + PATH = "arguments" ENCODING = "utf-8" def __init__( self, username: str, function_title: str, provider_name: Optional[str] ): - # We need to use the same path as the FileStorage here - # because it is attached the volume in the docker image - if provider_name is None: - self.user_arguments_directory = os.path.join( - settings.MEDIA_ROOT, username, "arguments" - ) - else: - self.user_arguments_directory = os.path.join( - settings.MEDIA_ROOT, - username, - provider_name, - function_title, - "arguments", - ) - - os.makedirs(self.user_arguments_directory, exist_ok=True) + ### In this case arguments are always stored in user folder + self.sub_path = PathBuilder.sub_path( + working_dir=WorkingDir.USER_STORAGE, + username=username, + function_title=function_title, + provider_name=provider_name, + extra_sub_path=self.PATH, + ) + self.absolute_path = PathBuilder.absolute_path( + working_dir=WorkingDir.USER_STORAGE, + username=username, + function_title=function_title, + provider_name=provider_name, + extra_sub_path=self.PATH, + ) def _get_arguments_path(self, job_id: str) -> str: """Construct the full path for a arguments file.""" return os.path.join( - self.user_arguments_directory, f"{job_id}{self.ARGUMENTS_FILE_EXTENSION}" + self.absolute_path, f"{job_id}{self.ARGUMENTS_FILE_EXTENSION}" ) def get(self, job_id: str) -> Optional[str]: @@ -56,7 +58,7 @@ def get(self, job_id: str) -> Optional[str]: logger.info( "Arguments file for job ID '%s' not found in directory '%s'.", job_id, - self.user_arguments_directory, + arguments_path, ) return None diff --git a/gateway/api/services/storage/enums/__init__.py b/gateway/api/services/storage/enums/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/gateway/api/services/storage/enums/working_dir.py b/gateway/api/services/storage/enums/working_dir.py new file mode 100644 index 000000000..5329faa1b --- /dev/null +++ b/gateway/api/services/storage/enums/working_dir.py @@ -0,0 +1,20 @@ +""" +This class defines WorkingDir enum for Storage services: +""" + +from enum import Enum + + +class WorkingDir(Enum): + """ + This Enum has the values: + USER_STORAGE + PROVIDER_STORAGE + + Both values are being used to identify in + Storages service the path to be used by + the user or the provider + """ + + USER_STORAGE = 1 + PROVIDER_STORAGE = 2 diff --git a/gateway/api/services/file_storage.py b/gateway/api/services/storage/file_storage.py similarity index 58% rename from gateway/api/services/file_storage.py rename to gateway/api/services/storage/file_storage.py index 6f2e18885..e483ba5f4 100644 --- a/gateway/api/services/file_storage.py +++ b/gateway/api/services/storage/file_storage.py @@ -5,30 +5,16 @@ import logging import mimetypes import os -from enum import Enum from typing import Optional, Tuple from wsgiref.util import FileWrapper -from django.conf import settings from django.core.files import File +from api.services.storage.path_builder import PathBuilder +from api.services.storage.enums.working_dir import WorkingDir from utils import sanitize_file_path -class WorkingDir(Enum): - """ - This Enum has the values: - USER_STORAGE - PROVIDER_STORAGE - - Both values are being used to identify in - FileStorage service the path to be used - """ - - USER_STORAGE = 1 - PROVIDER_STORAGE = 2 - - logger = logging.getLogger("gateway") @@ -50,83 +36,20 @@ def __init__( function_title: str, provider_name: Optional[str], ) -> None: - self.sub_path = None - self.absolute_path = None - self.username = username - - if working_dir is WorkingDir.USER_STORAGE: - self.sub_path = self.__get_user_sub_path(function_title, provider_name) - elif working_dir is WorkingDir.PROVIDER_STORAGE: - self.sub_path = self.__get_provider_sub_path(function_title, provider_name) - - self.absolute_path = self.__get_absolute_path(self.sub_path) - - def __get_user_sub_path( - self, function_title: str, provider_name: Optional[str] - ) -> str: - """ - This method returns the sub-path where the user or the function - will store files - - Args: - function_title (str): in case the function is from a - provider it will identify the function folder - provider_name (str | None): in case a provider is provided it will - identify the folder for the specific function - - Returns: - str: storage sub-path. - - In case the function is from a provider that sub-path would - be: username/provider_name/function_title - - In case the function is from a user that path would - be: username/ - """ - if provider_name is None: - path = os.path.join(self.username) - else: - path = os.path.join(self.username, provider_name, function_title) - - return sanitize_file_path(path) - - def __get_provider_sub_path(self, function_title: str, provider_name: str) -> str: - """ - This method returns the provider sub-path where the user - or the function will store files - - Args: - function_title (str): in case the function is from a provider - it will identify the function folder - provider_name (str): in case a provider is provided - it will identify the folder for the specific function - - Returns: - str: storage sub-path following the format provider_name/function_title/ - """ - path = os.path.join(provider_name, function_title) - - return sanitize_file_path(path) - - def __get_absolute_path(self, sub_path: str) -> str: - """ - This method returns the absolute path where the user - or the function will store files - - Args: - sub_path (str): the sub-path that we will use to build - the absolute path - - Returns: - str: storage path. - """ - path = os.path.join(settings.MEDIA_ROOT, sub_path) - sanitized_path = sanitize_file_path(path) - - # Create directory if it doesn't exist - if not os.path.exists(sanitized_path): - os.makedirs(sanitized_path, exist_ok=True) - logger.debug("Path %s was created.", sanitized_path) - - return sanitized_path + self.sub_path = PathBuilder.sub_path( + working_dir=working_dir, + username=username, + function_title=function_title, + provider_name=provider_name, + extra_sub_path=None, + ) + self.absolute_path = PathBuilder.absolute_path( + working_dir=working_dir, + username=username, + function_title=function_title, + provider_name=provider_name, + extra_sub_path=None, + ) def get_files(self) -> list[str]: """ diff --git a/gateway/api/services/storage/logs_storage.py b/gateway/api/services/storage/logs_storage.py new file mode 100644 index 000000000..07973d4ac --- /dev/null +++ b/gateway/api/services/storage/logs_storage.py @@ -0,0 +1,92 @@ +""" +This module handle the access to the logs store +""" +import logging +import os +from typing import Optional + +from api.services.storage.path_builder import PathBuilder +from api.services.storage.enums.working_dir import WorkingDir + + +logger = logging.getLogger("gateway") + + +class LogsStorage: + """ + The main objective of this class is to manage the access to logs generated by a Job. + + Attributes: + username (str): storage user's username + working_dir (WorkingDir(Enum)): working directory + function_title (str): title of the function in case is needed to build the path + provider_name (str | None): name of the provider in caseis needed to build the path + """ + + FILE_EXTENSION = ".log" + PATH = "logs" + ENCODING = "utf-8" + + def __init__( + self, + username: str, + working_dir: WorkingDir, + function_title: str, + provider_name: Optional[str], + ) -> None: + self.sub_path = PathBuilder.sub_path( + working_dir=working_dir, + username=username, + function_title=function_title, + provider_name=provider_name, + extra_sub_path=self.PATH, + ) + self.absolute_path = PathBuilder.absolute_path( + working_dir=working_dir, + username=username, + function_title=function_title, + provider_name=provider_name, + extra_sub_path=self.PATH, + ) + + def _get_logs_path(self, job_id: str) -> str: + """ + Return the path to a log file from the id of a Job + + Args: + job_id (str): the id for the job to get the log file + + Returns: + Optional[str]: path to the log file + """ + return os.path.join(self.absolute_path, f"{job_id}{self.FILE_EXTENSION}") + + def get(self, job_id: str) -> Optional[str]: + """ + Retrieve a log file for the given job id + + Args: + job_id (str): the id for the job to get the logs + + Returns: + Optional[str]: content of the file + """ + log_path = self._get_logs_path(job_id) + if not os.path.exists(log_path): + logger.info( + "Log file for job ID '%s' not found in directory '%s'.", + job_id, + log_path, + ) + return None + + try: + with open(log_path, "r", encoding=self.ENCODING) as log_file: + return log_file.read() + except (UnicodeDecodeError, IOError) as e: + logger.error( + "Failed to read log file for job ID '%s': %s", + job_id, + str(e), + ) + return None diff --git a/gateway/api/services/storage/path_builder.py b/gateway/api/services/storage/path_builder.py new file mode 100644 index 000000000..335264032 --- /dev/null +++ b/gateway/api/services/storage/path_builder.py @@ -0,0 +1,170 @@ +""" +Builder class to manage the different paths generated in the application +""" +import logging +import os +from typing import Optional + +from django.conf import settings + +from api.services.storage.enums.working_dir import WorkingDir +from utils import sanitize_file_path + + +logger = logging.getLogger("gateway") + + +class PathBuilder: + """ + This class manages the logic for the user and provider paths generated in the Object Storage + """ + + @staticmethod + def __get_user_sub_path( + username: str, + function_title: str, + provider_name: Optional[str], + extra_sub_path: Optional[str], + ) -> str: + """ + This method returns the sub-path where the user or the function + will store files + + Args: + username (str): IBMiD of the user + function_title (str): in case the function is from a + provider it will identify the function folder + provider_name (str | None): in case a provider is provided it will + identify the folder for the specific function + extra_sub_path (str | None): any additional subpath that we want + to introduce in the path + + Returns: + str: storage sub-path. + - In case the function is from a provider that sub-path would + be: username/provider_name/function_title/{extra_sub_path} + - In case the function is from a user that path would + be: username/{extra_sub_path} + """ + if provider_name is None: + path = os.path.join(username) + else: + path = os.path.join(username, provider_name, function_title) + + if extra_sub_path is not None: + path = os.path.join(path, extra_sub_path) + + return sanitize_file_path(path) + + @staticmethod + def __get_provider_sub_path( + function_title: str, provider_name: str, extra_sub_path: Optional[str] + ) -> str: + """ + This method returns the provider sub-path where the user + or the function will store files + + Args: + function_title (str): in case the function is from a provider + it will identify the function folder + provider_name (str): in case a provider is provided + it will identify the folder for the specific function + extra_sub_path (str | None): any additional subpath that we want + to introduce in the path + + Returns: + str: storage sub-path following the format provider_name/function_title/{extra_sub_path} + """ + path = os.path.join(provider_name, function_title) + + if extra_sub_path is not None: + path = os.path.join(path, extra_sub_path) + + return sanitize_file_path(path) + + @staticmethod + def sub_path( + working_dir: WorkingDir, + username: str, + function_title: str, + provider_name: Optional[str], + extra_sub_path: Optional[str], + ): + """ + This method returns the relative path for the required interaction. + + Args: + working_dir (WorkingDir): configuration for the generation of + the directory + username (str): IBMiD of the user + function_title (str): in case the function is from a provider + it will identify the function folder + provider_name (str): in case a provider is provided + it will identify the folder for the specific function + extra_sub_path (str | None): any additional subpath that we want + to introduce in the path + + Returns: + str: storage relative path. + """ + + sub_path = None + if working_dir is WorkingDir.USER_STORAGE: + sub_path = PathBuilder.__get_user_sub_path( + username=username, + function_title=function_title, + provider_name=provider_name, + extra_sub_path=extra_sub_path, + ) + elif working_dir is WorkingDir.PROVIDER_STORAGE: + sub_path = PathBuilder.__get_provider_sub_path( + function_title=function_title, + provider_name=provider_name, + extra_sub_path=extra_sub_path, + ) + + return sub_path + + @staticmethod + def absolute_path( + working_dir: WorkingDir, + username: str, + function_title: str, + provider_name: Optional[str], + extra_sub_path: Optional[str], + ) -> str: + """ + This method returns the aboslute path for the required interaction + and it creates it if it doesn't exist. + + Args: + working_dir (WorkingDir): configuration for the generation of + the directory + username (str): IBMiD of the user + function_title (str): in case the function is from a provider + it will identify the function folder + provider_name (str): in case a provider is provided + it will identify the folder for the specific function + extra_sub_path (str | None): any additional subpath that we want + to introduce in the path + + Returns: + str: storage relative path. + """ + + sub_path = PathBuilder.sub_path( + working_dir=working_dir, + username=username, + function_title=function_title, + provider_name=provider_name, + extra_sub_path=extra_sub_path, + ) + path = os.path.join(settings.MEDIA_ROOT, sub_path) + sanitized_path = sanitize_file_path(path) + + # Create directory if it doesn't exist + if not os.path.exists(sanitized_path): + os.makedirs(sanitized_path, exist_ok=True) + logger.debug("Path %s was created.", sanitized_path) + + return sanitized_path diff --git a/gateway/api/services/result_storage.py b/gateway/api/services/storage/result_storage.py similarity index 100% rename from gateway/api/services/result_storage.py rename to gateway/api/services/storage/result_storage.py diff --git a/gateway/api/use_cases/files/delete.py b/gateway/api/use_cases/files/delete.py index c520970d2..c32e0b014 100644 --- a/gateway/api/use_cases/files/delete.py +++ b/gateway/api/use_cases/files/delete.py @@ -2,7 +2,7 @@ # pylint: disable=duplicate-code import logging from django.contrib.auth.models import AbstractUser -from api.services.file_storage import FileStorage, WorkingDir +from api.services.storage import FileStorage, WorkingDir from api.repositories.functions import FunctionRepository from api.domain.exceptions.not_found_error import NotFoundError diff --git a/gateway/api/use_cases/files/download.py b/gateway/api/use_cases/files/download.py index a63ba02ba..be81754cf 100644 --- a/gateway/api/use_cases/files/download.py +++ b/gateway/api/use_cases/files/download.py @@ -2,7 +2,7 @@ # pylint: disable=duplicate-code import logging from django.contrib.auth.models import AbstractUser -from api.services.file_storage import FileStorage, WorkingDir +from api.services.storage import FileStorage, WorkingDir from api.repositories.functions import FunctionRepository from api.domain.exceptions.not_found_error import NotFoundError diff --git a/gateway/api/use_cases/files/list.py b/gateway/api/use_cases/files/list.py index 3c93a52b1..e807bd2de 100644 --- a/gateway/api/use_cases/files/list.py +++ b/gateway/api/use_cases/files/list.py @@ -2,7 +2,7 @@ # pylint: disable=duplicate-code import logging from django.contrib.auth.models import AbstractUser -from api.services.file_storage import FileStorage, WorkingDir +from api.services.storage import FileStorage, WorkingDir from api.repositories.functions import FunctionRepository from api.domain.exceptions.not_found_error import NotFoundError diff --git a/gateway/api/use_cases/files/provider_delete.py b/gateway/api/use_cases/files/provider_delete.py index 7f748cfa6..e2db7823d 100644 --- a/gateway/api/use_cases/files/provider_delete.py +++ b/gateway/api/use_cases/files/provider_delete.py @@ -4,7 +4,7 @@ from django.contrib.auth.models import AbstractUser from api.access_policies.providers import ProviderAccessPolicy from api.repositories.providers import ProviderRepository -from api.services.file_storage import FileStorage, WorkingDir +from api.services.storage import FileStorage, WorkingDir from api.repositories.functions import FunctionRepository from api.domain.exceptions.not_found_error import NotFoundError diff --git a/gateway/api/use_cases/files/provider_download.py b/gateway/api/use_cases/files/provider_download.py index 58806b406..d3908bae2 100644 --- a/gateway/api/use_cases/files/provider_download.py +++ b/gateway/api/use_cases/files/provider_download.py @@ -4,7 +4,7 @@ from django.contrib.auth.models import AbstractUser from api.access_policies.providers import ProviderAccessPolicy from api.repositories.providers import ProviderRepository -from api.services.file_storage import FileStorage, WorkingDir +from api.services.storage import FileStorage, WorkingDir from api.repositories.functions import FunctionRepository from api.domain.exceptions.not_found_error import NotFoundError diff --git a/gateway/api/use_cases/files/provider_list.py b/gateway/api/use_cases/files/provider_list.py index 810bd5284..eeceb390e 100644 --- a/gateway/api/use_cases/files/provider_list.py +++ b/gateway/api/use_cases/files/provider_list.py @@ -3,7 +3,7 @@ import logging from django.contrib.auth.models import AbstractUser from api.access_policies.providers import ProviderAccessPolicy -from api.services.file_storage import FileStorage, WorkingDir +from api.services.storage import FileStorage, WorkingDir from api.repositories.functions import FunctionRepository from api.repositories.providers import ProviderRepository from api.domain.exceptions.not_found_error import NotFoundError diff --git a/gateway/api/use_cases/files/provider_upload.py b/gateway/api/use_cases/files/provider_upload.py index c3d0f5f0b..1cbe4abf3 100644 --- a/gateway/api/use_cases/files/provider_upload.py +++ b/gateway/api/use_cases/files/provider_upload.py @@ -5,7 +5,7 @@ from django.core.files import File from api.access_policies.providers import ProviderAccessPolicy from api.repositories.providers import ProviderRepository -from api.services.file_storage import FileStorage, WorkingDir +from api.services.storage import FileStorage, WorkingDir from api.repositories.functions import FunctionRepository from api.domain.exceptions.not_found_error import NotFoundError diff --git a/gateway/api/use_cases/files/upload.py b/gateway/api/use_cases/files/upload.py index 5b67afa8b..276c1660a 100644 --- a/gateway/api/use_cases/files/upload.py +++ b/gateway/api/use_cases/files/upload.py @@ -3,7 +3,7 @@ import logging from django.core.files import File from django.contrib.auth.models import AbstractUser -from api.services.file_storage import FileStorage, WorkingDir +from api.services.storage import FileStorage, WorkingDir from api.repositories.functions import FunctionRepository from api.domain.exceptions.not_found_error import NotFoundError diff --git a/gateway/api/use_cases/jobs/retrieve.py b/gateway/api/use_cases/jobs/retrieve.py index 5cdc01d36..9982672c7 100644 --- a/gateway/api/use_cases/jobs/retrieve.py +++ b/gateway/api/use_cases/jobs/retrieve.py @@ -6,7 +6,7 @@ from api.models import Job from api.repositories.jobs import JobsRepository from api.access_policies.jobs import JobAccessPolicies -from api.services.result_storage import ResultStorage +from api.services.storage import ResultStorage logger = logging.getLogger("gateway.use_cases.jobs") diff --git a/gateway/api/use_cases/jobs/save_result.py b/gateway/api/use_cases/jobs/save_result.py index 214a0401f..a192d503b 100644 --- a/gateway/api/use_cases/jobs/save_result.py +++ b/gateway/api/use_cases/jobs/save_result.py @@ -8,7 +8,7 @@ from api.repositories.jobs import JobsRepository from api.domain.exceptions.not_found_error import NotFoundError from api.access_policies.jobs import JobAccessPolicies -from api.services.result_storage import ResultStorage +from api.services.storage import ResultStorage from api.models import Job logger = logging.getLogger("gateway.use_cases.jobs") diff --git a/gateway/tests/api/test_v1_program.py b/gateway/tests/api/test_v1_program.py index 4e17bcca4..292763b53 100644 --- a/gateway/tests/api/test_v1_program.py +++ b/gateway/tests/api/test_v1_program.py @@ -10,7 +10,7 @@ from rest_framework.test import APITestCase from api.models import Job, Program -from api.services.arguments_storage import ArgumentsStorage +from api.services.storage import ArgumentsStorage class TestProgramApi(APITestCase): From bda5367bd00d46e74d5ed62da345c25d1418446c Mon Sep 17 00:00:00 2001 From: Goyo Date: Fri, 7 Nov 2025 21:57:54 +0100 Subject: [PATCH 2/4] Update logs endpoint (#1750) * Update logs endpoint behavior * update tests * change gitignore to upload logs * disable logs tests * address comment * disable logs tests 2 * disable logs tests 3 * disable logs tests 4 * disable logs tests 5 * disable logs tests 6 * Revert "disable logs tests" This reverts commit bf235c7556f5b871c8313f20a4902ad7cfd604c3. --- gateway/.gitignore | 3 +- gateway/api/access_policies/jobs.py | 22 +++ gateway/api/use_cases/jobs/get_logs.py | 29 ++-- gateway/tests/api/test_job.py | 130 +++++++----------- .../57fc2e4d-267f-40c6-91a3-38153272e764.log | 1 + .../1a7947f9-6ae8-4e3d-ac1e-e7d608deec85.log | 1 + 6 files changed, 94 insertions(+), 92 deletions(-) create mode 100644 gateway/tests/resources/fake_media/test_user/logs/57fc2e4d-267f-40c6-91a3-38153272e764.log create mode 100644 gateway/tests/resources/fake_media/test_user_2/default/Docker-Image-Program/logs/1a7947f9-6ae8-4e3d-ac1e-e7d608deec85.log diff --git a/gateway/.gitignore b/gateway/.gitignore index 2dfb26dee..16b4924ac 100644 --- a/gateway/.gitignore +++ b/gateway/.gitignore @@ -137,4 +137,5 @@ GitHub.sublime-settings !.vscode/extensions.json .history -tests/resources/fake_media/* +tests/resources/fake_media/**/arguments/* +!tests/resources/fake_media/**/logs/*.log diff --git a/gateway/api/access_policies/jobs.py b/gateway/api/access_policies/jobs.py index f9a2227a0..7bf4ef47b 100644 --- a/gateway/api/access_policies/jobs.py +++ b/gateway/api/access_policies/jobs.py @@ -68,6 +68,28 @@ def can_read_result(user: type[AbstractUser], job: Job) -> bool: ) return has_access + @staticmethod + def can_read_logs(user: type[AbstractUser], job: Job) -> bool: + """ + Checks if the user has permissions to read the result of a job: + + Args: + user: Django user from the request + job: Job instance against to check the permission + + Returns: + bool: True or False in case the user has permissions + """ + + has_access = user.id == job.author.id + if not has_access: + logger.warning( + "User [%s] has no access to read the result of the job [%s].", + user.username, + job.author, + ) + return has_access + @staticmethod def can_save_result(user: type[AbstractUser], job: Job) -> bool: """ diff --git a/gateway/api/use_cases/jobs/get_logs.py b/gateway/api/use_cases/jobs/get_logs.py index cf3cbee5a..21cccc0a7 100644 --- a/gateway/api/use_cases/jobs/get_logs.py +++ b/gateway/api/use_cases/jobs/get_logs.py @@ -6,10 +6,13 @@ from django.contrib.auth.models import AbstractUser +from api.access_policies.jobs import JobAccessPolicies from api.domain.exceptions.not_found_error import NotFoundError from api.domain.exceptions.forbidden_error import ForbiddenError from api.repositories.jobs import JobsRepository from api.access_policies.providers import ProviderAccessPolicy +from api.services.storage.enums.working_dir import WorkingDir +from api.services.storage.logs_storage import LogsStorage NO_LOGS_MSG: Final[str] = "No available logs" @@ -37,14 +40,22 @@ def execute(self, job_id: UUID, user: AbstractUser) -> str: if job is None: raise NotFoundError(f"Job [{job_id}] not found") - # Case 1: Provider function - check provider access policy - if job.program and job.program.provider: - if ProviderAccessPolicy.can_access(user, job.program.provider): - return job.logs + if not JobAccessPolicies.can_read_logs(user, job): + raise ForbiddenError(f"You don't have access to job [{job_id}]") - # Case 2: User is the author of the job - elif user == job.author: - return job.logs + logs_storage = LogsStorage( + username=user.username, + working_dir=WorkingDir.USER_STORAGE, + function_title=job.program.title, + provider_name=job.program.provider.name if job.program.provider else None, + ) - # Access denied for all other cases - raise ForbiddenError(f"You don't have access to job [{job_id}]") + logs = logs_storage.get(job_id) + + if logs is None: + raise NotFoundError(f"Logs for job[{job_id}] are not found") + + if len(logs) == 0: + return "No logs available" + + return logs diff --git a/gateway/tests/api/test_job.py b/gateway/tests/api/test_job.py index b981a65df..75b47b93f 100644 --- a/gateway/tests/api/test_job.py +++ b/gateway/tests/api/test_job.py @@ -22,6 +22,16 @@ def _authorize(self, username="test_user"): user = models.User.objects.get(username=username) self.client.force_authenticate(user=user) + def _fake_media_root(self): + media_root = os.path.join( + os.path.dirname(os.path.abspath(__file__)), + "..", + "resources", + "fake_media", + ) + media_root = os.path.normpath(os.path.join(os.getcwd(), media_root)) + return media_root + def test_job_non_auth_user(self): """Tests job list non-authorized.""" url = reverse("v1:jobs-list") @@ -256,15 +266,7 @@ def test_job_provider_list_pagination(self): def test_job_detail(self): """Tests job detail authorized.""" - media_root = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - "resources", - "fake_media", - ) - media_root = os.path.normpath(os.path.join(os.getcwd(), media_root)) - - with self.settings(MEDIA_ROOT=media_root): + with self.settings(MEDIA_ROOT=self._fake_media_root()): self._authorize() jobs_response = self.client.get( @@ -276,15 +278,7 @@ def test_job_detail(self): def test_job_detail_without_result_param(self): """Tests job detail authorized.""" - media_root = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - "resources", - "fake_media", - ) - media_root = os.path.normpath(os.path.join(os.getcwd(), media_root)) - - with self.settings(MEDIA_ROOT=media_root): + with self.settings(MEDIA_ROOT=self._fake_media_root()): self._authorize() jobs_response = self.client.get( @@ -297,15 +291,7 @@ def test_job_detail_without_result_param(self): def test_job_detail_without_result_file(self): """Tests job detail authorized.""" - media_root = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - "resources", - "fake_media", - ) - media_root = os.path.normpath(os.path.join(os.getcwd(), media_root)) - - with self.settings(MEDIA_ROOT=media_root): + with self.settings(MEDIA_ROOT=self._fake_media_root()): self._authorize() jobs_response = self.client.get( @@ -340,15 +326,7 @@ def test_not_authorized_job_detail(self): def test_job_save_result(self): """Tests job results save.""" - media_root = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - "resources", - "fake_media", - ) - media_root = os.path.normpath(os.path.join(os.getcwd(), media_root)) - - with self.settings(MEDIA_ROOT=media_root): + with self.settings(MEDIA_ROOT=self._fake_media_root()): self._authorize() job_id = "57fc2e4d-267f-40c6-91a3-38153272e764" jobs_response = self.client.post( @@ -465,15 +443,7 @@ def test_user_has_access_to_job_result_from_provider_function(self): User has access to job result from a function provider as the authot of the job """ - media_root = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - "resources", - "fake_media", - ) - media_root = os.path.normpath(os.path.join(os.getcwd(), media_root)) - - with self.settings(MEDIA_ROOT=media_root): + with self.settings(MEDIA_ROOT=self._fake_media_root()): self._authorize() jobs_response = self.client.get( @@ -488,15 +458,7 @@ def test_provider_admin_has_no_access_to_job_result_from_provider_function(self) A provider admin has no access to job result from a function provider if it's not the author of the job """ - media_root = os.path.join( - os.path.dirname(os.path.abspath(__file__)), - "..", - "resources", - "fake_media", - ) - media_root = os.path.normpath(os.path.join(os.getcwd(), media_root)) - - with self.settings(MEDIA_ROOT=media_root): + with self.settings(MEDIA_ROOT=self._fake_media_root()): user = models.User.objects.get(username="test_user_3") self.client.force_authenticate(user=user) @@ -529,47 +491,51 @@ def test_stop_job(self): def test_job_logs_by_author_for_function_without_provider(self): """Tests job log by job author.""" - self._authorize() + with self.settings(MEDIA_ROOT=self._fake_media_root()): + self._authorize() - jobs_response = self.client.get( - reverse("v1:jobs-logs", args=["57fc2e4d-267f-40c6-91a3-38153272e764"]), - format="json", - ) - self.assertEqual(jobs_response.status_code, status.HTTP_200_OK) - self.assertEqual(jobs_response.data.get("logs"), "log entry 2") + jobs_response = self.client.get( + reverse("v1:jobs-logs", args=["57fc2e4d-267f-40c6-91a3-38153272e764"]), + format="json", + ) + self.assertEqual(jobs_response.status_code, status.HTTP_200_OK) + self.assertEqual(jobs_response.data.get("logs"), "log entry 2") def test_job_logs_by_author_for_function_with_provider(self): """Tests job log by job author.""" - self._authorize() + with self.settings(MEDIA_ROOT=self._fake_media_root()): + self._authorize() - jobs_response = self.client.get( - reverse("v1:jobs-logs", args=["1a7947f9-6ae8-4e3d-ac1e-e7d608deec85"]), - format="json", - ) - self.assertEqual(jobs_response.status_code, status.HTTP_403_FORBIDDEN) + jobs_response = self.client.get( + reverse("v1:jobs-logs", args=["1a7947f9-6ae8-4e3d-ac1e-e7d608deec85"]), + format="json", + ) + self.assertEqual(jobs_response.status_code, status.HTTP_403_FORBIDDEN) def test_job_logs_by_function_provider(self): """Tests job log by fuction provider.""" - user = models.User.objects.get(username="test_user_2") - self.client.force_authenticate(user=user) + with self.settings(MEDIA_ROOT=self._fake_media_root()): + user = models.User.objects.get(username="test_user_2") + self.client.force_authenticate(user=user) - jobs_response = self.client.get( - reverse("v1:jobs-logs", args=["1a7947f9-6ae8-4e3d-ac1e-e7d608deec85"]), - format="json", - ) - self.assertEqual(jobs_response.status_code, status.HTTP_200_OK) - self.assertEqual(jobs_response.data.get("logs"), "log entry 1") + jobs_response = self.client.get( + reverse("v1:jobs-logs", args=["1a7947f9-6ae8-4e3d-ac1e-e7d608deec85"]), + format="json", + ) + self.assertEqual(jobs_response.status_code, status.HTTP_200_OK) + self.assertEqual(jobs_response.data.get("logs"), "log entry 1") def test_job_logs(self): """Tests job log non-authorized.""" - user = models.User.objects.get(username="test_user_3") - self.client.force_authenticate(user=user) + with self.settings(MEDIA_ROOT=self._fake_media_root()): + user = models.User.objects.get(username="test_user_3") + self.client.force_authenticate(user=user) - jobs_response = self.client.get( - reverse("v1:jobs-logs", args=["1a7947f9-6ae8-4e3d-ac1e-e7d608deec85"]), - format="json", - ) - self.assertEqual(jobs_response.status_code, status.HTTP_403_FORBIDDEN) + jobs_response = self.client.get( + reverse("v1:jobs-logs", args=["1a7947f9-6ae8-4e3d-ac1e-e7d608deec85"]), + format="json", + ) + self.assertEqual(jobs_response.status_code, status.HTTP_403_FORBIDDEN) def test_runtime_jobs_post(self): """Tests runtime jobs POST endpoint.""" diff --git a/gateway/tests/resources/fake_media/test_user/logs/57fc2e4d-267f-40c6-91a3-38153272e764.log b/gateway/tests/resources/fake_media/test_user/logs/57fc2e4d-267f-40c6-91a3-38153272e764.log new file mode 100644 index 000000000..6949f9ab1 --- /dev/null +++ b/gateway/tests/resources/fake_media/test_user/logs/57fc2e4d-267f-40c6-91a3-38153272e764.log @@ -0,0 +1 @@ +log entry 2 \ No newline at end of file diff --git a/gateway/tests/resources/fake_media/test_user_2/default/Docker-Image-Program/logs/1a7947f9-6ae8-4e3d-ac1e-e7d608deec85.log b/gateway/tests/resources/fake_media/test_user_2/default/Docker-Image-Program/logs/1a7947f9-6ae8-4e3d-ac1e-e7d608deec85.log new file mode 100644 index 000000000..a78b9817a --- /dev/null +++ b/gateway/tests/resources/fake_media/test_user_2/default/Docker-Image-Program/logs/1a7947f9-6ae8-4e3d-ac1e-e7d608deec85.log @@ -0,0 +1 @@ +log entry 1 \ No newline at end of file From 250595eb06873801b7acf0b94fcc6d49d4b32683 Mon Sep 17 00:00:00 2001 From: Goyo Date: Mon, 17 Nov 2025 15:18:31 +0100 Subject: [PATCH 3/4] Provider logs endpoint (#1762) * Update logs endpoint behavior * update tests * change gitignore to upload logs * disable logs tests * address comment * disable logs tests 2 * disable logs tests 3 * disable logs tests 4 * disable logs tests 5 * disable logs tests 6 * provider logs endpoint implementation * provider tests * Revert "disable logs tests" This reverts commit bf235c7556f5b871c8313f20a4902ad7cfd604c3. --- gateway/api/access_policies/providers.py | 2 +- gateway/api/use_cases/jobs/provider_logs.py | 64 +++++++++++++++++ .../api/v1/views/jobs_new/provider_logs.py | 69 +++++++++++++++++++ gateway/tests/api/test_job.py | 54 +++++++++++++++ .../1a7947f9-6ae8-4e3d-ac1e-e7d608deec85.log | 1 + 5 files changed, 189 insertions(+), 1 deletion(-) create mode 100644 gateway/api/use_cases/jobs/provider_logs.py create mode 100644 gateway/api/v1/views/jobs_new/provider_logs.py create mode 100644 gateway/tests/resources/fake_media/default/Docker-Image-Program/logs/1a7947f9-6ae8-4e3d-ac1e-e7d608deec85.log diff --git a/gateway/api/access_policies/providers.py b/gateway/api/access_policies/providers.py index 6d598c944..97be008a7 100644 --- a/gateway/api/access_policies/providers.py +++ b/gateway/api/access_policies/providers.py @@ -29,7 +29,7 @@ def can_access(user, provider: Provider) -> bool: """ user_groups = set(user.groups.all()) - admin_groups = set(provider.admin_groups.all() if provider else []) + admin_groups = set(provider.admin_groups.all()) user_is_admin = bool(user_groups.intersection(admin_groups)) if not user_is_admin: logger.warning( diff --git a/gateway/api/use_cases/jobs/provider_logs.py b/gateway/api/use_cases/jobs/provider_logs.py new file mode 100644 index 000000000..6f84bc3b6 --- /dev/null +++ b/gateway/api/use_cases/jobs/provider_logs.py @@ -0,0 +1,64 @@ +""" +Use case: retrieve job logs. +""" +from typing import Final +from uuid import UUID + +from django.contrib.auth.models import AbstractUser + +from api.access_policies.jobs import JobAccessPolicies +from api.domain.exceptions.not_found_error import NotFoundError +from api.domain.exceptions.forbidden_error import ForbiddenError +from api.repositories.jobs import JobsRepository +from api.access_policies.providers import ProviderAccessPolicy +from api.services.storage.enums.working_dir import WorkingDir +from api.services.storage.logs_storage import LogsStorage + + +NO_LOGS_MSG: Final[str] = "No available logs" +NO_LOGS_MSG_2: Final[str] = "No logs yet." + + +class GetProviderJobLogsUseCase: + """Use case for retrieving job logs.""" + + jobs_repository = JobsRepository() + + def execute(self, job_id: UUID, user: AbstractUser) -> str: + """Return the logs of a job if the user has access. + + Args: + job_id (str): Unique identifier of the job. + user (AbstractUser): User requesting the logs. + + Raises: + NotFoundError: If the job does not exist. + + Returns: + str: Job logs if accessible, otherwise a message indicating no logs are available. + """ + job = self.jobs_repository.get_job_by_id(job_id) + if job is None: + raise NotFoundError(f"Job [{job_id}] not found 1") + + if not job.program.provider or not ProviderAccessPolicy.can_access( + user, job.program.provider + ): + raise ForbiddenError(f"You don't have access to job [{job_id}]") + + logs_storage = LogsStorage( + username=user.username, + working_dir=WorkingDir.PROVIDER_STORAGE, + function_title=job.program.title, + provider_name=job.program.provider.name, + ) + + logs = logs_storage.get(job_id) + + if logs is None: + logs = job.logs + + if not logs or logs == NO_LOGS_MSG or logs == NO_LOGS_MSG_2: + raise NotFoundError(f"Logs for job[{job_id}] are not found") + + return logs diff --git a/gateway/api/v1/views/jobs_new/provider_logs.py b/gateway/api/v1/views/jobs_new/provider_logs.py new file mode 100644 index 000000000..6ff98452f --- /dev/null +++ b/gateway/api/v1/views/jobs_new/provider_logs.py @@ -0,0 +1,69 @@ +""" +API endpoint for retrieving job logs. +""" + +# pylint: disable=duplicate-code, abstract-method + +from typing import Any, cast +from uuid import UUID + +from django.contrib.auth.models import AbstractUser +from drf_yasg.utils import swagger_auto_schema +from rest_framework import permissions, serializers, status +from rest_framework.decorators import api_view, permission_classes +from rest_framework.request import Request +from rest_framework.response import Response + +from api.use_cases.jobs.provider_logs import GetProviderJobLogsUseCase +from api.v1.endpoint_decorator import endpoint +from api.v1.endpoint_handle_exceptions import endpoint_handle_exceptions +from api.v1.views.swagger_utils import standard_error_responses + + +class JobProviderLogsOutputSerializer(serializers.Serializer): + """ + Serializer for job logs response. + """ + + logs = serializers.CharField() + + +def serialize_output(logs: str) -> dict[str, Any]: + """ + Serialize logs into the standard response format. + + Args: + logs: The job logs as a string. + + Returns: + A dictionary with the serialized logs. + """ + return JobProviderLogsOutputSerializer({"logs": logs}).data + + +@swagger_auto_schema( + method="get", + operation_description="Retrieve logs for a given job as provider.", + responses={ + status.HTTP_200_OK: JobProviderLogsOutputSerializer, + **standard_error_responses(not_found_example="Job [XXXX] not found"), + }, +) +@endpoint("jobs//provider-logs", name="jobs-provider-logs") +@api_view(["GET"]) +@permission_classes([permissions.IsAuthenticated]) +@endpoint_handle_exceptions +def provider_logs(request: Request, job_id: UUID) -> Response: + """ + Retrieve logs for a specific job. + + Args: + request: The HTTP request object. + job_id: The UUID of the job (path parameter). + + Returns: + Response containing the serialized job logs. + """ + user = cast(AbstractUser, request.user) + logs = GetProviderJobLogsUseCase().execute(job_id, user) + return Response(serialize_output(logs)) diff --git a/gateway/tests/api/test_job.py b/gateway/tests/api/test_job.py index 75b47b93f..f91089fca 100644 --- a/gateway/tests/api/test_job.py +++ b/gateway/tests/api/test_job.py @@ -525,6 +525,60 @@ def test_job_logs_by_function_provider(self): self.assertEqual(jobs_response.status_code, status.HTTP_200_OK) self.assertEqual(jobs_response.data.get("logs"), "log entry 1") + def test_job_provider_logs(self): + """Tests job log by fuction provider.""" + with self.settings(MEDIA_ROOT=self._fake_media_root()): + user = models.User.objects.get(username="test_user_2") + self.client.force_authenticate(user=user) + + jobs_response = self.client.get( + reverse( + "v1:jobs-provider-logs", + args=["1a7947f9-6ae8-4e3d-ac1e-e7d608deec85"], + ), + format="json", + ) + + self.assertEqual(jobs_response.status_code, status.HTTP_200_OK) + self.assertEqual(jobs_response.data.get("logs"), "provider log entry 1") + + def test_job_provider_logs_forbidden(self): + """Tests job log by fuction provider.""" + with self.settings(MEDIA_ROOT=self._fake_media_root()): + user = models.User.objects.get(username="test_user") + self.client.force_authenticate(user=user) + + jobs_response = self.client.get( + reverse( + "v1:jobs-provider-logs", + args=["1a7947f9-6ae8-4e3d-ac1e-e7d608deec85"], + ), + format="json", + ) + + self.assertEqual(jobs_response.status_code, status.HTTP_403_FORBIDDEN) + + def test_job_provider_logs_not_fount_empty(self): + """Tests job log by fuction provider.""" + with self.settings(MEDIA_ROOT=self._fake_media_root()): + user = models.User.objects.get(username="test_user_3") + self.client.force_authenticate(user=user) + + job_id = "1a7947f9-6ae8-4e3d-ac1e-e7d608deec87" + jobs_response = self.client.get( + reverse( + "v1:jobs-provider-logs", + args=[job_id], + ), + format="json", + ) + + self.assertEqual(jobs_response.status_code, status.HTTP_404_NOT_FOUND) + self.assertEqual( + jobs_response.data.get("message"), + f"Logs for job[{job_id}] are not found", + ) + def test_job_logs(self): """Tests job log non-authorized.""" with self.settings(MEDIA_ROOT=self._fake_media_root()): diff --git a/gateway/tests/resources/fake_media/default/Docker-Image-Program/logs/1a7947f9-6ae8-4e3d-ac1e-e7d608deec85.log b/gateway/tests/resources/fake_media/default/Docker-Image-Program/logs/1a7947f9-6ae8-4e3d-ac1e-e7d608deec85.log new file mode 100644 index 000000000..715a74481 --- /dev/null +++ b/gateway/tests/resources/fake_media/default/Docker-Image-Program/logs/1a7947f9-6ae8-4e3d-ac1e-e7d608deec85.log @@ -0,0 +1 @@ +provider log entry 1 \ No newline at end of file From f09f7b4f64618d157d10cb3535523c31ecbad89f Mon Sep 17 00:00:00 2001 From: Goyo Date: Sat, 22 Nov 2025 00:08:17 +0100 Subject: [PATCH 4/4] fixed check logs (#1777) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fixed check logs * fix logs length * Add failed message instead of replace logs * tests modified * Update gateway/api/domain/function/check_logs.py Co-authored-by: Elena Peña Tapia <57907331+ElePT@users.noreply.github.com> * fix test * change from mb to bytes * fix black * change values.yaml --------- Co-authored-by: Elena Peña Tapia <57907331+ElePT@users.noreply.github.com> --- .../charts/gateway/values.yaml | 2 +- gateway/api/domain/function/check_logs.py | 35 +++++++++---------- gateway/main/settings.py | 4 +-- gateway/tests/api/management/test_commands.py | 11 ++++-- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/charts/qiskit-serverless/charts/gateway/values.yaml b/charts/qiskit-serverless/charts/gateway/values.yaml index b952dad5c..3610d46a6 100644 --- a/charts/qiskit-serverless/charts/gateway/values.yaml +++ b/charts/qiskit-serverless/charts/gateway/values.yaml @@ -62,7 +62,7 @@ application: corsOrigins: "http://localhost" dependencies: dynamicDependencies: "requirements-dynamic-dependencies.txt" - logsMaximumSize: 50 # in MB + logsMaximumSize: 52428800 # 50Mb in bytes cos: claimName: gateway-claim diff --git a/gateway/api/domain/function/check_logs.py b/gateway/api/domain/function/check_logs.py index e8291bffc..d8f7ab104 100644 --- a/gateway/api/domain/function/check_logs.py +++ b/gateway/api/domain/function/check_logs.py @@ -1,7 +1,6 @@ """This class contain methods to manage the logs in the application.""" import logging -import sys from typing import Union from django.conf import settings @@ -27,35 +26,35 @@ def check_logs(logs: Union[str, None], job: Job) -> str: logs with error message and metadata. """ - max_mb = int(settings.FUNCTIONS_LOGS_SIZE_LIMIT) - max_bytes = max_mb * 1024**2 - - if job.status == Job.FAILED and logs in ["", None]: + if job.status == Job.FAILED and not logs: logs = f"Job {job.id} failed due to an internal error." logger.warning("Job %s failed due to an internal error.", job.id) return logs - logs_size = sys.getsizeof(logs) - if logs_size == 0: - return logs + if not logs: + return "" + + max_bytes = int(settings.FUNCTIONS_LOGS_SIZE_LIMIT) + + logs_size = len(logs) if logs_size > max_bytes: logger.warning( "Job %s is exceeding the maximum size for logs %s MB > %s MB.", job.id, logs_size, - max_mb, + max_bytes, ) - ratio = max_bytes / logs_size - new_length = max(1, int(len(logs) * ratio)) - - # truncate logs depending of the ratio - logs = logs[:new_length] - logs += ( - "\nLogs exceeded maximum allowed size (" - + str(max_mb) - + " MB) and could not be stored." + + # truncate logs discarding older + logs = logs[-max_bytes:] + + logs = ( + "[Logs exceeded maximum allowed size (" + + str(max_bytes / (1024**2)) + + " MB). Logs have been truncated, discarding the oldest entries first.]\n" + + logs ) return logs diff --git a/gateway/main/settings.py b/gateway/main/settings.py index 3dfa16c55..9b380c84c 100644 --- a/gateway/main/settings.py +++ b/gateway/main/settings.py @@ -420,5 +420,5 @@ SECURE_CONTENT_TYPE_NOSNIFF = True SESSION_COOKIE_AGE = 3600 -# Functions logs size limite in MB -FUNCTIONS_LOGS_SIZE_LIMIT = os.environ.get("FUNCTIONS_LOGS_SIZE_LIMIT", "50") +# Functions logs size limite in Bytes +FUNCTIONS_LOGS_SIZE_LIMIT = os.environ.get("FUNCTIONS_LOGS_SIZE_LIMIT", "52428800") diff --git a/gateway/tests/api/management/test_commands.py b/gateway/tests/api/management/test_commands.py index 37440106f..6429a03cb 100644 --- a/gateway/tests/api/management/test_commands.py +++ b/gateway/tests/api/management/test_commands.py @@ -101,13 +101,18 @@ def test_check_long_logs(self): """Test logs checker for very long logs in this case more than 1MB.""" with self.settings( - FUNCTIONS_LOGS_SIZE_LIMIT="1", + FUNCTIONS_LOGS_SIZE_LIMIT="100", ): job = MagicMock() job.id = "42" job.status = "RUNNING" - logs = check_logs(logs=("A" * (1_200_000)), job=job) + log_to_test = "A" * 120 + "B" + logs = check_logs(logs=log_to_test, job=job) self.assertIn( - "Logs exceeded maximum allowed size (1 MB) and could not be stored.", + "[Logs exceeded maximum allowed size (9.5367431640625e-05 MB). Logs have been truncated, discarding the oldest entries first.]", + logs, + ) + self.assertIn( + "AAAAAAAAAAB", logs, )