diff --git a/src/api/endpoints/submit/data_source/models/__init__.py b/src/api/endpoints/submit/data_source/models/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/submit/data_source/models/response/__init__.py b/src/api/endpoints/submit/data_source/models/response/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/submit/data_source/models/response/duplicate.py b/src/api/endpoints/submit/data_source/models/response/duplicate.py new file mode 100644 index 00000000..12367372 --- /dev/null +++ b/src/api/endpoints/submit/data_source/models/response/duplicate.py @@ -0,0 +1,11 @@ +from pydantic import BaseModel + +from src.collectors.enums import URLStatus +from src.db.models.impl.flag.url_validated.enums import URLType + + +class SubmitDataSourceURLDuplicateSubmissionResponse(BaseModel): + message: str + url_id: int + url_type: URLType | None + url_status: URLStatus \ No newline at end of file diff --git a/src/api/endpoints/submit/data_source/response.py b/src/api/endpoints/submit/data_source/models/response/standard.py similarity index 100% rename from src/api/endpoints/submit/data_source/response.py rename to src/api/endpoints/submit/data_source/models/response/standard.py diff --git a/src/api/endpoints/submit/data_source/queries/__init__.py b/src/api/endpoints/submit/data_source/queries/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/api/endpoints/submit/data_source/query.py b/src/api/endpoints/submit/data_source/queries/core.py similarity index 92% rename from src/api/endpoints/submit/data_source/query.py rename to src/api/endpoints/submit/data_source/queries/core.py index 6d7360f5..b3d1ff46 100644 --- a/src/api/endpoints/submit/data_source/query.py +++ b/src/api/endpoints/submit/data_source/queries/core.py @@ -1,9 +1,10 @@ from typing import Any +from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.asyncio import AsyncSession +from src.api.endpoints.submit.data_source.models.response.standard import SubmitDataSourceURLProposalResponse from src.api.endpoints.submit.data_source.request import DataSourceSubmissionRequest -from src.api.endpoints.submit.data_source.response import SubmitDataSourceURLProposalResponse from src.collectors.enums import URLStatus from src.core.enums import BatchStatus from src.db.models.impl.batch.sqlalchemy import Batch @@ -26,14 +27,19 @@ def __init__(self, request: DataSourceSubmissionRequest): super().__init__() self.request = request - async def run(self, session: AsyncSession) -> Any: + async def run( + self, + session: AsyncSession + ) -> SubmitDataSourceURLProposalResponse: full_url = FullURL(full_url=self.request.source_url) + # Begin by attempting to submit the full URL url = URL( url=full_url.id_form, scheme=full_url.scheme, trailing_slash=full_url.has_trailing_slash, name=self.request.name, + description=self.request.description, status=URLStatus.OK, source=URLSource.MANUAL, ) @@ -41,6 +47,7 @@ async def run(self, session: AsyncSession) -> Any: session.add(url) await session.flush() + # Standard Path url_id: int = url.id # Add Batch diff --git a/src/api/endpoints/submit/data_source/queries/duplicate.py b/src/api/endpoints/submit/data_source/queries/duplicate.py new file mode 100644 index 00000000..75346cf6 --- /dev/null +++ b/src/api/endpoints/submit/data_source/queries/duplicate.py @@ -0,0 +1,58 @@ +from http import HTTPStatus + +from fastapi import HTTPException +from sqlalchemy import select, RowMapping +from sqlalchemy.ext.asyncio import AsyncSession + +from src.api.endpoints.submit.data_source.models.response.duplicate import \ + SubmitDataSourceURLDuplicateSubmissionResponse +from src.db.models.impl.flag.url_validated.sqlalchemy import FlagURLValidated +from src.db.models.impl.url.core.sqlalchemy import URL +from src.db.queries.base.builder import QueryBuilderBase + + +class GetDataSourceDuplicateQueryBuilder(QueryBuilderBase): + + def __init__( + self, + url: str + ): + super().__init__() + self.url = url + + async def run(self, session: AsyncSession) -> None: + """ + Raises: + HTTPException including details on the duplicate result. + """ + + query = ( + select( + URL.id, + URL.status, + FlagURLValidated.type + ) + .outerjoin( + FlagURLValidated, + FlagURLValidated.url_id == URL.id + ) + .where( + URL.url == self.url + ) + ) + mapping: RowMapping = await self.sh.mapping( + query=query, + session=session + ) + + model = SubmitDataSourceURLDuplicateSubmissionResponse( + message="Duplicate URL found", + url_id=mapping[URL.id], + url_status=mapping[URL.status], + url_type=mapping[FlagURLValidated.type] + ) + raise HTTPException( + detail=model.model_dump(mode='json'), + status_code=HTTPStatus.CONFLICT + ) + diff --git a/src/api/endpoints/submit/data_source/request.py b/src/api/endpoints/submit/data_source/request.py index 409fe254..fe541923 100644 --- a/src/api/endpoints/submit/data_source/request.py +++ b/src/api/endpoints/submit/data_source/request.py @@ -11,6 +11,7 @@ class DataSourceSubmissionRequest(RequestBase): name: str record_type: RecordType source_url: str + description: str | None = None # Optional URL DS Metadata coverage_start: date | None = None diff --git a/src/api/endpoints/submit/data_source/wrapper.py b/src/api/endpoints/submit/data_source/wrapper.py index 32794150..20e5e158 100644 --- a/src/api/endpoints/submit/data_source/wrapper.py +++ b/src/api/endpoints/submit/data_source/wrapper.py @@ -1,8 +1,10 @@ from fastapi import HTTPException -from src.api.endpoints.submit.data_source.query import SubmitDataSourceURLProposalQueryBuilder +from src.api.endpoints.submit.data_source.models.response.standard import SubmitDataSourceURLProposalResponse +from src.api.endpoints.submit.data_source.queries.core import SubmitDataSourceURLProposalQueryBuilder + +from src.api.endpoints.submit.data_source.queries.duplicate import GetDataSourceDuplicateQueryBuilder from src.api.endpoints.submit.data_source.request import DataSourceSubmissionRequest -from src.api.endpoints.submit.data_source.response import SubmitDataSourceURLProposalResponse from src.db.client.async_ import AsyncDatabaseClient from src.db.queries.urls_exist.model import URLExistsResult from src.db.queries.urls_exist.query import URLsExistInDBQueryBuilder @@ -21,15 +23,18 @@ async def submit_data_source_url_proposal( detail="Invalid URL" ) + full_url = FullURL(request.source_url) + url_exists_results: URLExistsResult = (await adb_client.run_query_builder( URLsExistInDBQueryBuilder( - full_urls=[FullURL(request.source_url)] + full_urls=[full_url] ) ))[0] if url_exists_results.exists: - raise HTTPException( - status_code=400, - detail="URL already exists in database." + await adb_client.run_query_builder( + GetDataSourceDuplicateQueryBuilder( + url=full_url.id_form + ) ) return await adb_client.run_query_builder( diff --git a/src/api/endpoints/submit/routes.py b/src/api/endpoints/submit/routes.py index ee315493..37f4a3c9 100644 --- a/src/api/endpoints/submit/routes.py +++ b/src/api/endpoints/submit/routes.py @@ -1,8 +1,13 @@ from fastapi import APIRouter, Depends from src.api.dependencies import get_async_core -from src.api.endpoints.submit.data_source.query import SubmitDataSourceURLProposalQueryBuilder + +from src.api.endpoints.submit.data_source.models.response.duplicate import \ + SubmitDataSourceURLDuplicateSubmissionResponse +from src.api.endpoints.submit.data_source.models.response.standard import SubmitDataSourceURLProposalResponse +from src.api.endpoints.submit.data_source.queries.core import SubmitDataSourceURLProposalQueryBuilder from src.api.endpoints.submit.data_source.request import DataSourceSubmissionRequest +from src.api.endpoints.submit.data_source.wrapper import submit_data_source_url_proposal from src.api.endpoints.submit.url.models.request import URLSubmissionRequest from src.api.endpoints.submit.url.models.response import URLSubmissionResponse from src.api.endpoints.submit.url.queries.core import SubmitURLQueryBuilder @@ -12,7 +17,9 @@ submit_router = APIRouter(prefix="/submit", tags=["submit"]) -@submit_router.post("/url") +@submit_router.post( + "/url" +) async def submit_url( request: URLSubmissionRequest, access_info: AccessInfo = Depends(get_access_info), @@ -25,13 +32,20 @@ async def submit_url( ) ) -@submit_router.post("/data-source") +@submit_router.post( + "/data-source", + response_model=SubmitDataSourceURLProposalResponse, + responses={ + 409: { + "model": SubmitDataSourceURLDuplicateSubmissionResponse + } + } +) async def submit_data_source( request: DataSourceSubmissionRequest, async_core: AsyncCore = Depends(get_async_core), ): - return await async_core.adb_client.run_query_builder( - SubmitDataSourceURLProposalQueryBuilder( - request=request, - ) + return await submit_data_source_url_proposal( + request=request, + adb_client=async_core.adb_client ) diff --git a/src/api/endpoints/submit/url/models/request.py b/src/api/endpoints/submit/url/models/request.py index 34ec9df9..4e5656b0 100644 --- a/src/api/endpoints/submit/url/models/request.py +++ b/src/api/endpoints/submit/url/models/request.py @@ -9,4 +9,5 @@ class URLSubmissionRequest(RequestBase): record_type: RecordType | None = None name: str | None = None location_id: int | None = None - agency_id: int | None = None \ No newline at end of file + agency_id: int | None = None + description: str | None = None \ No newline at end of file diff --git a/src/api/endpoints/submit/url/queries/core.py b/src/api/endpoints/submit/url/queries/core.py index 9f3e7117..0d2c1c84 100644 --- a/src/api/endpoints/submit/url/queries/core.py +++ b/src/api/endpoints/submit/url/queries/core.py @@ -62,6 +62,7 @@ async def run(self, session: AsyncSession) -> URLSubmissionResponse: scheme=url_and_scheme.scheme, source=URLSource.MANUAL, status=URLStatus.OK, + description=self.request.description, trailing_slash=url_and_scheme.url.endswith('/'), ) session.add(url_insert) diff --git a/tests/automated/integration/api/submit/data_source/test_core.py b/tests/automated/integration/api/submit/data_source/test_core.py index 49df1dd4..eed0cd00 100644 --- a/tests/automated/integration/api/submit/data_source/test_core.py +++ b/tests/automated/integration/api/submit/data_source/test_core.py @@ -32,6 +32,7 @@ async def test_submit_data_source( json=DataSourceSubmissionRequest( source_url="https://example.com/", name="Example name", + description="Example description", record_type=RecordType.COMPLAINTS_AND_MISCONDUCT, coverage_start=date(year=2025, month=8, day=9), coverage_end=date(year=2025, month=8, day=10), @@ -74,6 +75,7 @@ async def test_submit_data_source( assert url.trailing_slash == True assert url.source == URLSource.MANUAL assert url.status == URLStatus.OK + assert url.description == "Example description" # Check for Batch batch: Batch = await adb_client.one_or_none_model(Batch) diff --git a/tests/automated/integration/api/submit/data_source/test_duplicate.py b/tests/automated/integration/api/submit/data_source/test_duplicate.py new file mode 100644 index 00000000..ea16e1ec --- /dev/null +++ b/tests/automated/integration/api/submit/data_source/test_duplicate.py @@ -0,0 +1,38 @@ +import pytest +from fastapi import HTTPException + +from src.api.endpoints.submit.data_source.models.response.duplicate import SubmitDataSourceURLDuplicateSubmissionResponse +from src.api.endpoints.submit.data_source.request import DataSourceSubmissionRequest +from src.collectors.enums import URLStatus +from src.core.enums import RecordType +from src.db.dtos.url.mapping_.simple import SimpleURLMapping +from src.db.models.impl.flag.url_validated.enums import URLType +from tests.helpers.api_test_helper import APITestHelper +from tests.helpers.data_creator.models.creation_info.locality import LocalityCreationInfo + + +@pytest.mark.asyncio +async def test_submit_data_source_duplicate( + api_test_helper: APITestHelper, + test_agency_id: int, + pittsburgh_locality: LocalityCreationInfo, + test_url_data_source_mapping: SimpleURLMapping +): + + ath = api_test_helper + try: + ath.request_validator.post_v3( + url="submit/data-source", + json=DataSourceSubmissionRequest( + source_url=test_url_data_source_mapping.url, + name="Test Name", + record_type=RecordType.RECORDS_REQUEST_INFO + ).model_dump(mode='json') + ) + except HTTPException as e: + response = e.detail['detail'] + model = SubmitDataSourceURLDuplicateSubmissionResponse(**response) + assert model.url_id == test_url_data_source_mapping.url_id + assert model.url_type == URLType.DATA_SOURCE + assert model.url_status == URLStatus.OK + assert model.message == "Duplicate URL found" diff --git a/tests/automated/integration/api/submit/test_url_maximal.py b/tests/automated/integration/api/submit/test_url_maximal.py index 150b5409..e57770fb 100644 --- a/tests/automated/integration/api/submit/test_url_maximal.py +++ b/tests/automated/integration/api/submit/test_url_maximal.py @@ -32,6 +32,7 @@ async def test_maximal( request=URLSubmissionRequest( url="www.example.com", record_type=RecordType.INCARCERATION_RECORDS, + description="Example description", name="Example URL", location_id=pittsburgh_locality.location_id, agency_id=agency_id, @@ -48,6 +49,7 @@ async def test_maximal( url: URL = urls[0] assert url.id == url_id assert url.url == "www.example.com" + assert url.description == "Example description" links: list[LinkUserSubmittedURL] = await adb_client.get_all(LinkUserSubmittedURL) assert len(links) == 1 diff --git a/tests/automated/integration/conftest.py b/tests/automated/integration/conftest.py index 6837bae0..6e2be0f0 100644 --- a/tests/automated/integration/conftest.py +++ b/tests/automated/integration/conftest.py @@ -12,6 +12,7 @@ from src.core.logger import AsyncCoreLogger from src.db.client.async_ import AsyncDatabaseClient from src.db.client.sync import DatabaseClient +from src.db.dtos.url.mapping_.simple import SimpleURLMapping from src.db.models.impl.flag.url_validated.enums import URLType from src.security.dtos.access_info import AccessInfo from src.security.enums import Permissions @@ -217,6 +218,21 @@ async def test_url_data_source_id( ) return url_id +@pytest_asyncio.fixture +async def test_url_data_source_mapping( + db_data_creator: DBDataCreator, + test_agency_id: int +) -> SimpleURLMapping: + url_mapping: SimpleURLMapping = (await db_data_creator.create_validated_urls( + record_type=RecordType.CRIME_STATISTICS, + validation_type=URLType.DATA_SOURCE, + ))[0] + await db_data_creator.link_urls_to_agencies( + url_ids=[url_mapping.url_id], + agency_ids=[test_agency_id] + ) + return url_mapping + @pytest_asyncio.fixture async def test_url_meta_url_id( db_data_creator: DBDataCreator,