From db629647a1fd29f5cbeb8f1d277f057fd31ba5e0 Mon Sep 17 00:00:00 2001 From: Jan Range <30547301+JR-1991@users.noreply.github.com> Date: Wed, 24 Sep 2025 14:13:45 +0200 Subject: [PATCH 01/12] Update Docker Compose files and test workflow Standardized quoting in docker-compose-base.yml and removed unused version fields. Enhanced unit test service in docker-compose-test-all.yml to dynamically fetch Dataverse version, install required dependencies, and improve readiness checks for Dataverse before running tests. --- docker/docker-compose-base.yml | 47 +++++++++++++++--------------- docker/docker-compose-test-all.yml | 35 +++++++++++++++++----- 2 files changed, 50 insertions(+), 32 deletions(-) diff --git a/docker/docker-compose-base.yml b/docker/docker-compose-base.yml index 196a1a3..d3d8f66 100644 --- a/docker/docker-compose-base.yml +++ b/docker/docker-compose-base.yml @@ -1,8 +1,7 @@ -version: "2.4" name: pydataverse services: dataverse: - container_name: "dataverse" + container_name: 'dataverse' hostname: dataverse image: ${DATAVERSE_IMAGE} restart: on-failure @@ -18,7 +17,7 @@ services: -Ddataverse.pid.fake.authority=10.5072 -Ddataverse.pid.fake.shoulder=FK2/ ports: - - "8080:8080" + - '8080:8080' networks: - dataverse depends_on: @@ -45,18 +44,18 @@ services: timeout: 240s dv_initializer: - container_name: "dv_initializer" + container_name: 'dv_initializer' image: ${CONFIGBAKER_IMAGE} - restart: "no" + restart: 'no' command: - sh - -c - - "fix-fs-perms.sh dv" + - 'fix-fs-perms.sh dv' volumes: - ${PWD}/dv/data:/dv postgres: - container_name: "postgres" + container_name: 'postgres' hostname: postgres image: postgres:${POSTGRES_VERSION} restart: on-failure @@ -64,49 +63,49 @@ services: - POSTGRES_USER=${DATAVERSE_DB_USER} - POSTGRES_PASSWORD=${DATAVERSE_DB_PASSWORD} ports: - - "5432:5432" + - '5432:5432' networks: - dataverse solr_initializer: - container_name: "solr_initializer" + container_name: 'solr_initializer' image: ${CONFIGBAKER_IMAGE} - restart: "no" + restart: 'no' command: - sh - -c - - "fix-fs-perms.sh solr && cp -a /template/* /solr-template" + - 'fix-fs-perms.sh solr && cp -a /template/* /solr-template' volumes: - ${PWD}/solr/data:/var/solr - ${PWD}/solr/conf:/solr-template solr: - container_name: "solr" - hostname: "solr" + container_name: 'solr' + hostname: 'solr' image: solr:${SOLR_VERSION} depends_on: solr_initializer: condition: service_completed_successfully restart: on-failure ports: - - "8983:8983" + - '8983:8983' networks: - dataverse command: - - "solr-precreate" - - "collection1" - - "/template" + - 'solr-precreate' + - 'collection1' + - '/template' volumes: - ${PWD}/solr/data:/var/solr - ${PWD}/solr/conf:/template smtp: - container_name: "smtp" - hostname: "smtp" + container_name: 'smtp' + hostname: 'smtp' image: maildev/maildev:2.0.5 restart: on-failure expose: - - "25" # smtp server + - '25' # smtp server environment: - MAILDEV_SMTP_PORT=25 - MAILDEV_MAIL_DIRECTORY=/mail @@ -116,10 +115,10 @@ services: - /mail:mode=770,size=128M,uid=1000,gid=1000 bootstrap: - container_name: "bootstrap" - hostname: "bootstrap" + container_name: 'bootstrap' + hostname: 'bootstrap' image: ${CONFIGBAKER_IMAGE} - restart: "no" + restart: 'no' networks: - dataverse volumes: @@ -127,7 +126,7 @@ services: command: - sh - -c - - "bootstrap.sh -e /.env dev" + - 'bootstrap.sh -e /.env dev' depends_on: dataverse: condition: service_healthy diff --git a/docker/docker-compose-test-all.yml b/docker/docker-compose-test-all.yml index 0e357a9..6d9643e 100644 --- a/docker/docker-compose-test-all.yml +++ b/docker/docker-compose-test-all.yml @@ -1,11 +1,9 @@ -version: "2.4" services: unit-tests: container_name: unit-tests image: python:${PYTHON_VERSION}-slim environment: BASE_URL: http://dataverse:8080 - DV_VERSION: 6.3 networks: - dataverse volumes: @@ -13,18 +11,39 @@ services: - ../dv:/dv command: - sh - - -c + - -ceu - | + set -o pipefail + + # Install curl + apt-get update && apt-get install -y curl + + # Wait for Dataverse to be ready + echo "โณ Waiting for Dataverse to be ready..." + until curl -f http://dataverse:8080/api/info/version >/dev/null 2>&1; do + echo " Waiting for Dataverse API..." + sleep 5 + done + echo "โœ… Dataverse is ready!" + + # Fetch Dataverse version dynamically + echo "๐Ÿ” Fetching Dataverse version..." + export DV_VERSION=$$(curl -s http://dataverse:8080/api/info/version | grep -o '"version":"[^"]*"' | cut -d'"' -f4) + echo " Detected Dataverse version: $$DV_VERSION" + # Fetch the API Token from the local file - export $(grep "API_TOKEN" "dv/bootstrap.exposed.env") + export $$(grep "API_TOKEN" "dv/bootstrap.exposed.env") export API_TOKEN_SUPERUSER=$$API_TOKEN + cd /pydataverse - # Run the unit tests + # Install Poetry and dependencies python3 -m pip install --upgrade pip - python3 -m pip install pytest pytest-cov - python3 -m pip install -e . - python3 -m pytest > /dv/unit-tests.log + python3 -m pip install . + python3 -m pip install pytest pytest-cov pytest-asyncio tox selenium + + echo "๐Ÿงช Starting pytest with DV_VERSION=${DV_VERSION}..." + python3 -m pytest -vv --tb=short 2>&1 | tee /dv/unit-tests.log depends_on: bootstrap: From f9b7434b9672941dd5456b738340847ea45d75ee Mon Sep 17 00:00:00 2001 From: Jan Range <30547301+JR-1991@users.noreply.github.com> Date: Wed, 24 Sep 2025 14:13:52 +0200 Subject: [PATCH 02/12] Add default Dataverse version to env file Introduces DV_VERSION=6.7.1 to local-test.env for use as the default Dataverse version, which will be dynamically detected in tests. --- local-test.env | 3 +++ 1 file changed, 3 insertions(+) diff --git a/local-test.env b/local-test.env index 1eeeaa0..3ea9dd7 100644 --- a/local-test.env +++ b/local-test.env @@ -7,3 +7,6 @@ CONFIGBAKER_IMAGE=docker.io/gdcc/configbaker:unstable # Services POSTGRES_VERSION=15 SOLR_VERSION=9.3.0 + +# Default Dataverse version (will be dynamically detected in tests) +DV_VERSION=6.7.1 From 80ae1fbf99ba96541c5b45d05ff922bb334d7b3a Mon Sep 17 00:00:00 2001 From: Jan Range <30547301+JR-1991@users.noreply.github.com> Date: Wed, 24 Sep 2025 14:14:03 +0200 Subject: [PATCH 03/12] Improve test container handling and output in run-tests.sh Enhanced messaging and status reporting for test containers, including better wait feedback, exit code handling, and conditional log output. The script now distinguishes between missing log files and container logs, and provides clearer start/stop notifications for containers. --- run-tests.sh | 48 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) diff --git a/run-tests.sh b/run-tests.sh index fe346d4..b6b2c1b 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -44,7 +44,10 @@ export PYTHON_VERSION=${p} printf "\n๐Ÿš€ Preparing containers\n" printf " Using PYTHON_VERSION=${p}\n\n" -# Run all containers +# Start all containers (infrastructure + tests) +printf "\n๐Ÿš€ Starting all containers...\n" +printf " The test container will wait for Dataverse and fetch the version automatically\n\n" + docker compose \ -f docker/docker-compose-base.yml \ -f ./docker/docker-compose-test-all.yml \ @@ -52,19 +55,37 @@ docker compose \ up -d printf "\n๐Ÿ”Ž Running pyDataverse tests\n" -printf " Logs will be printed once finished...\n\n" +printf " Test container will handle version detection and testing...\n\n" # Check if "unit-test" container has finished +WAIT_COUNT=0 while [ -n "$(docker ps -f "name=unit-tests" -f "status=running" -q)" ]; do - printf " Waiting for unit-tests container to finish...\n" + WAIT_COUNT=$((WAIT_COUNT + 1)) + printf " โณ Waiting for tests to complete... (${WAIT_COUNT})\n" sleep 5 done # Check if "unit-test" container has failed -if [ "$(docker inspect -f '{{.State.ExitCode}}' unit-tests)" -ne 0 ]; then - printf "\nโŒ Unit tests failed. Printing logs...\n" - docker logs unit-tests - printf "\n Stopping containers\n" +EXIT_CODE=$(docker inspect -f '{{.State.ExitCode}}' unit-tests 2>/dev/null) +if [ -z "$EXIT_CODE" ]; then + printf "\nโŒ Unit tests container not found or failed to start\n" + EXIT_CODE=1 +else + printf "\n๐Ÿ“‹ Unit tests completed with exit code: ${EXIT_CODE}\n" +fi + +if [ "${EXIT_CODE}" -ne 0 ]; then + printf "\nโŒ Unit tests failed. Showing test results...\n\n" + printf "=== PYTEST OUTPUT ===\n" + if [ -f "dv/unit-tests.log" ]; then + cat dv/unit-tests.log + else + printf "โš ๏ธ Test log file not found, showing container logs instead:\n" + docker logs unit-tests + fi + printf "\n=== END PYTEST OUTPUT ===\n" + + printf "\n๐Ÿงน Stopping containers...\n" docker compose \ -f docker/docker-compose-base.yml \ -f ./docker/docker-compose-test-all.yml \ @@ -74,11 +95,18 @@ if [ "$(docker inspect -f '{{.State.ExitCode}}' unit-tests)" -ne 0 ]; then fi # Print test results -printf "\n" -cat dv/unit-tests.log -printf "\n\nโœ… Unit tests passed\n\n" +printf "\nโœ… Unit tests passed! Showing results...\n\n" +printf "=== PYTEST RESULTS ===\n" +if [ -f "dv/unit-tests.log" ]; then + cat dv/unit-tests.log +else + printf "โš ๏ธ Test log file not found, showing container logs:\n" + docker logs unit-tests +fi +printf "\n=== END PYTEST RESULTS ===\n" # Stop all containers +printf "\n๐Ÿงน Stopping containers...\n" docker compose \ -f docker/docker-compose-base.yml \ -f ./docker/docker-compose-test-all.yml \ From aab9a9355b007612be85b3f5ea21417ea62b25b6 Mon Sep 17 00:00:00 2001 From: Jan Range <30547301+JR-1991@users.noreply.github.com> Date: Wed, 24 Sep 2025 16:10:29 +0200 Subject: [PATCH 04/12] Add response status checks to upload tests Inserted response.raise_for_status() calls after API requests in test_upload.py to ensure HTTP errors are caught during testing. Also refactored some assert statements for improved readability. --- tests/api/test_upload.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/api/test_upload.py b/tests/api/test_upload.py index e8d8fef..19b9b5b 100644 --- a/tests/api/test_upload.py +++ b/tests/api/test_upload.py @@ -161,6 +161,8 @@ def test_file_replacement_wo_metadata(self): json_str=df.json(), ) + response.raise_for_status() + # Retrieve file ID file_id = response.json()["data"]["files"][0]["dataFile"]["id"] @@ -182,6 +184,8 @@ def test_file_replacement_wo_metadata(self): is_filepid=False, ) + response.raise_for_status() + # Assert file_id = response.json()["data"]["files"][0]["dataFile"]["id"] content = data_api.get_datafile(file_id, is_pid=False).text From 54000dc1888fdca71fcc980c6816ebe2242517e7 Mon Sep 17 00:00:00 2001 From: Jan Range <30547301+JR-1991@users.noreply.github.com> Date: Wed, 24 Sep 2025 16:41:17 +0200 Subject: [PATCH 05/12] Improve error handling in API and upload tests Replaced status assertions with explicit error handling in test_api.py and test_upload.py. Now exceptions are raised with API error messages when responses are not OK, improving test reliability and clarity. --- tests/api/test_api.py | 14 +++++++++++--- tests/api/test_upload.py | 9 ++++++++- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/tests/api/test_api.py b/tests/api/test_api.py index 817c4b3..fbdb148 100644 --- a/tests/api/test_api.py +++ b/tests/api/test_api.py @@ -174,14 +174,22 @@ def test_token_right_create_dataset_rights(self): ) ) resp = api_su.create_dataset(":root", ds.json()) - pid = resp.json()["data"]["persistentId"] - assert resp.json()["status"] == "OK" + resp.raise_for_status() + + resp = resp.json() + pid = resp["data"]["persistentId"] + + if resp["status"] != "OK": + raise Exception(resp["message"]) # with pytest.raises(ApiAuthorizationError): # resp = api_nru.get_dataset(pid) resp = api_su.delete_dataset(pid) - assert resp.json()["status"] == "OK" + resp.raise_for_status() + resp = resp.json() + if resp["status"] != "OK": + raise Exception(resp["message"]) def test_token_should_not_be_exposed_on_error(self): BASE_URL = os.getenv("BASE_URL") diff --git a/tests/api/test_upload.py b/tests/api/test_upload.py index 19b9b5b..0054fc2 100644 --- a/tests/api/test_upload.py +++ b/tests/api/test_upload.py @@ -184,7 +184,8 @@ def test_file_replacement_wo_metadata(self): is_filepid=False, ) - response.raise_for_status() + if not response.ok: + raise Exception(response.json()["message"]) # Assert file_id = response.json()["data"]["files"][0]["dataFile"]["id"] @@ -222,6 +223,9 @@ def test_file_replacement_w_metadata(self): json_str=df.json(), ) + if not response.ok: + raise Exception(response.json()["message"]) + # Retrieve file ID file_id = response.json()["data"]["files"][0]["dataFile"]["id"] @@ -248,6 +252,9 @@ def test_file_replacement_w_metadata(self): is_filepid=False, ) + if not response.ok: + raise Exception(response.json()["message"]) + # Assert file_id = response.json()["data"]["files"][0]["dataFile"]["id"] data_file = api.get_dataset(pid).json()["data"]["latestVersion"]["files"][0] From 51ca1a7c820e2d6de0a4fb2393b7488a5440485f Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Wed, 24 Sep 2025 10:53:25 -0400 Subject: [PATCH 06/12] install test deps --- README.md | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c8bb1de..cd4ba81 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,16 @@ export $(grep "API_TOKEN" "dv/bootstrap.exposed.env") export API_TOKEN_SUPERUSER=$API_TOKEN ``` -**3. Run the test(s) with pytest** +**3. Install test dependencies + +``` +python3 -m venv venv # or conda create -n pydataverse +source venv/bin/activate +pip install -e . +python3 -m pip install pytest pytest-cov pytest-asyncio tox selenium +``` + +**4. Run the test(s) with pytest** ```bash python -m pytest -v From fc7c58a0ea0e1e21e48e94ffb59847910105b72a Mon Sep 17 00:00:00 2001 From: Jan Range <30547301+JR-1991@users.noreply.github.com> Date: Wed, 24 Sep 2025 18:37:32 +0200 Subject: [PATCH 07/12] Update response checks and assertions in upload tests Replaces usage of 'response.ok' with 'response.is_success' for response validation. Refactors assertions to use single-line format with error messages as arguments for improved readability. --- tests/api/test_upload.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/api/test_upload.py b/tests/api/test_upload.py index 0054fc2..e8d53c2 100644 --- a/tests/api/test_upload.py +++ b/tests/api/test_upload.py @@ -184,7 +184,7 @@ def test_file_replacement_wo_metadata(self): is_filepid=False, ) - if not response.ok: + if not response.is_success: raise Exception(response.json()["message"]) # Assert @@ -223,7 +223,7 @@ def test_file_replacement_w_metadata(self): json_str=df.json(), ) - if not response.ok: + if not response.is_success: raise Exception(response.json()["message"]) # Retrieve file ID @@ -252,7 +252,7 @@ def test_file_replacement_w_metadata(self): is_filepid=False, ) - if not response.ok: + if not response.is_success: raise Exception(response.json()["message"]) # Assert From 57d0cda92c004536556fc04dd1575e59f66fce91 Mon Sep 17 00:00:00 2001 From: Jan Range <30547301+JR-1991@users.noreply.github.com> Date: Wed, 24 Sep 2025 18:52:32 +0200 Subject: [PATCH 08/12] Update README with Dataverse log instructions Added information about the location of Dataverse logs and guidance for sharing logs on Zulip in case of internal server errors. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cd4ba81..a4f3fa4 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ In order to run the tests, you need to have a Dataverse instance running. We hav ./run_tests.sh -p 3.8 ``` -Once finished, you can find the test results in the `dv/unit-tests.log` file and in the terminal. +Once finished, you can find the test results in the `dv/unit-tests.log` file and in the terminal. The Dataverse logs are saved in the `dv/dataverse-logs.log` file and can be helpful in cases of internal server errors. If you happen to encounter such an error, please share the logs with us at the [Zulip](https://dataverse.zulipchat.com/#narrow/stream/377090-python) so we can investigate the issue. ## Manual setup From 501e55b9ce0823c621833b72617d018d973970f0 Mon Sep 17 00:00:00 2001 From: Jan Range <30547301+JR-1991@users.noreply.github.com> Date: Wed, 24 Sep 2025 18:52:39 +0200 Subject: [PATCH 09/12] Capture Dataverse logs after test run Adds a step to save Dataverse container logs to dv/dataverse-logs.log after unit tests complete, aiding in debugging and post-test analysis. --- run-tests.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/run-tests.sh b/run-tests.sh index b6b2c1b..3e34bdc 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -67,6 +67,7 @@ done # Check if "unit-test" container has failed EXIT_CODE=$(docker inspect -f '{{.State.ExitCode}}' unit-tests 2>/dev/null) + if [ -z "$EXIT_CODE" ]; then printf "\nโŒ Unit tests container not found or failed to start\n" EXIT_CODE=1 @@ -74,6 +75,10 @@ else printf "\n๐Ÿ“‹ Unit tests completed with exit code: ${EXIT_CODE}\n" fi +# Capture Dataverse logs after tests complete +printf "\n๐Ÿ“ Capturing Dataverse logs...\n" +docker logs dataverse > dv/dataverse-logs.log 2>&1 + if [ "${EXIT_CODE}" -ne 0 ]; then printf "\nโŒ Unit tests failed. Showing test results...\n\n" printf "=== PYTEST OUTPUT ===\n" @@ -84,6 +89,7 @@ if [ "${EXIT_CODE}" -ne 0 ]; then docker logs unit-tests fi printf "\n=== END PYTEST OUTPUT ===\n" + printf "\n๐Ÿงน Stopping containers...\n" docker compose \ From 3679d5f145a3471efc1686f3f7c19fa7e2e73d60 Mon Sep 17 00:00:00 2001 From: Jan Range <30547301+JR-1991@users.noreply.github.com> Date: Wed, 24 Sep 2025 18:52:44 +0200 Subject: [PATCH 10/12] Update dv volume path in test compose file Changed the dv volume mount in docker-compose-test-all.yml to use an absolute path based on ${PWD} for improved reliability and consistency. --- docker/docker-compose-test-all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose-test-all.yml b/docker/docker-compose-test-all.yml index 6d9643e..7305e8e 100644 --- a/docker/docker-compose-test-all.yml +++ b/docker/docker-compose-test-all.yml @@ -8,7 +8,7 @@ services: - dataverse volumes: - ${PWD}:/pydataverse - - ../dv:/dv + - ${PWD}/dv:/dv command: - sh - -ceu From 4a08f0436d3307e36823ea228cea5d5f2d5a53c7 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 10 Oct 2025 10:34:00 -0400 Subject: [PATCH 11/12] stop mounting secrets (not needed) --- docker/docker-compose-base.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/docker-compose-base.yml b/docker/docker-compose-base.yml index d3d8f66..f28ca67 100644 --- a/docker/docker-compose-base.yml +++ b/docker/docker-compose-base.yml @@ -29,7 +29,6 @@ services: condition: service_completed_successfully volumes: - ${PWD}/dv/data:/dv - - ${PWD}:/secrets tmpfs: - /dumps:mode=770,size=2052M,uid=1000,gid=1000 - /tmp:mode=770,size=2052M,uid=1000,gid=1000 From a47ef54e6f41d5df1e0bffe0f020e343cd70a93e Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Fri, 10 Oct 2025 10:39:39 -0400 Subject: [PATCH 12/12] fix solr.NumFieldLimitingUpdateRequestProcessorFactory error --- local-test.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/local-test.env b/local-test.env index 3ea9dd7..3a5fc11 100644 --- a/local-test.env +++ b/local-test.env @@ -6,7 +6,7 @@ CONFIGBAKER_IMAGE=docker.io/gdcc/configbaker:unstable # Services POSTGRES_VERSION=15 -SOLR_VERSION=9.3.0 +SOLR_VERSION=9.8.0 # Default Dataverse version (will be dynamically detected in tests) DV_VERSION=6.7.1