datopian · pdelboca · Jan 3, 2023 · Jan 3, 2023 · Jan 3, 2023 · Jan 4, 2023
diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml
@@ -9,61 +9,50 @@ on:
 
 jobs:
   test:
+    strategy:
+      matrix:
+        ckan-version: ["2.9", "2.10"]
+      fail-fast: false
+
+    name: CKAN ${{ matrix.ckan-version }}
     runs-on: ubuntu-20.04
 
+    container:
+      image: openknowledge/ckan-dev:${{ matrix.ckan-version }}
     services:
+      solr:
+        image: ckan/ckan-solr:${{ matrix.ckan-version }}
       postgres:
-        image: postgres:11-alpine
+        image: ckan/ckan-postgres-dev:${{ matrix.ckan-version }}
         env:
-          POSTGRES_PASSWORD: ckan
-          POSTGRES_USER: ckan
-          POSTGRES_DB: ckan
-        options: >-
-          --health-cmd pg_isready
-          --health-interval 5s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 5432:5432
-
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
       redis:
-        image: redis
-        options: >-
-          --health-cmd "redis-cli ping"
-          --health-interval 5s
-          --health-timeout 5s
-          --health-retries 5
-        ports:
-          - 6379:6379
-
-      solr:
-        image: ckan/solr
-        env:
-          CKAN_SOLR_PASSWORD: ckan
-        ports:
-          - 8983:8983
-
-    strategy:
-      matrix:
-        python-version: [ 2.7, 3.7 ]
-        ckan-version: [ 2.8, 2.9 ]
-        exclude:
-          - python-version: 3.7
-            ckan-version: 2.8
+          image: redis:3
     env:
-      CKAN_PATH: ./ckan
-      VIRTUAL_ENV: "1"  # let's fake a virtual environment, why not
+      CKAN_SQLALCHEMY_URL: postgresql://ckan_default:pass@postgres/ckan_test
+      CKAN_DATASTORE_WRITE_URL: postgresql://datastore_write:pass@postgres/datastore_test
+      CKAN_DATASTORE_READ_URL: postgresql://datastore_read:pass@postgres/datastore_test
+      CKAN_SOLR_URL: http://solr:8983/solr/ckan
+      CKAN_REDIS_URL: redis://redis:6379/1
 
     steps:
-      - uses: actions/checkout@v2
-      - name: Install Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python-version }}
-      - name: Set up the test environment
-        run: |
-          make ckan-install CKAN_VERSION=${{ matrix.ckan-version }}
-          make create-test-db
-          make dev-setup
-      - name: Run tests
-        run: make test
+    - uses: actions/checkout@v3
+    - name: Install requirements
+      run: |
+        pip install -r dev-requirements.txt
+        pip install -r requirements.txt
+        # Replace default path to CKAN core config file with the one on the container
+        sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
+    - name: Setup ckan.plugins for CKAN 2.9
+      if: ${{ matrix.ckan-version == '2.9' }}
+      run: |
+        ckan config-tool test.ini ckan.plugins=blob_storage
+    - name: Setup extension
+      run: |
+        pip install -e .
+        ckan -c test.ini db init
+    - name: Run tests
+      run: pytest --ckan-ini=test.ini --disable-warnings ckanext/blob_storage/tests
diff --git a/Makefile b/Makefile
@@ -71,16 +71,6 @@ else
 endif
 
 
-dev-requirements.%.txt: dev-requirements.in
-	$(PIP_COMPILE) --no-index dev-requirements.in -o $@
-
-requirements.%.txt: requirements.in
-	$(PIP_COMPILE) --no-index requirements.in -o $@
-
-## Update requirements files for the current Python version
-requirements: $(SENTINELS)/requirements
-.PHONEY: requirements
-
 ## Install this extension to the current Python environment
 install: $(SENTINELS)/install
 .PHONY: install

diff --git a/ckanext/blob_storage/__init__.py b/ckanext/blob_storage/__init__.py
@@ -1 +0,0 @@
-__version__ = '0.8.2'

diff --git a/ckanext/blob_storage/actions.py b/ckanext/blob_storage/actions.py
@@ -7,7 +7,6 @@
 from ckan.plugins import toolkit
 from giftless_client import LfsClient
 from giftless_client.exc import LfsError
-from six import ensure_text
 
 from . import helpers
 
@@ -153,7 +152,10 @@ def get_download_authz_token(context, org_name, package_name, resource_id, activ
     if len(authz_result['granted_scopes']) == 0:
         raise toolkit.NotAuthorized("You are not authorized to download this resource")
 
-    return ensure_text(authz_result['token'])
+    if not isinstance(authz_result['token'], bytes):
+        raise TypeError("Expecting token of type bytes not '%s'" % type(authz_result['token']))
+
+    return authz_result["token"].decode('utf-8')
 
 
 def _get_resource(context, data_dict):

diff --git a/ckanext/blob_storage/cli.py b/ckanext/blob_storage/cli.py
@@ -15,7 +15,6 @@
 from flask import Response
 from giftless_client import LfsClient
 from giftless_client.types import ObjectAttributes
-from six import binary_type, string_types
 from sqlalchemy.orm import load_only
 from sqlalchemy.orm.attributes import flag_modified
 from werkzeug.wsgi import FileWrapper
@@ -158,7 +157,7 @@ def _save_downloaded_response_data(response, file_name):
     to a temporary file
     """
     with open(file_name, 'wb') as f:
-        if isinstance(response.response, (string_types, binary_type)):
+        if isinstance(response.response, (str, bytes)):
             _log().debug("Response contains inline string data, saving to %s", file_name)
             f.write(response.response)
         elif isinstance(response.response, FileWrapper):

diff --git a/ckanext/blob_storage/helpers.py b/ckanext/blob_storage/helpers.py
@@ -4,7 +4,7 @@
 from typing import Any, Dict, Optional
 
 import ckan.plugins.toolkit as toolkit
-from six.moves.urllib.parse import urlparse
+from urllib.parse import urlparse
 
 SERVER_URL_CONF_KEY = 'ckanext.blob_storage.storage_service_url'
 STORAGE_NAMESPACE_CONF_KEY = 'ckanext.blob_storage.storage_namespace'

diff --git a/ckanext/blob_storage/tests/__init__.py b/ckanext/blob_storage/tests/__init__.py
@@ -3,7 +3,7 @@
 
 from ckan import model
 from ckan.tests import helpers
-from mock import patch
+from unittest.mock import patch
 
 
 class FunctionalTestBase(helpers.FunctionalTestBase):

diff --git a/ckanext/blob_storage/tests/test_blueprint.py b/ckanext/blob_storage/tests/test_blueprint.py
@@ -1,7 +1,8 @@
-import mock
 import pytest
+
 from ckan.plugins import toolkit
 from ckan.tests import factories
+from unittest import mock
 
 
 @pytest.mark.usefixtures('clean_db')

diff --git a/dev-requirements.in b/dev-requirements.in
diff --git a/dev-requirements.py2.txt b/dev-requirements.py2.txt
diff --git a/dev-requirements.py3.txt b/dev-requirements.py3.txt
diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -0,0 +1 @@
+pytest-ckan
diff --git a/requirements.in b/requirements.in
diff --git a/requirements.py2.txt b/requirements.py2.txt
diff --git a/requirements.py3.txt b/requirements.py3.txt
diff --git a/requirements.txt b/requirements.txt
@@ -0,0 +1,4 @@
+typing_extensions
+giftless-client
+-r https://raw.githubusercontent.com/datopian/ckanext-authz-service/d179b19469faff7e461191db53e31c60d19003bc/requirements.in
+git+https://github.com/datopian/ckanext-authz-service.git@d179b19469faff7e461191db53e31c60d19003bc#egg=ckanext-authz-service