diff --git a/README.md b/README.md index da9d2020..094ce6f7 100644 --- a/README.md +++ b/README.md @@ -9,11 +9,9 @@ Code Yellow backend framework for SPA webapps with REST-like API. ## Running the tests -There are two ways to run the tests: -- Run directly `./setup.py test` (requires you to have python3 and postgres installed) -- Run with docker `docker-compose run binder ./setup.py test` - - Access the test database directly by with `docker-compose run db psql -h db -U postgres`. - - It may be possible to recreate the test database (for example when you added/changed models). One way of achieving this is to just remove all the docker images that were build `docker-compose rm`. The database will be created during the setup in `tests/__init__.py`. +Run with docker `docker compose run binder ./setup.py test` (but you may need to run `docker compose build db binder` first) +- Access the test database directly by with `docker compose run db psql -h db -U postgres`. +- It may be possible to recreate the test database (for example when you added/changed models). One way of achieving this is to just remove all the docker images that were build `docker compose rm`. The database will be created during the setup in `tests/__init__.py`. The tests are set up in such a way that there is no need to keep migration files. The setup procedure in `tests/__init__.py` handles the preparation of the database by directly calling some build-in Django commands. @@ -21,19 +19,4 @@ To only run a selection of the tests, use the `-s` flag like `./setup.py test -s ## MySQL support -MySQL is supported, but only with the goal to replace it with -PostgreSQL. This means it has a few limitations: - -- `where` filtering on `with` relations is not supported. -- Only integer primary keys are supported. -- When fetching large number of records using `with` or the ids are big, be sure to increase `GROUP_CONCAT` max string length by: - -``` -DATABASES = { - 'default': { - 'OPTIONS': { - 'init_command': 'SET SESSION group_concat_max_len = 1000000', - }, - }, -} -``` +MySQL was supported at some point, but not anymore I guess. diff --git a/binder/json.py b/binder/json.py index 00508124..99d3aad7 100644 --- a/binder/json.py +++ b/binder/json.py @@ -27,7 +27,7 @@ # dateutil.relativedelta serializer, if available try: from dateutil.relativedelta import relativedelta - from relativedeltafield import format_relativedelta + from relativedeltafield.utils import format_relativedelta SERIALIZERS[relativedelta] = format_relativedelta except ImportError: pass diff --git a/binder/models.py b/binder/models.py index c65e8f3b..a0d2640d 100644 --- a/binder/models.py +++ b/binder/models.py @@ -321,6 +321,25 @@ def clean_qualifier(self, qualifier, value): return qualifier, cleaned_value +class RelativeDeltaFieldFilter(FieldFilter): + name = 'RelativeDeltaFieldFilter' + fields = [] + allowed_qualifiers = [None, 'in', 'gt', 'gte', 'lt', 'lte', 'range', 'isnull'] + + def clean_value(self, qualifier, v): + from relativedeltafield.utils import parse_relativedelta + try: + return parse_relativedelta(v) + except ValueError: + raise ValidationError(v + ' is not a valid (extended) ISO8601 interval specification') + +try: + from relativedeltafield import RelativeDeltaField + RelativeDeltaFieldFilter.fields.append(RelativeDeltaField) +except ImportError: + pass + + class TimeFieldFilter(FieldFilter): fields = [models.TimeField] # Maybe allow __startswith? And __year etc? diff --git a/ci-requirements.txt b/ci-requirements.txt index 1a8d2496..3876259f 100644 --- a/ci-requirements.txt +++ b/ci-requirements.txt @@ -9,3 +9,5 @@ coverage django-hijack<3.0.0 openpyxl pika +python-dateutil >= 2.6.0 +django-relativedelta >= 2.0.0 diff --git a/docker-compose.yml b/docker-compose.yml index b0aeb0c7..efbdbbe6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,8 @@ services: db: - image: postgres:11.5 + image: postgres:13.21 + environment: + - POSTGRES_HOST_AUTH_METHOD=trust # Insecure, but fine for just for running tests. binder: build: . command: tail -f /dev/null diff --git a/setup.py b/setup.py index ac7e800a..79c8bc92 100755 --- a/setup.py +++ b/setup.py @@ -44,6 +44,7 @@ 'pika == 1.3.2', ], tests_require=[ + 'django-relativedelta >= 2.0.0', 'django-hijack >= 2.1.10, < 3.0.0', ( 'mysqlclient >= 1.3.12' diff --git a/tests/__init__.py b/tests/__init__.py index 0d8e9148..60aa4c0b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -31,105 +31,105 @@ 'USER': 'postgres', } -settings.configure(**{ - 'DEBUG': True, - 'SECRET_KEY': 'testy mctestface', - 'ALLOWED_HOSTS': ['*'], - 'DATABASES': { - 'default': db_settings, - }, - 'MIDDLEWARE': [ - # TODO: Try to reduce the set of absolutely required middlewares - 'request_id.middleware.RequestIdMiddleware', - 'django.contrib.sessions.middleware.SessionMiddleware', - 'django.middleware.csrf.CsrfViewMiddleware', - 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'binder.plugins.token_auth.middleware.TokenAuthMiddleware', - ], - 'INSTALLED_APPS': [ - # TODO: Try to reduce the set of absolutely required applications - 'django.contrib.auth', - 'django.contrib.contenttypes', - 'django.contrib.sessions', - 'binder', - 'binder.plugins.token_auth', - 'tests', - 'tests.testapp', - ], - 'MIGRATION_MODULES': { - 'testapp': None, - 'auth': None, - 'sessions': None, - 'contenttypes': None, - 'binder': None, - 'token_auth': None, - }, - 'USE_TZ': True, - 'TIME_ZONE': 'UTC', - 'ROOT_URLCONF': 'tests.testapp.urls', - 'LOGGING': { - 'version': 1, - 'handlers': { - 'console': { - 'level': 'DEBUG', - 'class': 'logging.StreamHandler', - }, +if not settings.configured: + settings.configure(**{ + 'DEBUG': True, + 'SECRET_KEY': 'testy mctestface', + 'ALLOWED_HOSTS': ['*'], + 'DATABASES': { + 'default': db_settings, }, - 'loggers': { - # We override only this one to avoid logspam - # while running tests. Django warnings are - # stil shown. - 'binder': { - 'handlers': ['console'], - 'level': 'ERROR', - }, - } - }, - 'BINDER_PERMISSION': { - 'default': [ - ('auth.reset_password_user', None), - ('auth.view_user', 'own'), - ('auth.activate_user', None), - ('auth.unmasquerade_user', None), # If you are masquarade, the user must be able to unmasquarade - ('auth.login_user', None), - ('auth.signup_user', None), - ('auth.logout_user', None), - ('auth.change_own_password_user', None), + 'MIDDLEWARE': [ + # TODO: Try to reduce the set of absolutely required middlewares + 'request_id.middleware.RequestIdMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'binder.plugins.token_auth.middleware.TokenAuthMiddleware', ], - # Basic permissions which can be used to override stuff - 'testapp.view_country': [ - - ] - }, - 'GROUP_PERMISSIONS': { - 'admin': [ - 'testapp.view_country' - ] - }, - 'GROUP_CONTAINS': { - 'admin': [] - }, - 'INTERNAL_MEDIA_HEADER': 'X-Accel-Redirect', - 'INTERNAL_MEDIA_LOCATION': '/internal/media/', -}) + 'INSTALLED_APPS': [ + # TODO: Try to reduce the set of absolutely required applications + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'binder', + 'binder.plugins.token_auth', + 'tests', + 'tests.testapp', + ], + 'MIGRATION_MODULES': { + 'testapp': None, + 'auth': None, + 'sessions': None, + 'contenttypes': None, + 'binder': None, + 'token_auth': None, + }, + 'USE_TZ': True, + 'TIME_ZONE': 'UTC', + 'ROOT_URLCONF': 'tests.testapp.urls', + 'LOGGING': { + 'version': 1, + 'handlers': { + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', + }, + }, + 'loggers': { + # We override only this one to avoid logspam + # while running tests. Django warnings are + # stil shown. + 'binder': { + 'handlers': ['console'], + 'level': 'ERROR', + }, + } + }, + 'BINDER_PERMISSION': { + 'default': [ + ('auth.reset_password_user', None), + ('auth.view_user', 'own'), + ('auth.activate_user', None), + ('auth.unmasquerade_user', None), # If you are masquarade, the user must be able to unmasquarade + ('auth.login_user', None), + ('auth.signup_user', None), + ('auth.logout_user', None), + ('auth.change_own_password_user', None), + ], + # Basic permissions which can be used to override stuff + 'testapp.view_country': [ -setup() + ] + }, + 'GROUP_PERMISSIONS': { + 'admin': [ + 'testapp.view_country' + ] + }, + 'GROUP_CONTAINS': { + 'admin': [] + }, + 'INTERNAL_MEDIA_HEADER': 'X-Accel-Redirect', + 'INTERNAL_MEDIA_LOCATION': '/internal/media/', + }) + setup() -# Do the dance to ensure the models are synched to the DB. -# This saves us from having to include migrations -from django.core.management.commands.migrate import Command as MigrationCommand # noqa -from django.db import connections # noqa -from django.db.migrations.executor import MigrationExecutor # noqa + # Do the dance to ensure the models are synched to the DB. + # This saves us from having to include migrations + from django.core.management.commands.migrate import Command as MigrationCommand # noqa + from django.db import connections # noqa + from django.db.migrations.executor import MigrationExecutor # noqa -# This is oh so hacky.... -cmd = MigrationCommand() -cmd.verbosity = 0 -connection = connections['default'] -executor = MigrationExecutor(connection) -cmd.sync_apps(connection, executor.loader.unmigrated_apps) + # This is oh so hacky.... + cmd = MigrationCommand() + cmd.verbosity = 0 + connection = connections['default'] + executor = MigrationExecutor(connection) + cmd.sync_apps(connection, executor.loader.unmigrated_apps) -# Hack to make the view_country permission, which doesn't work with the MigrationCommand somehow -from django.contrib.auth.models import Group, Permission, ContentType -content_type = ContentType.objects.get_or_create(app_label='testapp', model='country')[0] -Permission.objects.get_or_create(content_type=content_type, codename='view_country') -call_command('define_groups') + # Hack to make the view_country permission, which doesn't work with the MigrationCommand somehow + from django.contrib.auth.models import Group, Permission, ContentType + content_type = ContentType.objects.get_or_create(app_label='testapp', model='country')[0] + Permission.objects.get_or_create(content_type=content_type, codename='view_country') + call_command('define_groups') diff --git a/tests/plugins/test_loaded_values_mixin.py b/tests/plugins/test_loaded_values_mixin.py index d09a51af..bd579441 100644 --- a/tests/plugins/test_loaded_values_mixin.py +++ b/tests/plugins/test_loaded_values_mixin.py @@ -1,3 +1,4 @@ +from dateutil.relativedelta import relativedelta from django.test import TestCase from ..testapp.models import Animal, Zoo, Caretaker, ZooEmployee @@ -37,6 +38,7 @@ def test_old_values_after_initialization_are_identical_to_current_but_unchanged( 'zoo_of_birth': artis.id, 'caretaker': caretaker.id, 'deleted': False, + 'feeding_period': relativedelta(days=1), 'birth_date': None, }, scooby.get_old_values()) @@ -77,6 +79,7 @@ def test_old_values_after_change_are_marked_as_changed_and_old_values_returns_ol 'zoo_of_birth': artis.id, 'caretaker': None, 'deleted': False, + 'feeding_period': relativedelta(days=1), 'birth_date': None, }, scooby.get_old_values()) @@ -112,6 +115,7 @@ def test_old_values_return_current_value_after_fresh_fetch_from_db(self): 'zoo_of_birth': artis.id, 'caretaker': None, 'deleted': False, + 'feeding_period': relativedelta(days=1), 'birth_date': None, }, scooby.get_old_values()) @@ -150,6 +154,7 @@ def test_recursion_depth_issue_with_loaded_values_and_only(self): 'zoo_of_birth': artis.id, 'caretaker': None, 'deleted': False, + 'feeding_period': None, 'birth_date': None, }, scooby.get_old_values()) diff --git a/tests/test_filter_relative_delta.py b/tests/test_filter_relative_delta.py new file mode 100644 index 00000000..3b78017d --- /dev/null +++ b/tests/test_filter_relative_delta.py @@ -0,0 +1,55 @@ +from dateutil.relativedelta import relativedelta +from django.contrib.auth.models import User +from django.test import Client, TestCase +from json import loads +from .testapp.models import Animal + + +class FilterRelativeDeltaTest(TestCase): + def setUp(self): + super().setUp() + u = User(username='testuser', is_active=True, is_superuser=True) + u.set_password('test') + u.save() + self.client = Client() + r = self.client.login(username='testuser', password='test') + self.assertTrue(r) + + def test_filter(self): + bokito = Animal.objects.create(name='Bokito', feeding_period=relativedelta(hours=5)) + harambe = Animal.objects.create(name='Harambe', feeding_period=relativedelta(days=2)) + self.assertEqual(bokito.id, Animal.objects.filter(feeding_period__lt=relativedelta(days=1)).get().id) + response = self.client.get('/animal/?.feeding_period:gt=P1DT6H') + content = loads(response.content) + self.assertEqual(200, response.status_code) + self.assertEqual(1, len(content['data'])) + self.assertEqual(harambe.id, content['data'][0]['id']) + + def test_sort(self): + bokito = Animal.objects.create(name='Bokito', feeding_period=relativedelta(hours=5)) + harambe = Animal.objects.create(name='Harambe', feeding_period=relativedelta(days=2)) + otto = Animal.objects.create(name='Otto', feeding_period=relativedelta(days=1, hours=1)) + + sanity = list(Animal.objects.all().order_by('feeding_period')) + self.assertEqual(bokito.id, sanity[0].id) + self.assertEqual(otto.id, sanity[1].id) + self.assertEqual(harambe.id, sanity[2].id) + + response = self.client.get('/animal/?order_by=feeding_period') + content = loads(response.content) + self.assertEqual(200, response.status_code) + self.assertEqual(3, len(content['data'])) + self.assertEqual(bokito.id, content['data'][0]['id']) + self.assertEqual(otto.id, content['data'][1]['id']) + self.assertEqual(harambe.id, content['data'][2]['id']) + + response = self.client.get('/animal/?order_by=-feeding_period') + content = loads(response.content) + self.assertEqual(200, response.status_code) + self.assertEqual(3, len(content['data'])) + self.assertEqual(harambe.id, content['data'][0]['id']) + self.assertEqual(otto.id, content['data'][1]['id']) + self.assertEqual(bokito.id, content['data'][2]['id']) + + def test_default_value(self): + self.assertEqual(relativedelta(days=1), Animal.objects.create(name='Default').feeding_period) diff --git a/tests/testapp/models/animal.py b/tests/testapp/models/animal.py index fde79d7b..7f7ced4f 100644 --- a/tests/testapp/models/animal.py +++ b/tests/testapp/models/animal.py @@ -1,8 +1,10 @@ +from dateutil.relativedelta import relativedelta from django.db import models from django.db.models import Value, F, Func from binder.models import BinderModel, ContextAnnotation from binder.exceptions import BinderValidationError from binder.plugins.loaded_values import LoadedValuesMixin +from relativedeltafield import RelativeDeltaField class Concat(Func): @@ -15,6 +17,7 @@ class Animal(LoadedValuesMixin, BinderModel): zoo = models.ForeignKey('Zoo', on_delete=models.CASCADE, related_name='animals', blank=True, null=True) zoo_of_birth = models.ForeignKey('Zoo', on_delete=models.CASCADE, related_name='+', blank=True, null=True) # might've been born outside captivity caretaker = models.ForeignKey('Caretaker', on_delete=models.PROTECT, related_name='animals', blank=True, null=True) + feeding_period = RelativeDeltaField(default=relativedelta(days=1)) deleted = models.BooleanField(default=False) # Softdelete birth_date = models.DateField(blank=True, null=True)