diff --git a/CHANGELOG.md b/CHANGELOG.md index 9a1dd073b5..ee47434543 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to ### Added - ✨(backend) allow to create a new user in a marketing system +- ✨(backend) manage reconciliation requests for user accounts #1708 ### Changed diff --git a/src/backend/core/admin.py b/src/backend/core/admin.py index 8832903079..4f1d1dff88 100644 --- a/src/backend/core/admin.py +++ b/src/backend/core/admin.py @@ -1,12 +1,14 @@ """Admin classes and registrations for core app.""" -from django.contrib import admin +from django.contrib import admin, messages from django.contrib.auth import admin as auth_admin +from django.shortcuts import redirect from django.utils.translation import gettext_lazy as _ from treebeard.admin import TreeAdmin -from . import models +from core import models +from core.tasks.user_reconciliation import user_reconciliation_csv_import_job class TemplateAccessInline(admin.TabularInline): @@ -104,6 +106,71 @@ class UserAdmin(auth_admin.UserAdmin): search_fields = ("id", "sub", "admin_email", "email", "full_name") +@admin.register(models.UserReconciliationCsvImport) +class UserReconciliationCsvImportAdmin(admin.ModelAdmin): + """Admin class for UserReconciliationCsvImport model.""" + + list_display = ("id", "created_at", "status") + + def save_model(self, request, obj, form, change): + """Override save_model to trigger the import task on creation.""" + super().save_model(request, obj, form, change) + + if not change: + user_reconciliation_csv_import_job.delay(obj.pk) + messages.success(request, _("Import job created and queued.")) + return redirect("..") + + +@admin.action(description=_("Process selected user reconciliations")) +def process_reconciliation(_modeladmin, _request, queryset): + """ + Admin action to process selected user reconciliations. + The action will process only entries that are ready and have both emails checked. + + Its action is threefold: + - Transfer document accesses from inactive to active user, updating roles as needed. + - Activate the active user and deactivate the inactive user. + """ + processable_entries = queryset.filter( + status="ready", active_email_checked=True, inactive_email_checked=True + ) + + # Prepare the bulk operations + updated_documentaccess = [] + removed_documentaccess = [] + update_users_active_status = [] + + for entry in processable_entries: + new_updated_documentaccess, new_removed_documentaccess = ( + entry.process_documentaccess_reconciliation() + ) + updated_documentaccess += new_updated_documentaccess + removed_documentaccess += new_removed_documentaccess + + entry.active_user.is_active = True + entry.inactive_user.is_active = False + update_users_active_status.append(entry.active_user) + update_users_active_status.append(entry.inactive_user) + + # Actually perform the bulk operations + models.DocumentAccess.objects.bulk_update(updated_documentaccess, ["user", "role"]) + + if removed_documentaccess: + ids_to_delete = [rd.id for rd in removed_documentaccess] + models.DocumentAccess.objects.filter(id__in=ids_to_delete).delete() + + models.User.objects.bulk_update(update_users_active_status, ["is_active"]) + + +@admin.register(models.UserReconciliation) +class UserReconciliationAdmin(admin.ModelAdmin): + """Admin class for UserReconciliation model.""" + + list_display = ["id", "created_at", "status"] + actions = [process_reconciliation] + + @admin.register(models.Template) class TemplateAdmin(admin.ModelAdmin): """Template admin interface declaration.""" diff --git a/src/backend/core/migrations/0028_userreconciliationcsvimport_userreconciliation.py b/src/backend/core/migrations/0028_userreconciliationcsvimport_userreconciliation.py new file mode 100644 index 0000000000..069d12a539 --- /dev/null +++ b/src/backend/core/migrations/0028_userreconciliationcsvimport_userreconciliation.py @@ -0,0 +1,151 @@ +# Generated by Django 5.2.9 on 2025-12-15 19:09 + +import uuid + +import django.db.models.deletion +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("core", "0027_auto_20251120_0956"), + ] + + operations = [ + migrations.CreateModel( + name="UserReconciliationCsvImport", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + help_text="primary key for the record as UUID", + primary_key=True, + serialize=False, + verbose_name="id", + ), + ), + ( + "created_at", + models.DateTimeField( + auto_now_add=True, + help_text="date and time at which a record was created", + verbose_name="created on", + ), + ), + ( + "updated_at", + models.DateTimeField( + auto_now=True, + help_text="date and time at which a record was last updated", + verbose_name="updated on", + ), + ), + ("file", models.FileField(upload_to="imports/")), + ( + "status", + models.CharField( + choices=[ + ("pending", "Pending"), + ("running", "Running"), + ("done", "Done"), + ("error", "Error"), + ], + default="pending", + max_length=20, + ), + ), + ("logs", models.TextField(blank=True)), + ], + options={ + "verbose_name": "user reconciliation CSV import", + "verbose_name_plural": "user reconciliation CSV imports", + }, + ), + migrations.CreateModel( + name="UserReconciliation", + fields=[ + ( + "id", + models.UUIDField( + default=uuid.uuid4, + editable=False, + help_text="primary key for the record as UUID", + primary_key=True, + serialize=False, + verbose_name="id", + ), + ), + ( + "created_at", + models.DateTimeField( + auto_now_add=True, + help_text="date and time at which a record was created", + verbose_name="created on", + ), + ), + ( + "updated_at", + models.DateTimeField( + auto_now=True, + help_text="date and time at which a record was last updated", + verbose_name="updated on", + ), + ), + ( + "active_email", + models.EmailField( + max_length=254, verbose_name="Active email address" + ), + ), + ( + "inactive_email", + models.EmailField( + max_length=254, verbose_name="Email address to deactivate" + ), + ), + ("active_email_checked", models.BooleanField(default=False)), + ("inactive_email_checked", models.BooleanField(default=False)), + ( + "status", + models.CharField( + choices=[ + ("pending", "Pending"), + ("ready", "Ready"), + ("done", "Done"), + ("error", "Error"), + ], + default="pending", + max_length=20, + ), + ), + ("logs", models.TextField(blank=True)), + ( + "active_user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="active_user", + to=settings.AUTH_USER_MODEL, + ), + ), + ( + "inactive_user", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="inactive_user", + to=settings.AUTH_USER_MODEL, + ), + ), + ], + options={ + "verbose_name": "user reconciliation", + "verbose_name_plural": "user reconciliations", + }, + ), + ] diff --git a/src/backend/core/models.py b/src/backend/core/models.py index c17d3ec449..233a044637 100644 --- a/src/backend/core/models.py +++ b/src/backend/core/models.py @@ -1,6 +1,7 @@ """ Declare and configure the models for the impress core application """ + # pylint: disable=too-many-lines import hashlib @@ -32,14 +33,14 @@ from timezone_field import TimeZoneField from treebeard.mp_tree import MP_Node, MP_NodeManager, MP_NodeQuerySet -from .choices import ( +from core.choices import ( PRIVILEGED_ROLES, LinkReachChoices, LinkRoleChoices, RoleChoices, get_equivalent_link_definition, ) -from .validators import sub_validator +from core.validators import sub_validator logger = getLogger(__name__) @@ -265,6 +266,136 @@ def teams(self): return [] +class UserReconciliation(BaseModel): + """Model to run batch jobs to replace an active user by another one""" + + active_email = models.EmailField(_("Active email address")) + inactive_email = models.EmailField(_("Email address to deactivate")) + active_email_checked = models.BooleanField(default=False) + inactive_email_checked = models.BooleanField(default=False) + active_user = models.ForeignKey( + User, + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="active_user", + ) + inactive_user = models.ForeignKey( + User, + on_delete=models.CASCADE, + null=True, + blank=True, + related_name="inactive_user", + ) + + status = models.CharField( + max_length=20, + choices=[ + ("pending", _("Pending")), + ("ready", _("Ready")), + ("done", _("Done")), + ("error", _("Error")), + ], + default="pending", + ) + logs = models.TextField(blank=True) + + class Meta: + verbose_name = _("user reconciliation") + verbose_name_plural = _("user reconciliations") + + def __str__(self): + return f"Reconciliation from {self.inactive_email} to {self.active_email}" + + def save(self, *args, **kwargs): + """ + For pending queries, identify the actual users and send validation emails + """ + if self.status == "pending": + self.active_user = User.objects.filter(email=self.active_email).first() + self.inactive_user = User.objects.filter(email=self.inactive_email).first() + + if self.active_user and self.inactive_user: + email_subject = _("Account reconciliation request") + email_content = _( + """ + Please click here. + """ + ) + if not self.active_email_checked: + self.active_user.email_user(email_subject, email_content) + if not self.inactive_email_checked: + self.inactive_user.email_user(email_subject, email_content) + self.status = "ready" + else: + self.status = "error" + self.logs = "Error: Both active and inactive users need to exist." + + super().save(*args, **kwargs) + + def process_documentaccess_reconciliation(self): + """ + Process the reconciliation by transferring document accesses from the inactive user + to the active user. + """ + updated_accesses = [] + removed_accesses = [] + inactive_accesses = DocumentAccess.objects.filter(user=self.inactive_user) + + # Check documents where the active user already has access + documents_with_both_users = inactive_accesses.values_list("document", flat=True) + existing_accesses = DocumentAccess.objects.filter(user=self.active_user).filter( + document__in=documents_with_both_users + ) + existing_roles_per_doc = dict(existing_accesses.values_list("document", "role")) + + for entry in inactive_accesses: + if entry.document_id in existing_roles_per_doc: + # Update role if needed + existing_role = existing_roles_per_doc[entry.document_id] + max_role = RoleChoices.max(entry.role, existing_role) + if existing_role != max_role: + existing_access = existing_accesses.get(document=entry.document) + existing_access.role = max_role + updated_accesses.append(existing_access) + removed_accesses.append(entry) + else: + entry.user = self.active_user + updated_accesses.append(entry) + + self.logs += f"""Requested update for {len(updated_accesses)} DocumentAccess items + and deletion for {len(removed_accesses)} DocumentAccess items.\n""" + self.status = "done" + self.save() + + return updated_accesses, removed_accesses + + +class UserReconciliationCsvImport(BaseModel): + """Model to import reconciliations requests from an external source + (eg, )""" + + file = models.FileField(upload_to="imports/") + status = models.CharField( + max_length=20, + choices=[ + ("pending", _("Pending")), + ("running", _("Running")), + ("done", _("Done")), + ("error", _("Error")), + ], + default="pending", + ) + logs = models.TextField(blank=True) + + class Meta: + verbose_name = _("user reconciliation CSV import") + verbose_name_plural = _("user reconciliation CSV imports") + + def __str__(self): + return f"User reconciliation CSV import {self.id}" + + class BaseAccess(BaseModel): """Base model for accesses to handle resources.""" diff --git a/src/backend/core/tasks/user_reconciliation.py b/src/backend/core/tasks/user_reconciliation.py new file mode 100644 index 0000000000..6e50fcbd0a --- /dev/null +++ b/src/backend/core/tasks/user_reconciliation.py @@ -0,0 +1,57 @@ +"""Processing tasks for user reconciliation CSV imports.""" + +import csv +import traceback + +from django.core.exceptions import ValidationError +from django.db import IntegrityError + +from botocore.exceptions import ClientError + +from core.models import UserReconciliation, UserReconciliationCsvImport + +from impress.celery_app import app + + +@app.task +def user_reconciliation_csv_import_job(job_id): + """Process a UserReconciliationCsvImport job. + Creates UserReconciliation entries from the CSV file. + """ + # Imports the CSV file, breaks it into UserReconciliation items + job = UserReconciliationCsvImport.objects.get(id=job_id) + job.status = "running" + job.save() + + try: + with job.file.open(mode="r") as f: + reader = csv.DictReader(f) + for row in reader: + active_email_checked = row["active_email_checked"] == "1" + inactive_email_checked = row["inactive_email_checked"] == "1" + + rec_entry = UserReconciliation.objects.create( + active_email=row["active_email"], + inactive_email=row["inactive_email"], + active_email_checked=active_email_checked, + inactive_email_checked=inactive_email_checked, + status="pending", + ) + rec_entry.save() + + job.status = "done" + job.logs = f"Import completed successfully. {reader.line_num} rows processed." + except ( + csv.Error, + KeyError, + ValueError, + ValidationError, + IntegrityError, + OSError, + ClientError, + ) as e: + # Catch expected I/O/CSV/model errors and record traceback in logs for debugging + job.status = "error" + job.logs = f"{e!s}\n{traceback.format_exc()}" + finally: + job.save() diff --git a/src/backend/core/tests/data/example_reconciliation.csv b/src/backend/core/tests/data/example_reconciliation.csv new file mode 100644 index 0000000000..4ed1239bb2 --- /dev/null +++ b/src/backend/core/tests/data/example_reconciliation.csv @@ -0,0 +1,6 @@ +active_email,inactive_email,active_email_checked,inactive_email_checked, +"user.test40@example.com","user.test41@example.com",0,0 +"user.test42@example.com","user.test43@example.com",0,1 +"user.test44@example.com","user.test45@example.com",1,0 +"user.test46@example.com","user.test47@example.com",1,1 +"user.test48@example.com","user.test49@example.com",1,1 \ No newline at end of file diff --git a/src/backend/core/tests/data/example_reconciliation_error.csv b/src/backend/core/tests/data/example_reconciliation_error.csv new file mode 100644 index 0000000000..9348b7798d --- /dev/null +++ b/src/backend/core/tests/data/example_reconciliation_error.csv @@ -0,0 +1,2 @@ +active_email,inactive_email,active_email_checked,inactive_email_checked, +"user.test40@example.com",,0,0 \ No newline at end of file diff --git a/src/backend/core/tests/test_models_user_reconciliation.py b/src/backend/core/tests/test_models_user_reconciliation.py new file mode 100644 index 0000000000..1f065a06cd --- /dev/null +++ b/src/backend/core/tests/test_models_user_reconciliation.py @@ -0,0 +1,250 @@ +""" +Unit tests for the UserReconciliationCsvImport model +""" + +from os import name +from pathlib import Path + +from django.core.files.base import ContentFile + +import pytest + +from core import factories, models +from core.admin import process_reconciliation +from core.tasks.user_reconciliation import user_reconciliation_csv_import_job + +pytestmark = pytest.mark.django_db + + +@pytest.fixture(name="import_example_csv") +def fixture_import_example_csv(): + """ + Import an example CSV file for user reconciliation + and return the created import object. + """ + # Create users referenced in the CSV + for i in range(40, 50): + factories.UserFactory(email=f"user.test{i}@example.com") + + example_csv_path = Path(__file__).parent / "data/example_reconciliation.csv" + with open(example_csv_path, "rb") as f: + csv_file = ContentFile(f.read(), name="example_reconciliation.csv") + csv_import = models.UserReconciliationCsvImport(file=csv_file) + csv_import.save() + + return csv_import + + +def test_user_reconciliation_csv_import_entry_is_created(import_example_csv): + """Test that a UserReconciliationCsvImport entry is created correctly.""" + assert import_example_csv.status == "pending" + assert import_example_csv.file.name.endswith("example_reconciliation.csv") + + +def test_incorrect_csv_format_handling(): + """Test that an incorrectly formatted CSV file is handled gracefully.""" + example_csv_path = Path(__file__).parent / "data/example_reconciliation_error.csv" + with open(example_csv_path, "rb") as f: + csv_file = ContentFile(f.read(), name="example_reconciliation_error.csv") + csv_import = models.UserReconciliationCsvImport(file=csv_file) + csv_import.save() + + assert csv_import.status == "pending" + + user_reconciliation_csv_import_job(csv_import.id) + csv_import.refresh_from_db() + + assert "This field cannot be blank." in csv_import.logs + assert csv_import.status == "error" + + +def test_job_creates_reconciliation_entries(import_example_csv): + """Test that the CSV import job creates UserReconciliation entries.""" + assert import_example_csv.status == "pending" + user_reconciliation_csv_import_job(import_example_csv.id) + + # Verify the job status changed + import_example_csv.refresh_from_db() + assert import_example_csv.status == "done" + assert "Import completed successfully" in import_example_csv.logs + + # Verify reconciliation entries were created + reconciliations = models.UserReconciliation.objects.all() + assert reconciliations.count() == 5 + + +def test_csv_import_reconciliation_data_is_correct(import_example_csv): + """Test that the data in created UserReconciliation entries matches the CSV.""" + user_reconciliation_csv_import_job(import_example_csv.id) + + reconciliations = models.UserReconciliation.objects.order_by("created_at") + first_entry = reconciliations.first() + + assert first_entry.active_email == "user.test40@example.com" + assert first_entry.inactive_email == "user.test41@example.com" + assert first_entry.active_email_checked is False + assert first_entry.inactive_email_checked is False + + for rec in reconciliations: + assert rec.status == "ready" + + +@pytest.fixture(name="user_reconciliation_users_and_docs") +def fixture_user_reconciliation_users_and_docs(): + """Fixture to create two users with overlapping document accesses + for reconciliation tests.""" + user_1 = factories.UserFactory(email="user.test1@example.com") + user_2 = factories.UserFactory(email="user.test2@example.com") + + # Create 10 distinct document accesses for each user + userdocs_u1 = [ + factories.UserDocumentAccessFactory(user=user_1, role="editor") + for _ in range(10) + ] + userdocs_u2 = [ + factories.UserDocumentAccessFactory(user=user_2, role="editor") + for _ in range(10) + ] + + # Make the first 3 documents of each list shared with the other user + # with a lower role + for ud in userdocs_u1[0:3]: + factories.UserDocumentAccessFactory( + user=user_2, document=ud.document, role="reader" + ) + + for ud in userdocs_u2[0:3]: + factories.UserDocumentAccessFactory( + user=user_1, document=ud.document, role="reader" + ) + + # Make the next 3 documents of each list shared with the other user + # with a higher role + for ud in userdocs_u1[3:6]: + factories.UserDocumentAccessFactory( + user=user_2, document=ud.document, role="owner" + ) + + for ud in userdocs_u2[3:6]: + factories.UserDocumentAccessFactory( + user=user_1, document=ud.document, role="owner" + ) + + return (user_1, user_2, userdocs_u1, userdocs_u2) + + +def test_user_reconciliation_is_created(user_reconciliation_users_and_docs): + """Test that a UserReconciliation entry can be created and saved.""" + user_1, user_2, _userdocs_u1, _userdocs_u2 = user_reconciliation_users_and_docs + rec = models.UserReconciliation.objects.create( + active_email=user_1.email, + inactive_email=user_2.email, + active_email_checked=False, + inactive_email_checked=True, + status="pending", + ) + + rec.save() + assert rec.status == "ready" + + +def test_user_reconciliation_only_starts_if_checks_are_made( + user_reconciliation_users_and_docs, +): + """Test that the admin action does not process entries + unless both email checks are confirmed. + """ + user_1, user_2, _userdocs_u1, _userdocs_u2 = user_reconciliation_users_and_docs + + # Create a reconciliation entry where only one email has been checked + rec = models.UserReconciliation.objects.create( + active_email=user_1.email, + inactive_email=user_2.email, + active_email_checked=True, + inactive_email_checked=False, + status="pending", + ) + rec.save() + + # Capture counts before running admin action + accesses_before_active = models.DocumentAccess.objects.filter(user=user_1).count() + accesses_before_inactive = models.DocumentAccess.objects.filter(user=user_2).count() + users_active_before = (user_1.is_active, user_2.is_active) + + # Call the admin action with the queryset containing our single rec + qs = models.UserReconciliation.objects.filter(id=rec.id) + process_reconciliation(None, None, qs) + + # Reload from DB and assert nothing was processed (checks prevent processing) + rec.refresh_from_db() + user_1.refresh_from_db() + user_2.refresh_from_db() + + assert rec.status == "ready" + assert ( + models.DocumentAccess.objects.filter(user=user_1).count() + == accesses_before_active + ) + assert ( + models.DocumentAccess.objects.filter(user=user_2).count() + == accesses_before_inactive + ) + assert (user_1.is_active, user_2.is_active) == users_active_before + + +def test_process_documentaccess_reconciliation( + user_reconciliation_users_and_docs, +): + """Use the fixture to verify accesses are consolidated on the active user.""" + user_1, user_2, userdocs_u1, userdocs_u2 = user_reconciliation_users_and_docs + + u1_2 = userdocs_u1[2] + u1_5 = userdocs_u1[5] + u2doc1 = userdocs_u2[1].document + u2doc5 = userdocs_u2[5].document + + rec = models.UserReconciliation.objects.create( + active_email=user_1.email, + inactive_email=user_2.email, + active_user=user_1, + inactive_user=user_2, + active_email_checked=True, + inactive_email_checked=True, + status="ready", + ) + + qs = models.UserReconciliation.objects.filter(id=rec.id) + process_reconciliation(None, None, qs) + + rec.refresh_from_db() + user_1.refresh_from_db() + user_2.refresh_from_db() + u1_2.refresh_from_db( + from_queryset=models.DocumentAccess.objects.select_for_update() + ) + u1_5.refresh_from_db( + from_queryset=models.DocumentAccess.objects.select_for_update() + ) + + # After processing, inactive user should have no accesses + # and active user should have one access per union document + # with the highest role + assert rec.status == "done" + assert "Requested update for 10 DocumentAccess items" in rec.logs + assert "and deletion for 12 DocumentAccess items" in rec.logs + assert models.DocumentAccess.objects.filter(user=user_2).count() == 0 + assert models.DocumentAccess.objects.filter(user=user_1).count() == 20 + assert u1_2.role == "editor" + assert u1_5.role == "owner" + + assert ( + models.DocumentAccess.objects.filter(user=user_1, document=u2doc1).first().role + == "editor" + ) + assert ( + models.DocumentAccess.objects.filter(user=user_1, document=u2doc5).first().role + == "owner" + ) + + assert user_1.is_active is True + assert user_2.is_active is False