diff --git a/docker-compose.yml b/docker-compose.yml index 42a8dd7c..fa725451 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -168,6 +168,46 @@ services: networks: - openspp + # Job Worker - background service for processing async queue jobs + # Runs alongside openspp/openspp-dev to execute delayed imports, etc. + # The runner auto-discovers databases every 5 minutes, so after installing + # new modules, jobs will be picked up within 5 minutes (or restart to force). + jobworker: + image: openspp-dev + profiles: + - dev + - ui + depends_on: + db: + condition: service_healthy + environment: + DB_HOST: db + DB_PORT: "5432" + DB_USER: odoo + DB_PASSWORD: odoo + DB_NAME: ${ODOO_DB_NAME:-openspp} + DB_FILTER: "^${ODOO_DB_NAME:-openspp}$$" + LIST_DB: "False" + ODOO_ADMIN_PASSWD: admin + ODOO_WORKERS: "0" + ODOO_CRON_THREADS: "0" + LOG_LEVEL: info + PROXY_MODE: "False" + QUEUE_JOB_CONCURRENCY: "2" + volumes: + - .:/mnt/extra-addons/openspp:ro,z + - odoo_data:/var/lib/odoo + entrypoint: ["/entrypoint.sh"] + command: + [ + "python", + "/mnt/extra-addons/odoo-job-worker/job_worker_runner.py", + "-c", + "/etc/odoo/odoo.conf", + ] + networks: + - openspp + # Unit Test Runner - one-off container for running module tests # Usage: docker compose run --rm test [options] # Note: Uses same image as odoo/odoo-dev - build any of them to update diff --git a/spp b/spp index 8570e39f..f65088bd 100755 --- a/spp +++ b/spp @@ -726,8 +726,8 @@ def cmd_resetdb(args): # Ensure db is running run(docker_compose("up", "-d", "db")) - # Stop Odoo but keep db - run(docker_compose("stop", "openspp", "openspp-dev"), check=False) + # Stop Odoo and job worker but keep db + run(docker_compose("stop", "openspp", "openspp-dev", "jobworker"), check=False) # Wait for db to be ready print("Waiting for database...") diff --git a/spp_import_match/README.rst b/spp_import_match/README.rst index 13787776..d47fdd5c 100644 --- a/spp_import_match/README.rst +++ b/spp_import_match/README.rst @@ -33,16 +33,15 @@ Key Capabilities - Define matching rules per model using field combinations to identify existing records -- Match on sub-fields within related records (e.g., household ID within - individual) -- Apply conditional matching rules only when specific imported values - are present +- Match on sub-fields within related records (e.g., ``parent_id/name``) +- Apply conditional matching rules only when a specific imported value + is present - Skip duplicate creation or update existing records when matches are found -- Process imports with more than 100 records asynchronously using - ``job_worker`` -- Clear one2many/many2many associations before update to prevent - duplicate entries +- Split imports exceeding 100 rows into chunks and process + asynchronously via ``job_worker`` +- Strip falsy one2many/many2many values on write to prevent duplicate + relational entries Key Models ~~~~~~~~~~ @@ -53,8 +52,8 @@ Key Models | ``spp.import.match`` | Matching rule configuration for a | | | specific model | +-----------------------------+----------------------------------------+ -| ``spp.import.match.fields`` | Individual fields used in a rule, | -| | supports sub-fields | +| ``spp.import.match.fields`` | Individual field in a rule, with | +| | optional sub-field | +-----------------------------+----------------------------------------+ Configuration @@ -63,23 +62,21 @@ Configuration After installing: 1. Navigate to **Registry > Configuration > Import Match** -2. Create a new matching rule and select the target model (e.g., +2. Create a matching rule and select the target model (e.g., ``res.partner``) 3. Add one or more fields to match on (e.g., national ID, or first name + date of birth) 4. Enable **Overwrite Match** to update existing records when matches are found 5. For conditional matching, enable **Is Conditional** on a field and - specify the expected imported value + set the expected imported value UI Location ~~~~~~~~~~~ - **Menu**: Registry > Configuration > Import Match -- **Import Dialog**: Matching applies automatically during CSV import - via the standard Odoo import interface -- **Queue Jobs**: Registry > Queue Jobs > Jobs (to monitor asynchronous - imports) +- **Import Dialog**: Select a matching rule and overwrite option from + the import sidebar Security ~~~~~~~~ @@ -94,11 +91,11 @@ Extension Points ~~~~~~~~~~~~~~~~ - Override ``spp.import.match._match_find()`` to customize matching - logic for specific use cases + logic - Override ``spp.import.match._usable_rules()`` to filter which rules apply based on context -- Inherits ``base.load()`` to inject matching logic into all model - imports +- Overrides ``base.load()`` to inject matching into all model imports +- Overrides ``base.write()`` to strip falsy one2many/many2many values Dependencies ~~~~~~~~~~~~ @@ -111,6 +108,288 @@ Dependencies .. contents:: :local: +Usage +===== + +Prerequisites +------------- + +- The module **OpenSPP Import Match** is installed +- You are logged in as a user with the **SPP Admin** + (``spp_security.group_spp_admin``) role + +Test 1: Create a Matching Rule +------------------------------ + +**Steps:** + +1. Navigate to **Registry > Configuration > Import Match** +2. Click **New** +3. Enter a name (e.g., "Match Partner by Name") +4. In the **Match Details** tab, set **Model** to ``Contact`` + (res.partner) +5. Verify that **Model Name** and **Model Description** auto-populate + as ``res.partner`` and ``Contact`` +6. In the **Fields** list, click **Add a line** +7. Select a non-relational field (e.g., ``Name``) +8. Verify that the **Sub-Field** column is read-only (greyed out) for + non-relational fields +9. Leave **Is Conditional** unchecked and **Imported Value** empty +10. Click **Save** + +**Expected:** + +- The rule appears in the list view with the name you entered +- The list view shows a drag handle for reordering by sequence + +Test 2: Verify Duplicate Field Validation +----------------------------------------- + +**Steps:** + +1. Open the matching rule created in Test 1 +2. In the **Fields** list, click **Add a line** +3. Select the same non-relational field (e.g., ``Name``) that already + exists in the list + +**Expected:** + +- A validation error appears: "Field 'Name', already exists!" +- The duplicate field is not added + +Test 3: Sub-Field on Relational Fields +-------------------------------------- + +**Steps:** + +1. Create a new matching rule for model ``Contact`` (res.partner) +2. In the **Fields** list, add a relational field (e.g., ``Parent`` + which is a Many2one) +3. Verify that the **Sub-Field** column becomes editable and required +4. Select a sub-field (e.g., ``Name``) +5. Save the rule + +**Expected:** + +- The field's computed name displays as ``parent_id/name`` + (field/sub-field format) +- The rule saves without error + +Test 4: Model Change Clears Fields +---------------------------------- + +**Steps:** + +1. Open the matching rule created in Test 3 +2. Change the **Model** to a different model (e.g., ``Country``) +3. Observe the **Fields** list + +**Expected:** + +- All previously configured fields are cleared from the list +- The field domain updates to show only fields from the newly selected + model + +Test 5: Import with Matching — Skip Duplicates +---------------------------------------------- + +**Steps:** + +1. Create a matching rule for ``Contact`` with the ``Name`` field and + **Overwrite Match** unchecked +2. Create a contact manually with Name = "Test Import Match" +3. Navigate to **Contacts** and click **Import records** (or use the + gear/action menu) +4. Upload a CSV file containing: + :: + + name,email + Test Import Match,updated@example.com + Brand New Contact,new@example.com + +5. In the import sidebar, locate the **Import Matching** section +6. Select the matching rule you created from the dropdown +7. Verify the **Overwrite Match** checkbox is unchecked +8. Verify the helper text reads: "Matched records will be skipped." +9. Click **Test** (dry run) + +**Expected:** + +- An info message appears: "1 to skip, 1 to create" +- No records are modified yet + +10. Click **Import** + +**Expected:** + +- A success notification appears: "1 skipped, 1 created" +- The existing "Test Import Match" contact retains its original email + (not updated) +- A new "Brand New Contact" record is created with email + ``new@example.com`` + +Test 6: Import with Matching — Overwrite +---------------------------------------- + +**Steps:** + +1. Using the same matching rule from Test 5, re-import the same CSV +2. In the import sidebar, select the matching rule +3. Check the **Overwrite Match** checkbox +4. Verify the helper text changes to: "Matched records will be + overwritten with imported data." +5. Click **Test** (dry run) + +**Expected:** + +- An info message appears showing counts for records to overwrite and to + create + +6. Click **Import** + +**Expected:** + +- A success notification with overwrite/create counts +- The "Test Import Match" contact now has email ``updated@example.com`` +- The "Brand New Contact" record either has a second copy created or is + matched (depending on whether a Name rule matched it from the previous + import) + +Test 7: Conditional Matching +---------------------------- + +**Steps:** + +1. Create a new matching rule for ``Contact`` with **Overwrite Match** + checked +2. Add the ``Name`` field to the match fields +3. Check **Is Conditional** on the Name field +4. Set **Imported Value** to "Test Import Match" +5. Save the rule +6. Import a CSV: + :: + + name,email + Test Import Match,conditional@example.com + Some Other Name,other@example.com + +7. Select this matching rule in the import sidebar + +**Expected:** + +- The row with name "Test Import Match" matches the condition and the + existing record is overwritten +- The row with name "Some Other Name" does not match the condition + (imported value differs from "Test Import Match"), so the entire rule + is skipped for that row and a new record is created + +Test 8: Import Matching Dropdown Visibility +------------------------------------------- + +**Steps:** + +1. Navigate to a model that has NO matching rules configured (e.g., + ``Countries``) +2. Open the import dialog + +**Expected:** + +- The **Import Matching** section does not appear in the sidebar (the + dropdown is hidden when no rules exist for the model) + +3. Navigate to a model that HAS matching rules (e.g., ``Contacts``) +4. Open the import dialog + +**Expected:** + +- The **Import Matching** section appears with a dropdown listing all + rules for that model +- The default selection is "-- No Matching --" +- The **Overwrite Match** checkbox is hidden until a rule is selected + +Test 9: Overwrite Match Default from Rule +----------------------------------------- + +**Steps:** + +1. Create a matching rule with **Overwrite Match** checked on the rule + form +2. Open the import dialog for the rule's model +3. Select this rule from the **Import Matching** dropdown + +**Expected:** + +- The **Overwrite Match** checkbox is automatically checked (inherits + the rule's default) +- The user can uncheck it to override the default for this import + +4. Select "-- No Matching --" from the dropdown + +**Expected:** + +- The **Overwrite Match** checkbox disappears + +Test 10: Asynchronous Import (Large File) +----------------------------------------- + +**Steps:** + +1. Create a matching rule for ``Contact`` with the ``Name`` field +2. Prepare a CSV with more than 100 rows of contact data (101+ rows of + data, not counting the header) +3. Open the import dialog and upload the CSV +4. Select the matching rule +5. Click **Import** (not Test) + +**Expected:** + +- A notification appears: "Successfully added on Queue" +- The browser navigates back to the previous page +- The import is processed in the background via job worker +- Check job status by navigating to **Settings > Technical > Queue Jobs + > Jobs** (requires technical user access) + +Test 11: Multiple Matches Error +------------------------------- + +**Steps:** + +1. Create two contacts with the same email: ``duplicate@example.com`` +2. Create a matching rule for ``Contact`` using the ``Email`` field +3. Import a CSV: + :: + + email,name + duplicate@example.com,Updated Name + +4. Select the matching rule and click **Import** + +**Expected:** + +- An error is raised: "Multiple matches found for '...'" (where '...' is + the name of the first matched record) +- No records are modified + +Test 12: Security — Non-Admin Access +------------------------------------ + +**Steps:** + +1. Log in as a user without the **SPP Admin** role +2. Try to navigate to **Registry > Configuration > Import Match** + +**Expected:** + +- The **Import Match** menu item is not visible +- The user cannot access the matching rule configuration + +3. Open the import dialog for any model + +**Expected:** + +- The **Import Matching** section does not appear (the ``searchRead`` on + ``spp.import.match`` returns no results due to access rules) + Bug Tracker =========== diff --git a/spp_import_match/models/base.py b/spp_import_match/models/base.py index bf120686..58c9bdbf 100644 --- a/spp_import_match/models/base.py +++ b/spp_import_match/models/base.py @@ -2,7 +2,8 @@ import logging import threading -from odoo import api, models +from odoo import _, api, models +from odoo.exceptions import UserError _logger = logging.getLogger(__name__) @@ -15,11 +16,34 @@ class Base(models.AbstractModel): @api.model def load(self, fields, data): + # Only apply import matching when explicitly selected via UI context + if "import_match_ids" not in self.env.context: + return super().load(fields, data) + + import_match_ids = self.env.context.get("import_match_ids", []) usable, field_to_match = self.env["spp.import.match"]._usable_rules( self._name, fields, - option_config_ids=self.env.context.get("import_match_ids", []), + option_config_ids=import_match_ids, ) + + # Error if user selected match configs but their fields aren't in the file + if import_match_ids and not usable: + selected_configs = self.env["spp.import.match"].browse(import_match_ids) + config_names = selected_configs.mapped("name") + required_fields = set() + for config in selected_configs: + for field_line in config.field_ids: + required_fields.add(field_line.field_id.field_description) + raise UserError( + _( + "The selected import match configuration(s) %(configs)s require field(s) " + "%(fields)s, but none of these fields are present in the imported file.", + configs=", ".join(filter(None, config_names)), + fields=", ".join(sorted(required_fields)), + ) + ) + # If overwrite_match is explicitly set via UI (context), use that. # Otherwise fall back to config records' overwrite_match setting. if "overwrite_match" in self.env.context: diff --git a/spp_import_match/models/base_import.py b/spp_import_match/models/base_import.py index a745c7d2..74a38763 100644 --- a/spp_import_match/models/base_import.py +++ b/spp_import_match/models/base_import.py @@ -66,10 +66,12 @@ def execute_import(self, fields, columns, options, dryrun=False): overwrite_match = options.get("overwrite_match", False) _import_match_local.counts = None + # Set import match context early so both sync and async paths have it + if import_match_ids: + self = self.with_context(import_match_ids=import_match_ids, overwrite_match=overwrite_match) + if dryrun or len(input_file_data) <= 100: _logger.info("Doing %s import", "dry-run" if dryrun else "normal") - if import_match_ids: - self = self.with_context(import_match_ids=import_match_ids, overwrite_match=overwrite_match) result = super().execute_import(fields, columns, options, dryrun=dryrun) counts = getattr(_import_match_local, "counts", None) if counts: diff --git a/spp_import_match/models/import_match.py b/spp_import_match/models/import_match.py index fc78a91d..5ee26de6 100644 --- a/spp_import_match/models/import_match.py +++ b/spp_import_match/models/import_match.py @@ -39,6 +39,26 @@ def _onchange_model_id(self): for rec in self: rec.field_ids = None + @api.constrains("name") + def _check_duplicate_name(self): + """Prevent duplicate match rule names.""" + for rec in self: + if rec.name and self.search_count([("name", "=", rec.name), ("id", "!=", rec.id)]): + raise ValidationError(_("A match rule with the name '%s' already exists!") % rec.name) + + @api.constrains("field_ids") + def _check_duplicate_fields(self): + """Prevent duplicate non-relational fields in the same match rule.""" + for rec in self: + seen = [] + for field_line in rec.field_ids: + if field_line.field_id.ttype in ("many2many", "one2many", "many2one"): + continue + key = field_line.field_id.id + if key in seen: + raise ValidationError(_("Field '%s', already exists!") % field_line.field_id.field_description) + seen.append(key) + @api.model def _match_find(self, model, converted_row, imported_row): usable, field_to_match = self._usable_rules(model._name, converted_row) @@ -53,6 +73,10 @@ def _match_find(self, model, converted_row, imported_row): break if field.field_id.name in converted_row: row_value = converted_row[field.field_id.name] + # Skip matching on empty values to avoid false matches + if not row_value: + combination_valid = False + break field_value = field.field_id.name add_to_domain = True if field.sub_field_id: @@ -66,11 +90,13 @@ def _match_find(self, model, converted_row, imported_row): domain.append((field_value, "=", row_value)) if not combination_valid: continue + if not domain: + continue match = model.search(domain) + if len(match) > 1: + raise ValidationError(_("Multiple records found for matching criteria: %s") % domain) if len(match) == 1: - return match - elif len(match) > 1: - raise ValidationError(_("Multiple matches found for '%s'!") % match[0].name) + return match[0] return model diff --git a/spp_import_match/readme/DESCRIPTION.md b/spp_import_match/readme/DESCRIPTION.md index 52f80a17..bbf8bcc2 100644 --- a/spp_import_match/readme/DESCRIPTION.md +++ b/spp_import_match/readme/DESCRIPTION.md @@ -3,34 +3,33 @@ Extends Odoo's base import functionality to match incoming records against exist ### Key Capabilities - Define matching rules per model using field combinations to identify existing records -- Match on sub-fields within related records (e.g., household ID within individual) -- Apply conditional matching rules only when specific imported values are present +- Match on sub-fields within related records (e.g., `parent_id/name`) +- Apply conditional matching rules only when a specific imported value is present - Skip duplicate creation or update existing records when matches are found -- Process imports with more than 100 records asynchronously using `job_worker` -- Clear one2many/many2many associations before update to prevent duplicate entries +- Split imports exceeding 100 rows into chunks and process asynchronously via `job_worker` +- Strip falsy one2many/many2many values on write to prevent duplicate relational entries ### Key Models -| Model | Description | -| ------------------------- | -------------------------------------------------------- | -| `spp.import.match` | Matching rule configuration for a specific model | -| `spp.import.match.fields` | Individual fields used in a rule, supports sub-fields | +| Model | Description | +| ------------------------- | ----------------------------------------------------- | +| `spp.import.match` | Matching rule configuration for a specific model | +| `spp.import.match.fields` | Individual field in a rule, with optional sub-field | ### Configuration After installing: 1. Navigate to **Registry > Configuration > Import Match** -2. Create a new matching rule and select the target model (e.g., `res.partner`) +2. Create a matching rule and select the target model (e.g., `res.partner`) 3. Add one or more fields to match on (e.g., national ID, or first name + date of birth) 4. Enable **Overwrite Match** to update existing records when matches are found -5. For conditional matching, enable **Is Conditional** on a field and specify the expected imported value +5. For conditional matching, enable **Is Conditional** on a field and set the expected imported value ### UI Location - **Menu**: Registry > Configuration > Import Match -- **Import Dialog**: Matching applies automatically during CSV import via the standard Odoo import interface -- **Queue Jobs**: Registry > Queue Jobs > Jobs (to monitor asynchronous imports) +- **Import Dialog**: Select a matching rule and overwrite option from the import sidebar ### Security @@ -40,9 +39,10 @@ After installing: ### Extension Points -- Override `spp.import.match._match_find()` to customize matching logic for specific use cases +- Override `spp.import.match._match_find()` to customize matching logic - Override `spp.import.match._usable_rules()` to filter which rules apply based on context -- Inherits `base.load()` to inject matching logic into all model imports +- Overrides `base.load()` to inject matching into all model imports +- Overrides `base.write()` to strip falsy one2many/many2many values ### Dependencies diff --git a/spp_import_match/readme/USAGE.md b/spp_import_match/readme/USAGE.md new file mode 100644 index 00000000..50f6acd2 --- /dev/null +++ b/spp_import_match/readme/USAGE.md @@ -0,0 +1,233 @@ +## Prerequisites + +- The module **OpenSPP Import Match** is installed +- You are logged in as a user with the **SPP Admin** (`spp_security.group_spp_admin`) role + +## Test 1: Create a Matching Rule + +**Steps:** + +1. Navigate to **Registry > Configuration > Import Match** +2. Click **New** +3. Enter a name (e.g., "Match Partner by Name") +4. In the **Match Details** tab, set **Model** to `Contact` (res.partner) +5. Verify that **Model Name** and **Model Description** auto-populate as `res.partner` and `Contact` +6. In the **Fields** list, click **Add a line** +7. Select a non-relational field (e.g., `Name`) +8. Verify that the **Sub-Field** column is read-only (greyed out) for non-relational fields +9. Leave **Is Conditional** unchecked and **Imported Value** empty +10. Click **Save** + +**Expected:** + +- The rule appears in the list view with the name you entered +- The list view shows a drag handle for reordering by sequence + +## Test 2: Verify Duplicate Field Validation + +**Steps:** + +1. Open the matching rule created in Test 1 +2. In the **Fields** list, click **Add a line** +3. Select the same non-relational field (e.g., `Name`) that already exists in the list + +**Expected:** + +- A validation error appears: "Field 'Name', already exists!" +- The duplicate field is not added + +## Test 3: Sub-Field on Relational Fields + +**Steps:** + +1. Create a new matching rule for model `Contact` (res.partner) +2. In the **Fields** list, add a relational field (e.g., `Parent` which is a Many2one) +3. Verify that the **Sub-Field** column becomes editable and required +4. Select a sub-field (e.g., `Name`) +5. Save the rule + +**Expected:** + +- The field's computed name displays as `parent_id/name` (field/sub-field format) +- The rule saves without error + +## Test 4: Model Change Clears Fields + +**Steps:** + +1. Open the matching rule created in Test 3 +2. Change the **Model** to a different model (e.g., `Country`) +3. Observe the **Fields** list + +**Expected:** + +- All previously configured fields are cleared from the list +- The field domain updates to show only fields from the newly selected model + +## Test 5: Import with Matching — Skip Duplicates + +**Steps:** + +1. Create a matching rule for `Contact` with the `Name` field and **Overwrite Match** unchecked +2. Create a contact manually with Name = "Test Import Match" +3. Navigate to **Contacts** and click **Import records** (or use the gear/action menu) +4. Upload a CSV file containing: + ``` + name,email + Test Import Match,updated@example.com + Brand New Contact,new@example.com + ``` +5. In the import sidebar, locate the **Import Matching** section +6. Select the matching rule you created from the dropdown +7. Verify the **Overwrite Match** checkbox is unchecked +8. Verify the helper text reads: "Matched records will be skipped." +9. Click **Test** (dry run) + +**Expected:** + +- An info message appears: "1 to skip, 1 to create" +- No records are modified yet + +10. Click **Import** + +**Expected:** + +- A success notification appears: "1 skipped, 1 created" +- The existing "Test Import Match" contact retains its original email (not updated) +- A new "Brand New Contact" record is created with email `new@example.com` + +## Test 6: Import with Matching — Overwrite + +**Steps:** + +1. Using the same matching rule from Test 5, re-import the same CSV +2. In the import sidebar, select the matching rule +3. Check the **Overwrite Match** checkbox +4. Verify the helper text changes to: "Matched records will be overwritten with imported data." +5. Click **Test** (dry run) + +**Expected:** + +- An info message appears showing counts for records to overwrite and to create + +6. Click **Import** + +**Expected:** + +- A success notification with overwrite/create counts +- The "Test Import Match" contact now has email `updated@example.com` +- The "Brand New Contact" record either has a second copy created or is matched (depending on whether a Name rule matched it from the previous import) + +## Test 7: Conditional Matching + +**Steps:** + +1. Create a new matching rule for `Contact` with **Overwrite Match** checked +2. Add the `Name` field to the match fields +3. Check **Is Conditional** on the Name field +4. Set **Imported Value** to "Test Import Match" +5. Save the rule +6. Import a CSV: + ``` + name,email + Test Import Match,conditional@example.com + Some Other Name,other@example.com + ``` +7. Select this matching rule in the import sidebar + +**Expected:** + +- The row with name "Test Import Match" matches the condition and the existing record is overwritten +- The row with name "Some Other Name" does not match the condition (imported value differs from "Test Import Match"), so the entire rule is skipped for that row and a new record is created + +## Test 8: Import Matching Dropdown Visibility + +**Steps:** + +1. Navigate to a model that has NO matching rules configured (e.g., `Countries`) +2. Open the import dialog + +**Expected:** + +- The **Import Matching** section does not appear in the sidebar (the dropdown is hidden when no rules exist for the model) + +3. Navigate to a model that HAS matching rules (e.g., `Contacts`) +4. Open the import dialog + +**Expected:** + +- The **Import Matching** section appears with a dropdown listing all rules for that model +- The default selection is "-- No Matching --" +- The **Overwrite Match** checkbox is hidden until a rule is selected + +## Test 9: Overwrite Match Default from Rule + +**Steps:** + +1. Create a matching rule with **Overwrite Match** checked on the rule form +2. Open the import dialog for the rule's model +3. Select this rule from the **Import Matching** dropdown + +**Expected:** + +- The **Overwrite Match** checkbox is automatically checked (inherits the rule's default) +- The user can uncheck it to override the default for this import + +4. Select "-- No Matching --" from the dropdown + +**Expected:** + +- The **Overwrite Match** checkbox disappears + +## Test 10: Asynchronous Import (Large File) + +**Steps:** + +1. Create a matching rule for `Contact` with the `Name` field +2. Prepare a CSV with more than 100 rows of contact data (101+ rows of data, not counting the header) +3. Open the import dialog and upload the CSV +4. Select the matching rule +5. Click **Import** (not Test) + +**Expected:** + +- A notification appears: "Successfully added on Queue" +- The browser navigates back to the previous page +- The import is processed in the background via job worker +- Check job status by navigating to **Settings > Technical > Queue Jobs > Jobs** (requires technical user access) + +## Test 11: Multiple Matches Error + +**Steps:** + +1. Create two contacts with the same email: `duplicate@example.com` +2. Create a matching rule for `Contact` using the `Email` field +3. Import a CSV: + ``` + email,name + duplicate@example.com,Updated Name + ``` +4. Select the matching rule and click **Import** + +**Expected:** + +- An error is raised: "Multiple matches found for '...'" (where '...' is the name of the first matched record) +- No records are modified + +## Test 12: Security — Non-Admin Access + +**Steps:** + +1. Log in as a user without the **SPP Admin** role +2. Try to navigate to **Registry > Configuration > Import Match** + +**Expected:** + +- The **Import Match** menu item is not visible +- The user cannot access the matching rule configuration + +3. Open the import dialog for any model + +**Expected:** + +- The **Import Matching** section does not appear (the `searchRead` on `spp.import.match` returns no results due to access rules) diff --git a/spp_import_match/static/description/index.html b/spp_import_match/static/description/index.html index e8c9a720..488ec8e0 100644 --- a/spp_import_match/static/description/index.html +++ b/spp_import_match/static/description/index.html @@ -380,16 +380,15 @@

Key Capabilities

@@ -410,8 +409,8 @@

Key Models

specific model spp.import.match.fields -Individual fields used in a rule, -supports sub-fields +Individual field in a rule, with +optional sub-field @@ -421,24 +420,22 @@

Configuration

After installing:

  1. Navigate to Registry > Configuration > Import Match
    2. -
  2. Create a new matching rule and select the target model (e.g., +
  3. Create a matching rule and select the target model (e.g., res.partner)
  4. Add one or more fields to match on (e.g., national ID, or first name + date of birth)
  5. Enable Overwrite Match to update existing records when matches are found
  6. For conditional matching, enable Is Conditional on a field and -specify the expected imported value
    7. +set the expected imported value

UI Location

@@ -464,11 +461,11 @@

Security

Extension Points

@@ -478,16 +475,337 @@

Dependencies

Table of contents

+
+

Usage

+
+

Prerequisites

+
    +
  • The module OpenSPP Import Match is installed
    • +
  • You are logged in as a user with the SPP Admin +(spp_security.group_spp_admin) role
    • +
+
+
+

Test 1: Create a Matching Rule

+

Steps:

+
    +
  1. Navigate to Registry > Configuration > Import Match
    2. +
  2. Click New
    3. +
  3. Enter a name (e.g., “Match Partner by Name”)
    4. +
  4. In the Match Details tab, set Model to Contact +(res.partner)
    5. +
  5. Verify that Model Name and Model Description auto-populate +as res.partner and Contact
    6. +
  6. In the Fields list, click Add a line
    7. +
  7. Select a non-relational field (e.g., Name)
    8. +
  8. Verify that the Sub-Field column is read-only (greyed out) for +non-relational fields
    9. +
  9. Leave Is Conditional unchecked and Imported Value empty
    10. +
  10. Click Save
    11. +
+

Expected:

+
    +
  • The rule appears in the list view with the name you entered
    • +
  • The list view shows a drag handle for reordering by sequence
    • +
+
+
+

Test 2: Verify Duplicate Field Validation

+

Steps:

+
    +
  1. Open the matching rule created in Test 1
    2. +
  2. In the Fields list, click Add a line
    3. +
  3. Select the same non-relational field (e.g., Name) that already +exists in the list
    4. +
+

Expected:

+
    +
  • A validation error appears: “Field ‘Name’, already exists!”
    • +
  • The duplicate field is not added
    • +
+
+
+

Test 3: Sub-Field on Relational Fields

+

Steps:

+
    +
  1. Create a new matching rule for model Contact (res.partner)
    2. +
  2. In the Fields list, add a relational field (e.g., Parent +which is a Many2one)
    3. +
  3. Verify that the Sub-Field column becomes editable and required
    4. +
  4. Select a sub-field (e.g., Name)
    5. +
  5. Save the rule
    6. +
+

Expected:

+
    +
  • The field’s computed name displays as parent_id/name +(field/sub-field format)
    • +
  • The rule saves without error
    • +
+
+
+

Test 4: Model Change Clears Fields

+

Steps:

+
    +
  1. Open the matching rule created in Test 3
    2. +
  2. Change the Model to a different model (e.g., Country)
    3. +
  3. Observe the Fields list
    4. +
+

Expected:

+
    +
  • All previously configured fields are cleared from the list
    • +
  • The field domain updates to show only fields from the newly selected +model
    • +
+
+
+

Test 5: Import with Matching — Skip Duplicates

+

Steps:

+
    +

  1. Create a matching rule for Contact with the Name field and +Overwrite Match unchecked

    +
    2. +

  2. Create a contact manually with Name = “Test Import Match”

    +
    3. +

  3. Navigate to Contacts and click Import records (or use the +gear/action menu)

    +
    4. +

  4. Upload a CSV file containing:

    +
    +name,email
+Test Import Match,updated@example.com
+Brand New Contact,new@example.com
+
    +
    5. +

  5. In the import sidebar, locate the Import Matching section

    +
    6. +

  6. Select the matching rule you created from the dropdown

    +
    7. +

  7. Verify the Overwrite Match checkbox is unchecked

    +
    8. +

  8. Verify the helper text reads: “Matched records will be skipped.”

    +
    9. +

  9. Click Test (dry run)

    +
    10. +
+

Expected:

+
    +
  • An info message appears: “1 to skip, 1 to create”
    • +
  • No records are modified yet
    • +
+
    +
  1. Click Import
    2. +
+

Expected:

+
    +
  • A success notification appears: “1 skipped, 1 created”
    • +
  • The existing “Test Import Match” contact retains its original email +(not updated)
    • +
  • A new “Brand New Contact” record is created with email +new@example.com
    • +
+
+
+

Test 6: Import with Matching — Overwrite

+

Steps:

+
    +
  1. Using the same matching rule from Test 5, re-import the same CSV
    2. +
  2. In the import sidebar, select the matching rule
    3. +
  3. Check the Overwrite Match checkbox
    4. +
  4. Verify the helper text changes to: “Matched records will be +overwritten with imported data.”
    5. +
  5. Click Test (dry run)
    6. +
+

Expected:

+
    +
  • An info message appears showing counts for records to overwrite and to +create
    • +
+
    +
  1. Click Import
    2. +
+

Expected:

+
    +
  • A success notification with overwrite/create counts
    • +
  • The “Test Import Match” contact now has email updated@example.com
    • +
  • The “Brand New Contact” record either has a second copy created or is +matched (depending on whether a Name rule matched it from the previous +import)
    • +
+
+
+

Test 7: Conditional Matching

+

Steps:

+
    +

  1. Create a new matching rule for Contact with Overwrite Match +checked

    +
    2. +

  2. Add the Name field to the match fields

    +
    3. +

  3. Check Is Conditional on the Name field

    +
    4. +

  4. Set Imported Value to “Test Import Match”

    +
    5. +

  5. Save the rule

    +
    6. +

  6. Import a CSV:

    +
    +name,email
+Test Import Match,conditional@example.com
+Some Other Name,other@example.com
+
    +
    7. +

  7. Select this matching rule in the import sidebar

    +
    8. +
+

Expected:

+
    +
  • The row with name “Test Import Match” matches the condition and the +existing record is overwritten
    • +
  • The row with name “Some Other Name” does not match the condition +(imported value differs from “Test Import Match”), so the entire rule +is skipped for that row and a new record is created
    • +
+
+
+

Test 8: Import Matching Dropdown Visibility

+

Steps:

+
    +
  1. Navigate to a model that has NO matching rules configured (e.g., +Countries)
    2. +
  2. Open the import dialog
    3. +
+

Expected:

+
    +
  • The Import Matching section does not appear in the sidebar (the +dropdown is hidden when no rules exist for the model)
    • +
+
    +
  1. Navigate to a model that HAS matching rules (e.g., Contacts)
    2. +
  2. Open the import dialog
    3. +
+

Expected:

+
    +
  • The Import Matching section appears with a dropdown listing all +rules for that model
    • +
  • The default selection is “– No Matching –”
    • +
  • The Overwrite Match checkbox is hidden until a rule is selected
    • +
+
+
+

Test 9: Overwrite Match Default from Rule

+

Steps:

+
    +
  1. Create a matching rule with Overwrite Match checked on the rule +form
    2. +
  2. Open the import dialog for the rule’s model
    3. +
  3. Select this rule from the Import Matching dropdown
    4. +
+

Expected:

+
    +
  • The Overwrite Match checkbox is automatically checked (inherits +the rule’s default)
    • +
  • The user can uncheck it to override the default for this import
    • +
+
    +
  1. Select “– No Matching –” from the dropdown
    2. +
+

Expected:

+
    +
  • The Overwrite Match checkbox disappears
    • +
+
+
+

Test 10: Asynchronous Import (Large File)

+

Steps:

+
    +
  1. Create a matching rule for Contact with the Name field
    2. +
  2. Prepare a CSV with more than 100 rows of contact data (101+ rows of +data, not counting the header)
    3. +
  3. Open the import dialog and upload the CSV
    4. +
  4. Select the matching rule
    5. +
  5. Click Import (not Test)
    6. +
+

Expected:

+
    +
  • A notification appears: “Successfully added on Queue”
    • +
  • The browser navigates back to the previous page
    • +
  • The import is processed in the background via job worker
    • +
  • Check job status by navigating to Settings > Technical > Queue Jobs +> Jobs (requires technical user access)
    • +
+
+
+

Test 11: Multiple Matches Error

+

Steps:

+
    +

  1. Create two contacts with the same email: duplicate@example.com

    +
    2. +

  2. Create a matching rule for Contact using the Email field

    +
    3. +

  3. Import a CSV:

    +
    +email,name
+duplicate@example.com,Updated Name
+
    +
    4. +

  4. Select the matching rule and click Import

    +
    5. +
+

Expected:

+
    +
  • An error is raised: “Multiple matches found for ‘…’” (where ‘…’ is +the name of the first matched record)
    • +
  • No records are modified
    • +
+
+
+

Test 12: Security — Non-Admin Access

+

Steps:

+
    +
  1. Log in as a user without the SPP Admin role
    2. +
  2. Try to navigate to Registry > Configuration > Import Match
    3. +
+

Expected:

+
    +
  • The Import Match menu item is not visible
    • +
  • The user cannot access the matching rule configuration
    • +
+
    +
  1. Open the import dialog for any model
    2. +
+

Expected:

+
    +
  • The Import Matching section does not appear (the searchRead on +spp.import.match returns no results due to access rules)
    • +
+
-

Bug Tracker

+

Bug Tracker

Bugs are tracked on GitHub Issues. In case of trouble, please check there if your issue has already been reported. If you spotted it first, help us to smash it by providing a detailed and welcomed @@ -495,15 +813,15 @@

Bug Tracker

Do not contact contributors directly about support or help with technical issues.

-

Credits

+

Credits

-

Authors

+

Authors

  • OpenSPP.org
-

Maintainers

+

Maintainers

Current maintainers:

jeremi gonzalesedwin1123

This module is part of the OpenSPP/OpenSPP2 project on GitHub.

diff --git a/spp_import_match/tests/test_base_load.py b/spp_import_match/tests/test_base_load.py index 74c17149..61885a0d 100644 --- a/spp_import_match/tests/test_base_load.py +++ b/spp_import_match/tests/test_base_load.py @@ -33,11 +33,25 @@ def _create_match_rule(self, field_ids_data, overwrite=True): self.env["spp.import.match.fields"].create(data) return match - def test_load_no_usable_rules(self): - """When no usable rules exist, load() passes through to super().""" + def test_load_no_context_passes_through(self): + """When import_match_ids not in context, load() passes through to default Odoo import.""" result = self.env["res.partner"].load( ["name", "email"], - [["NoRuleTest99xyz", "norule@test.com"]], + [["NoCtxTest99xyz", "noctx@test.com"]], + ) + self.assertFalse(result["messages"]) + partner = self.env["res.partner"].search([("name", "=", "NoCtxTest99xyz")]) + self.assertEqual(len(partner), 1) + + def test_load_no_usable_rules(self): + """When import_match_ids in context but no usable rules, load() passes through.""" + result = ( + self.env["res.partner"] + .with_context(import_match_ids=[]) + .load( + ["name", "email"], + [["NoRuleTest99xyz", "norule@test.com"]], + ) ) self.assertFalse(result["messages"]) partner = self.env["res.partner"].search([("name", "=", "NoRuleTest99xyz")]) @@ -46,10 +60,14 @@ def test_load_no_usable_rules(self): def test_load_match_overwrite_true(self): """Match found + overwrite=True -> record updated.""" partner = self.env["res.partner"].create({"name": "OverwriteTrue99xyz", "email": "old@test.com"}) - self._create_match_rule([{"field_id": self.name_field.id}], overwrite=True) - result = self.env["res.partner"].load( - ["name", "email"], - [["OverwriteTrue99xyz", "new@test.com"]], + match = self._create_match_rule([{"field_id": self.name_field.id}], overwrite=True) + result = ( + self.env["res.partner"] + .with_context(import_match_ids=[match.id]) + .load( + ["name", "email"], + [["OverwriteTrue99xyz", "new@test.com"]], + ) ) self.assertFalse(result["messages"]) partner.invalidate_recordset() @@ -58,20 +76,28 @@ def test_load_match_overwrite_true(self): def test_load_match_overwrite_false(self): """Match found + overwrite=False -> record skipped.""" partner = self.env["res.partner"].create({"name": "OverwriteFalse99xyz", "email": "original@test.com"}) - self._create_match_rule([{"field_id": self.name_field.id}], overwrite=False) - self.env["res.partner"].load( - ["name", "email"], - [["OverwriteFalse99xyz", "changed@test.com"]], + match = self._create_match_rule([{"field_id": self.name_field.id}], overwrite=False) + ( + self.env["res.partner"] + .with_context(import_match_ids=[match.id]) + .load( + ["name", "email"], + [["OverwriteFalse99xyz", "changed@test.com"]], + ) ) partner.invalidate_recordset() self.assertEqual(partner.email, "original@test.com") def test_load_no_match_creates_record(self): """No match found -> new record created.""" - self._create_match_rule([{"field_id": self.name_field.id}]) - result = self.env["res.partner"].load( - ["name", "email"], - [["BrandNewPartner99xyz", "brandnew@test.com"]], + match = self._create_match_rule([{"field_id": self.name_field.id}]) + result = ( + self.env["res.partner"] + .with_context(import_match_ids=[match.id]) + .load( + ["name", "email"], + [["BrandNewPartner99xyz", "brandnew@test.com"]], + ) ) self.assertFalse(result["messages"]) partner = self.env["res.partner"].search([("name", "=", "BrandNewPartner99xyz")]) @@ -81,13 +107,17 @@ def test_load_no_match_creates_record(self): def test_load_multiple_records_mixed(self): """Mix of matched and new records in one import.""" self.env["res.partner"].create({"name": "ExistingMixed99xyz", "email": "existing@test.com"}) - self._create_match_rule([{"field_id": self.name_field.id}], overwrite=True) - result = self.env["res.partner"].load( - ["name", "email"], - [ - ["ExistingMixed99xyz", "updated@test.com"], - ["NewMixed99xyz", "newmixed@test.com"], - ], + match = self._create_match_rule([{"field_id": self.name_field.id}], overwrite=True) + result = ( + self.env["res.partner"] + .with_context(import_match_ids=[match.id]) + .load( + ["name", "email"], + [ + ["ExistingMixed99xyz", "updated@test.com"], + ["NewMixed99xyz", "newmixed@test.com"], + ], + ) ) self.assertFalse(result["messages"]) existing = self.env["res.partner"].search([("name", "=", "ExistingMixed99xyz")]) @@ -117,11 +147,15 @@ def test_load_with_import_match_ids_context(self): def test_load_appends_id_field(self): """Auto-appends 'id' when not in fields list.""" - self._create_match_rule([{"field_id": self.name_field.id}]) + match = self._create_match_rule([{"field_id": self.name_field.id}]) fields = ["name", "email"] - self.env["res.partner"].load( - fields, - [["AppendIdTest99xyz", "appendid@test.com"]], + ( + self.env["res.partner"] + .with_context(import_match_ids=[match.id]) + .load( + fields, + [["AppendIdTest99xyz", "appendid@test.com"]], + ) ) self.assertIn("id", fields) diff --git a/spp_import_match/tests/test_import_match_model.py b/spp_import_match/tests/test_import_match_model.py index bc82b80e..801c3dc3 100644 --- a/spp_import_match/tests/test_import_match_model.py +++ b/spp_import_match/tests/test_import_match_model.py @@ -1,5 +1,6 @@ # Part of OpenSPP. See LICENSE file for full copyright and licensing details. +from odoo import Command from odoo.exceptions import ValidationError from odoo.tests import TransactionCase, tagged @@ -87,7 +88,7 @@ def test_match_find_no_match(self): self.assertFalse(result.id) def test_match_find_multiple_matches_raises(self): - """Test _match_find raises ValidationError on multiple matches.""" + """Test _match_find raises ValidationError when multiple matches found.""" self.env["res.partner"].create({"name": "DuplicateMatchTest"}) self.env["res.partner"].create({"name": "DuplicateMatchTest"}) match = self._create_match_rule([{"field_id": self.name_field.id}]) @@ -211,6 +212,39 @@ def test_match_find_sub_field(self): ) self.assertEqual(result, parent) + def test_constrains_duplicate_name(self): + """_check_duplicate_name raises on duplicate rule names.""" + self.env["spp.import.match"].create({"name": "DuplicateNameTest", "model_id": self.res_partner_model.id}) + with self.assertRaises(ValidationError): + self.env["spp.import.match"].create({"name": "DuplicateNameTest", "model_id": self.res_partner_model.id}) + + def test_constrains_duplicate_non_relational_field(self): + """_check_duplicate_fields raises on duplicate non-relational fields on save.""" + with self.assertRaises(ValidationError): + self.env["spp.import.match"].create( + { + "model_id": self.res_partner_model.id, + "field_ids": [ + Command.create({"field_id": self.name_field.id}), + Command.create({"field_id": self.name_field.id}), + ], + } + ) + + def test_constrains_allows_duplicate_relational_field(self): + """_check_duplicate_fields allows duplicate relational fields.""" + parent_id_field = self.env["ir.model.fields"].search( + [("name", "=", "parent_id"), ("model_id", "=", self.res_partner_model.id)], + limit=1, + ) + match = self._create_match_rule( + [ + {"field_id": parent_id_field.id}, + {"field_id": parent_id_field.id}, + ] + ) + self.assertEqual(len(match.field_ids), 2) + def test_match_find_multiple_combinations(self): """_match_find iterates rules; first matching single result wins.""" partner = self.env["res.partner"].create({"name": "MultiCombo99xyz", "email": "multicombo@test.com"}) diff --git a/spp_import_match/tests/test_res_partner_import_match.py b/spp_import_match/tests/test_res_partner_import_match.py index 4e1f7d82..19c47df3 100644 --- a/spp_import_match/tests/test_res_partner_import_match.py +++ b/spp_import_match/tests/test_res_partner_import_match.py @@ -110,21 +110,23 @@ def create_matching_name(self): def test_01_res_partner_change_email_by_name(self): """Change email based on given_name, family_name.""" - self.create_matching_given_family_name() + import_match = self.create_matching_given_family_name() file_path = self.get_file_path_2() record = self._base_import_record("res.partner", file_path) - record.execute_import(["given_name", "family_name", "name", "email"], [], OPTIONS) + options = dict(OPTIONS, import_match_ids=[import_match.id], overwrite_match=True) + record.execute_import(["given_name", "family_name", "name", "email"], [], options) self._test_applicant.env.cache.invalidate() self.assertEqual(self._test_applicant.email, "rufinorenaud@gmail.com") def test_02_res_partner_change_email_by_group_name(self): """Change email based on name.""" - self.create_matching_name() + import_match = self.create_matching_name() file_path = self.get_file_path_1() record = self._base_import_record("res.partner", file_path) - record.execute_import(["name", "email"], ["name", "email"], OPTIONS) + options = dict(OPTIONS, import_match_ids=[import_match.id], overwrite_match=True) + record.execute_import(["name", "email"], ["name", "email"], options) self._test_hh.env.cache.invalidate() self.assertEqual(self._test_hh.email, "renaudhh@gmail.com")