diff --git a/import_processor/README.rst b/import_processor/README.rst new file mode 100644 index 00000000000..72bed5aeee2 --- /dev/null +++ b/import_processor/README.rst @@ -0,0 +1,102 @@ +.. image:: https://odoo-community.org/readme-banner-image + :target: https://odoo-community.org/get-involved?utm_source=readme + :alt: Odoo Community Association + +================ +Import Processor +================ + +.. + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! This file is generated by oca-gen-addon-readme !! + !! changes will be overwritten. !! + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + !! source digest: sha256:118ec811b0d4cc99553b35e5fc3c561e8753f5488439d41a8cfdc16e51718f55 + !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +.. |badge1| image:: https://img.shields.io/badge/maturity-Beta-yellow.png + :target: https://odoo-community.org/page/development-status + :alt: Beta +.. |badge2| image:: https://img.shields.io/badge/license-AGPL--3-blue.png + :target: http://www.gnu.org/licenses/agpl-3.0-standalone.html + :alt: License: AGPL-3 +.. |badge3| image:: https://img.shields.io/badge/github-OCA%2Fserver--tools-lightgray.png?logo=github + :target: https://github.com/OCA/server-tools/tree/19.0/import_processor + :alt: OCA/server-tools +.. |badge4| image:: https://img.shields.io/badge/weblate-Translate%20me-F47D42.png + :target: https://translation.odoo-community.org/projects/server-tools-19-0/server-tools-19-0-import_processor + :alt: Translate me on Weblate +.. |badge5| image:: https://img.shields.io/badge/runboat-Try%20me-875A7B.png + :target: https://runboat.odoo-community.org/builds?repo=OCA/server-tools&target_branch=19.0 + :alt: Try me on Runboat + +|badge1| |badge2| |badge3| |badge4| |badge5| + +This modules offers users additional imports of JSON, CSV, XLSX and XML +files based on configurable code snippets. This allows import of data +from structured data without installation of modules. + +**Table of contents** + +.. contents:: + :local: + +Usage +===== + +To use this module, the administrator can create different processors in +Settings > Technical > Import Processors. It offers 3 different kind of +snippets depending on the import stage: + +1. Pre-Processor: This snippet is executed once after loading the file. + This snippet is useful to set global variables or set configurations + prior to the import of each entry +2. Processor: This snippet is executed on single entries or chunks of + entries depending on the configuration. +3. Post-Processor: This snippet is executed after the entries are + processed and can be used to clean up things in the database or log + the processed data. + +After configuration every user can use the processors Favorites > Import +Processor on the specified models. + +Bug Tracker +=========== + +Bugs are tracked on `GitHub Issues `_. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +`feedback `_. + +Do not contact contributors directly about support or help with technical issues. + +Credits +======= + +Authors +------- + +* initOS GmbH + +Contributors +------------ + +- Dhara Solanki +- Florian Kantelberg + +Maintainers +----------- + +This module is maintained by the OCA. + +.. image:: https://odoo-community.org/logo.png + :alt: Odoo Community Association + :target: https://odoo-community.org + +OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use. + +This module is part of the `OCA/server-tools `_ project on GitHub. + +You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute. diff --git a/import_processor/__init__.py b/import_processor/__init__.py new file mode 100644 index 00000000000..023f1072fde --- /dev/null +++ b/import_processor/__init__.py @@ -0,0 +1,4 @@ +# © 2022 initOS GmbH +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from . import models, wizards diff --git a/import_processor/__manifest__.py b/import_processor/__manifest__.py new file mode 100644 index 00000000000..affc7d1ff95 --- /dev/null +++ b/import_processor/__manifest__.py @@ -0,0 +1,36 @@ +# © 2022 initOS GmbH +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +{ + "name": "Import Processor", + "summary": "Generic import processor", + "license": "AGPL-3", + "version": "19.0.1.0.0", + "website": "https://github.com/OCA/server-tools", + "application": True, + "author": "initOS GmbH, Odoo Community Association (OCA)", + "category": "Tools", + "depends": [ + "base", + "web", + ], + "external_dependencies": { + "python": [ + "jsonpath_ng", + ], + }, + "data": [ + "security/ir.model.access.csv", + "views/import_processor_views.xml", + "wizards/import_processor_wizard_views.xml", + ], + "demo": [ + "data/processor_data.xml", + ], + "assets": { + "web.assets_backend": [ + "import_processor/static/src/*.js", + "import_processor/static/src/*.xml", + ], + }, +} diff --git a/import_processor/data/processor_data.xml b/import_processor/data/processor_data.xml new file mode 100644 index 00000000000..fcadaea1f06 --- /dev/null +++ b/import_processor/data/processor_data.xml @@ -0,0 +1,100 @@ + + + + Import Res Partner (CSV with chunk size) + + + csv + 2 + +regex = r"^[a-z0-9]+[\._]?[a-z0-9]+[@][a-zA-Z]+[.][a-zA-Z]{2,3}$" +fields_list = ["name", "email", "phone", "street", "street2", "zip", "city"] + + +for contact in entry: + email = contact.get("email") + if re.search(regex, email): + data = {key: value for key, value in contact.items() if key in fields_list} + rec = model.search([("email", "=", email)]) + if rec: + rec.update(data) + else: + rec = model.create(data) + + records |= rec + + +log("Processed the following records: %s", records) + + + + + Import Res Partner (XML) + + + xml + //Customer + +# Regex Email Validation +regex = r"^[a-z0-9]+[\._]?[a-z0-9]+[@][a-zA-Z]+[.][a-zA-Z]{2,3}$" + +fields_list = ["name", "email", "phone", "street", "street2", "zip", "city"] + +# Converts XML file to Dictionary +def xml_to_dict(xml_element): + result = {} + for child in xml_element: + if len(child) == 0: + result[child.tag] = child.text + else: + result[child.tag] = xml_to_dict(child) + return result + + +entry = xml_to_dict(entry) + +email = entry.get("email") +if re.search(regex, email): + rec = model.search([("email", "=", email)]) + data = {key: value for key, value in entry.items() if key in fields_list} + if rec: + rec.update(data) + else: + rec = model.create(data) + + records |= rec + + +log("Processed the following records: %s", records) + + + + + Import Res Partner (JSON) + + + json + 'contact' + +# Regex Email Validation +regex = r"^[a-z0-9]+[\._]?[a-z0-9]+[@][a-zA-Z]+[.][a-zA-Z]{2,3}$" +fields_list = ["name", "email", "phone", "street", "street2", "zip", "city"] + + +for contact in entry: + email = contact.get("email") + if re.search(regex, email): + rec = model.search([("email", "=", email)]) + data = {key: value for key, value in contact.items() if key in fields_list} + if rec: + rec.update(data) + else: + rec = model.create(data) + + records |= rec + + +log("Processed the following records: %s", records) + + + diff --git a/import_processor/models/__init__.py b/import_processor/models/__init__.py new file mode 100644 index 00000000000..332d2633a72 --- /dev/null +++ b/import_processor/models/__init__.py @@ -0,0 +1,4 @@ +# © 2022 initOS GmbH +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from . import import_processor diff --git a/import_processor/models/import_processor.py b/import_processor/models/import_processor.py new file mode 100644 index 00000000000..574f67c002d --- /dev/null +++ b/import_processor/models/import_processor.py @@ -0,0 +1,441 @@ +# © 2022 initOS GmbH +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). +import csv +import io +import json +import logging +import tempfile +import uuid +import zipfile +from itertools import tee + +import chardet +from lxml import etree +from pytz import timezone + +from odoo import api, fields, models +from odoo.exceptions import UserError +from odoo.tools import safe_eval + +_logger = logging.getLogger(__name__) + +try: + # Optional feature for xlsx + import openpyxl +except ImportError: + _logger.warning( + "The openpyxl python library is not installed. The XLSX feature isn't available" + ) + openpyxl = None + + +try: + # Optional feature for JSONPath + import jsonpath_ng as jsonpath +except ImportError: + _logger.warning( + "The jsonpath-ng python library is not installed. " + "The JSONPath feature isn't available" + ) + jsonpath = None + + +re = safe_eval.wrap_module( + __import__("re"), + [ + "compile", + "escape", + "findall", + "finditer", + "fullmatch", + "match", + "search", + "sub", + "subn", + ], +) + + +def chunking(items, size): + if size < 1: + yield from items + return + + chunk = [] + for item in items: + chunk.append(item) + if len(chunk) >= size: + yield chunk + chunk = [] + + if chunk: + yield chunk + + +class ImportProcessor(models.Model): + _name = "import.processor" + _description = "Generic Import Processor" + _order = "name" + + def _get_file_types(self): + return [ + ("csv", "CSV"), + ("json", "JSON"), + ("xlsx", "XLSX"), + ("xml", "XML"), + ] + + def _get_csv_delimiter(self): + return [ + ("comma", self.env._("Comma")), + ("semicolon", self.env._("Semicolon")), + ("tab", self.env._("Tab")), + ] + + def _get_compression(self): + return [ + ("zip_one", self.env._("Zipped File")), + ("zip_all", self.env._("Multiple Zipped Files")), + ] + + def _get_default_code(self, entry=False): + variables = self.default_variables() + if not entry: + for key in ("entry", "key"): + variables.pop(key, None) + + desc = "\n".join(f"# - {v}: {desc}" for v, desc in variables.items()) + return f"# Possible variables:\n{desc}\n\n" + + name = fields.Char(translate=True, required=True) + active = fields.Boolean(default=True) + model_id = fields.Many2one( + "ir.model", + string="Import Model", + ondelete="cascade", + required=True, + ) + model_name = fields.Char(related="model_id.model", related_sudo=True) + file_type = fields.Selection(_get_file_types, default="csv", required=True) + processor = fields.Text(default=lambda self: self._get_default_code(True)) + preprocessor = fields.Text(default=lambda self: self._get_default_code()) + postprocessor = fields.Text(default=lambda self: self._get_default_code()) + help_text = fields.Html(compute="_compute_help_text", store=False) + file_encoding = fields.Char() + compression = fields.Selection(_get_compression) + chunk_size = fields.Integer( + default=0, + help="If greater than 0 it script will get a list of entries instead to " + "speed up the processing up. Chunking is only supported for streamable data " + "line csv or xlsx", + ) + + csv_delimiter = fields.Selection( + _get_csv_delimiter, + "CSV Delimiter", + default="comma", + ) + csv_row_offset = fields.Integer("Row Offset", default=0) + csv_quotechar = fields.Char("Quote Char", default='"', size=1) + + json_path_entry = fields.Char("JSONPath Entry") + + xml_path_entry = fields.Char("XPath Entry") + xml_namespaces = fields.Text("XML Namespace") + + tabular_sheet_name = fields.Char("Sheet Name") + tabular_sheet_index = fields.Integer("Sheet Index", default=0) + tabular_col_offset = fields.Integer("Sheet Col Offset", default=0) + tabular_row_offset = fields.Integer("Sheet Row Offset", default=0) + + @api.model + def _search_display_name( + self, name="", args=None, operator="ilike", limit=100, name_get_uid=None + ): + return super(ImportProcessor, self.sudo())._search_display_name( + name=name, + args=args, + operator=operator, + limit=limit, + name_get_uid=name_get_uid, + ) + + def _compute_help_text(self): + variables = self.default_variables() + lines = [] + for var, desc in variables.items(): + var = (f"{v.strip()}" for v in var.split(",")) + lines.append(f"
  • {', '.join(sorted(var))}: {desc}
    • ") + + desc = "\n".join(lines) + self.update({"help_text": f"
      {desc}
    "}) + + def _get_eval_context(self): + self.ensure_one() + + def log(message, *args, level="info"): + level = { + "debug": logging.DEBUG, + "info": logging.INFO, + "warning": logging.WARNING, + "error": logging.ERROR, + "critical": logging.CRITICAL, + }.get(level, logging.INFO) + _logger.log(level, f"{self} | {message}", *args) + + return { + "datetime": safe_eval.datetime, + "env": self.env, + "log": log, + "model": self._get_model(), + "re": re, + "time": safe_eval.time, + "timezone": timezone, + "UserError": UserError, + } + + def _csv_get_delimiter(self): + self.ensure_one() + return { + "comma": ",", + "semicolon": ";", + "tab": "\t", + }.get(self.csv_delimiter, ",") + + def _get_model(self): + self.ensure_one() + return self.env[self.sudo().model_id.model] + + def default_variables(self): + """Informations about the available variables in the python code""" + return { + "entry": "The data entry to import into odoo", + "env": "Odoo Environment on which the import is triggered", + "header": "Will be used as header instead of reading it if CSV", + "key": "The key/index of the entry if JSON", + "log": "Logging functions", + "model": "Odoo Model of the record on which the action is triggered", + "nsmap": "The XML namespaces if xml", + "process_uuid": "Unique UUID for each import process", + "reader": "CSV reader if CSV. The importing continues", + "records": "The already imported records", + "root": "XML root if XML or JSON", + "sheet": "Tabular sheet if XLSX", + "datetime, re, time, timezone": "useful Python libraries", + "UserError": "Warning Exception to use with raise", + } + + def _process_entry(self, process_uuid, entry, localdict, **kwargs): + # Reset the pre-defined values + localdict.update(self._get_eval_context()) + localdict.update({"entry": entry, "process_uuid": process_uuid, **kwargs}) + + safe_eval.safe_eval(self.processor, localdict, mode="exec") + localdict.pop("entry", None) + for key in kwargs: + localdict.pop(key, None) + + def process_entry(self, process_uuid, entry, localdict, **kwargs): + # Reset the pre-defined values + self._process_entry(process_uuid, entry, localdict, **kwargs) + + def _pre_process(self, process_uuid, localdict): + if self.preprocessor: + localdict["process_uuid"] = process_uuid + localdict.update(self._get_eval_context()) + safe_eval.safe_eval(self.preprocessor, localdict, mode="exec") + + def _post_process(self, process_uuid, localdict): + # Reset the pre-defined values + if self.postprocessor: + localdict["process_uuid"] = process_uuid + localdict.update(self._get_eval_context()) + safe_eval.safe_eval(self.postprocessor, localdict, mode="exec") + + def _process_csv(self, process_uuid, file): + if isinstance(file, bytes): + encoding = self.file_encoding or chardet.detect(file)["encoding"] or "utf-8" + file = io.TextIOWrapper(io.BytesIO(file), encoding=encoding.lower()) + elif isinstance(file, str): + encoding = self.file_encoding or "utf-8" + file = io.TextIOWrapper(io.StringIO(file), encoding=encoding) + + reader = csv.reader( + file, + delimiter=self._csv_get_delimiter(), + quotechar=self.csv_quotechar or '"', + ) + reader, backup = tee(reader) + localdict = {"records": self._get_model(), "reader": reader} + + # Apply the pre-processor to initialize the environment + self._pre_process(process_uuid, localdict) + + localdict.pop("reader", None) + # The next line is the CSV field header. Afterwards parse every line + for _x in range(self.csv_row_offset): + next(backup) + + header = localdict.pop("header", None) + if header is None: + header = next(backup) + + def prepare_line(line): + return dict(list(zip(header, line, strict=False))[::-1]) + + for data in chunking(map(prepare_line, backup), self.chunk_size): + self.process_entry(process_uuid, data, localdict) + + # Do some final post processing + self._post_process(process_uuid, localdict) + return localdict.get("records", self._get_model()) + + def _process_json(self, process_uuid, file): + if isinstance(file, bytes): + encoding = self.file_encoding or chardet.detect(file)["encoding"] or "utf-8" + file = io.TextIOWrapper(io.BytesIO(file), encoding=encoding.lower()) + elif isinstance(file, str): + encoding = self.file_encoding or "utf-8" + file = io.TextIOWrapper(io.StringIO(file), encoding=encoding) + + root = json.load(file) + localdict = {"records": self._get_model(), "root": root} + + # Apply the pre-processor to initialize the environment + self._pre_process(process_uuid, localdict) + + # Check and apply the JSONpath if module is available + if self.json_path_entry and not jsonpath: + raise UserError( + self.env._( + "The JSONPath isn't available because the module jsonpath-ng is " + "missing. Please contact your administrator" + ) + ) + + if self.json_path_entry: + found = jsonpath.parse(self.json_path_entry).find(root) + if all(isinstance(x.path, jsonpath.Index) for x in found): + root = [x.value for x in found] + elif all(isinstance(x.path, jsonpath.Fields) for x in found): + root = {str(x.path): x.value for x in found} + else: + raise UserError(self.env._("Unexpected JSON file")) + + if isinstance(root, dict): + iterator = root.items() + else: + iterator = enumerate(root) + + # Iterate over the entries + for key, entry in iterator: + self.process_entry(process_uuid, entry, localdict, key=key) + + # Do some final post processing + self._post_process(process_uuid, localdict) + return localdict.get("records", self._get_model()) + + def _process_xml(self, process_uuid, file): + root = etree.fromstring(file) + + # Get the namespace + if self.xml_namespaces: + nsmap = safe_eval.safe_eval(self.xml_namespaces) + else: + nsmap = None + + localdict = {"records": self._get_model(), "root": root, "nsmap": nsmap} + + # Apply the pre-processor to initialize the environment + self._pre_process(process_uuid, localdict) + + for entry in root.xpath(self.xml_path_entry, namespaces=nsmap): + self.process_entry(process_uuid, entry, localdict) + + # Do some final post processing + self._post_process(process_uuid, localdict) + return localdict.get("records", self._get_model()) + + def _process_xlsx(self, process_uuid, file): + if not openpyxl: + raise UserError( + self.env._( + "The XLSX processing isn't available because openpyxl is " + "missing. Please contact your administrator" + ) + ) + + if isinstance(file, bytes): + with tempfile.NamedTemporaryFile("wb", suffix=".xlsx") as fp: + fp.write(file) + fp.flush() + workbook = openpyxl.load_workbook(fp.name) + elif isinstance(file, str): + with tempfile.NamedTemporaryFile("w", suffix=".xlsx") as fp: + fp.write(file) + fp.flush() + + workbook = openpyxl.load_workbook(fp.name) + + if self.tabular_sheet_name: + sheet = workbook[self.tabular_sheet_name] + else: + sheet_name = workbook.sheetnames[self.tabular_sheet_index] + sheet = workbook[sheet_name] + + localdict = {"records": self._get_model(), "sheet": sheet} + + # Apply the pre-processor to initialize the environment + self._pre_process(process_uuid, localdict) + + localdict.pop("workbook", None) + + reader = sheet.rows + for _x in range(self.tabular_row_offset): + next(reader) + + def prepare_line(line): + entry = [ + (h, cell.value) for h, cell in zip(header, line[col:], strict=False) + ] + return dict(entry[::-1]) + + # The next line is the field header. Afterwards parse every line + col = self.tabular_col_offset + header = [x.value for x in next(reader)[col:]] + for data in chunking(map(prepare_line, reader), self.chunk_size): + self.process_entry(process_uuid, data, localdict) + + # Do some final post processing + self._post_process(process_uuid, localdict) + return localdict.get("records", self._get_model()) + + def process(self, file): + self.ensure_one() + + process_uuid = str(uuid.uuid4()) + + method = getattr(self, f"_process_{self.file_type}", None) + if not callable(method): + raise NotImplementedError() + + if self.compression in ("zip_one", "zip_all"): + with tempfile.NamedTemporaryFile("wb+") as fp: + fp.write(file) + fp.flush() + + try: + zipped = zipfile.ZipFile(fp.name) + except zipfile.BadZipfile: + _logger.warning("File is no zip. Falling back: %s", self) + return method(process_uuid, file) + + if self.compression == "zip_one" and len(zipped.filelist) != 1: + raise UserError(self.env._("Expected only 1 file.")) + + result = self._get_model() + for zipped_file in zipped.filelist: + result |= method(process_uuid, zipped.read(zipped_file)) + return result + + return method(process_uuid, file) diff --git a/import_processor/pyproject.toml b/import_processor/pyproject.toml new file mode 100644 index 00000000000..4231d0cccb3 --- /dev/null +++ b/import_processor/pyproject.toml @@ -0,0 +1,3 @@ +[build-system] +requires = ["whool"] +build-backend = "whool.buildapi" diff --git a/import_processor/readme/CONTRIBUTORS.md b/import_processor/readme/CONTRIBUTORS.md new file mode 100644 index 00000000000..dbd7675ec94 --- /dev/null +++ b/import_processor/readme/CONTRIBUTORS.md @@ -0,0 +1,2 @@ +- Dhara Solanki \ +- Florian Kantelberg \ diff --git a/import_processor/readme/DESCRIPTION.md b/import_processor/readme/DESCRIPTION.md new file mode 100644 index 00000000000..c4047e569ad --- /dev/null +++ b/import_processor/readme/DESCRIPTION.md @@ -0,0 +1,3 @@ +This modules offers users additional imports of JSON, CSV, XLSX and XML +files based on configurable code snippets. This allows import of data +from structured data without installation of modules. diff --git a/import_processor/readme/USAGE.md b/import_processor/readme/USAGE.md new file mode 100644 index 00000000000..46f8ead3423 --- /dev/null +++ b/import_processor/readme/USAGE.md @@ -0,0 +1,15 @@ +To use this module, the administrator can create different processors in +Settings \> Technical \> Import Processors. It offers 3 different kind +of snippets depending on the import stage: + +1. Pre-Processor: This snippet is executed once after loading the file. + This snippet is useful to set global variables or set configurations + prior to the import of each entry +2. Processor: This snippet is executed on single entries or chunks of + entries depending on the configuration. +3. Post-Processor: This snippet is executed after the entries are + processed and can be used to clean up things in the database or log + the processed data. + +After configuration every user can use the processors Favorites \> +Import Processor on the specified models. diff --git a/import_processor/security/ir.model.access.csv b/import_processor/security/ir.model.access.csv new file mode 100644 index 00000000000..09230c6a463 --- /dev/null +++ b/import_processor/security/ir.model.access.csv @@ -0,0 +1,4 @@ +id,name,model_id:id,group_id:id,perm_read,perm_write,perm_create,perm_unlink +access_import_processor,access_import_processor,model_import_processor,base.group_user,1,0,0,0 +access_import_processor_admin,access_import_processor_admin,model_import_processor,base.group_system,1,1,1,1 +access_import_processor_wizard,access_import_processor_wizard,model_import_processor_wizard,base.group_user,1,1,1,1 diff --git a/import_processor/static/description/index.html b/import_processor/static/description/index.html new file mode 100644 index 00000000000..3eac3195a1f --- /dev/null +++ b/import_processor/static/description/index.html @@ -0,0 +1,451 @@ + + + + + +README.rst + + + +
    + + + +Odoo Community Association + +
    +

    Import Processor

    + +

    Beta License: AGPL-3 OCA/server-tools Translate me on Weblate Try me on Runboat

    +

    This modules offers users additional imports of JSON, CSV, XLSX and XML +files based on configurable code snippets. This allows import of data +from structured data without installation of modules.

    +

    Table of contents

    + +
    +

    Usage

    +

    To use this module, the administrator can create different processors in +Settings > Technical > Import Processors. It offers 3 different kind of +snippets depending on the import stage:

    +
      +
    1. Pre-Processor: This snippet is executed once after loading the file. +This snippet is useful to set global variables or set configurations +prior to the import of each entry
      2. +
    2. Processor: This snippet is executed on single entries or chunks of +entries depending on the configuration.
      3. +
    3. Post-Processor: This snippet is executed after the entries are +processed and can be used to clean up things in the database or log +the processed data.
      4. +
    +

    After configuration every user can use the processors Favorites > Import +Processor on the specified models.

    +
    +
    +

    Bug Tracker

    +

    Bugs are tracked on GitHub Issues. +In case of trouble, please check there if your issue has already been reported. +If you spotted it first, help us to smash it by providing a detailed and welcomed +feedback.

    +

    Do not contact contributors directly about support or help with technical issues.

    +
    +
    +

    Credits

    +
    +

    Authors

    +
      +
    • initOS GmbH
      • +
    +
    +
    +

    Contributors

    + +
    +
    +

    Maintainers

    +

    This module is maintained by the OCA.

    + +Odoo Community Association + +

    OCA, or the Odoo Community Association, is a nonprofit organization whose +mission is to support the collaborative development of Odoo features and +promote its widespread use.

    +

    This module is part of the OCA/server-tools project on GitHub.

    +

    You are welcome to contribute. To learn how please visit https://odoo-community.org/page/Contribute.

    +
    +
    +
    +
    + + diff --git a/import_processor/static/src/processor_menu.esm.js b/import_processor/static/src/processor_menu.esm.js new file mode 100644 index 00000000000..17b4ac98e32 --- /dev/null +++ b/import_processor/static/src/processor_menu.esm.js @@ -0,0 +1,58 @@ +import {Component} from "@odoo/owl"; +import {DropdownItem} from "@web/core/dropdown/dropdown_item"; +import {registry} from "@web/core/registry"; +import {useService} from "@web/core/utils/hooks"; +import {_t} from "@web/core/l10n/translation"; + +const favoriteMenuRegistry = registry.category("favoriteMenu"); + +/** + * Import Records menu + * + * This component is used to import the records for particular model. + * + * @extends Component + */ +export class GenImportMenu extends Component { + static template = "import_processor.ProcessorMenu"; + static components = {DropdownItem}; + + setup() { + this.action = useService("action"); + } + + /** + * @private + */ + _onImportClick() { + const {resModel} = this.env.searchModel; + this.action.doAction({ + type: "ir.actions.act_window", + name: _t("Import Processor"), + target: "new", + view_mode: "form", + res_model: "import.processor.wizard", + views: [[false, "form"]], + context: { + default_model: resModel, + }, + }); + } +} + +export const importerItem = { + Component: GenImportMenu, + groupNumber: 4, + isDisplayed: (env) => { + const {config, isSmall} = env; + return ( + !isSmall && + config.actionType === "ir.actions.act_window" && + ["kanban", "list"].includes(config.viewType) && + Boolean(JSON.parse(config.viewArch.getAttribute("import") || "1")) && + Boolean(JSON.parse(config.viewArch.getAttribute("create") || "1")) + ); + }, +}; + +favoriteMenuRegistry.add("importer", importerItem, {sequence: 1}); diff --git a/import_processor/static/src/processor_menu.xml b/import_processor/static/src/processor_menu.xml new file mode 100644 index 00000000000..db9c42984ac --- /dev/null +++ b/import_processor/static/src/processor_menu.xml @@ -0,0 +1,8 @@ + + + + + Import Processor + + + diff --git a/import_processor/tests/Customers.xml b/import_processor/tests/Customers.xml new file mode 100644 index 00000000000..23ea31ba966 --- /dev/null +++ b/import_processor/tests/Customers.xml @@ -0,0 +1,25 @@ + + + + + Great Lakes Food Market + (503) 555-7555 + max.ryne@example.com + 2732 Baker Blvd. + Eugene + OR + 97403 + USA + + + Max + (503) 555-7555 + max@example.com + 2732 Baker Blvd. + Eugene + OR + 97403 + USA + + + diff --git a/import_processor/tests/__init__.py b/import_processor/tests/__init__.py new file mode 100644 index 00000000000..a604ceca122 --- /dev/null +++ b/import_processor/tests/__init__.py @@ -0,0 +1 @@ +from . import test_import_processor diff --git a/import_processor/tests/contacts.csv b/import_processor/tests/contacts.csv new file mode 100644 index 00000000000..46fd57481c4 --- /dev/null +++ b/import_processor/tests/contacts.csv @@ -0,0 +1,5 @@ +phone,email,city,country_id,name,street,street2,zip +(870)-931-0505,max.ryne@example.com,Finton,233,Max Ryne,4311 Owagner Lane,Redmond,98052 +(870)-932-0505,jack.cooper@123.com,London,,Jace Cooper,Capital Exchange Way,Brentfod,TW8 0EX +(870)-932-0505,jack.cooper@example.com,London,,Jace Cooper,Capital Exchange Way,Brentford,TW8 0EX +(870)-859-0505,jack.cooper@example.com,London,,Jace Cooper,Capital Exchange Way,Brentford,TW8 0EX diff --git a/import_processor/tests/contacts.json b/import_processor/tests/contacts.json new file mode 100644 index 00000000000..2debb7e0f4b --- /dev/null +++ b/import_processor/tests/contacts.json @@ -0,0 +1,34 @@ +{ + "contact": [ + { + "phone": "(870)-931-0505", + "email": "max.ryne@example.com", + "city": "Finton", + "country_id": 223, + "name": "Max Ryne", + "street": "4311 Owagner Lane", + "street2": "Redmond", + "zip": "98052" + }, + { + "phone": "(870)-935-0505", + "email": "emanuel@example.com", + "city": "Miami", + "country_id": 223, + "name": "Emanuel Nguyen", + "street": "3853 Rinehart Road", + "street2": "", + "zip": "33128" + }, + { + "phone": "(870)-856-4586", + "email": "emanuel@example.com", + "city": "Miami", + "country_id": 223, + "name": "Emanuel Nguyen", + "street": "3853 Rinehart Road", + "street2": "", + "zip": "33128" + } + ] +} diff --git a/import_processor/tests/contacts.xml b/import_processor/tests/contacts.xml new file mode 100644 index 00000000000..9b66a1b71b3 --- /dev/null +++ b/import_processor/tests/contacts.xml @@ -0,0 +1,45 @@ + + + + + Great Lakes Food Market + (503) 555-7555 + max.ryne@example.com + 2732 Baker Blvd. + Eugene + OR + 97403 + USA + + + Annalise Barclay + 862-397-1266 + max@example.com + 4069 Jadewood Farms + Dover + New Jersey + 07801 + USA + + + Annalise Barclay + 862-397-1266 + max@example.com + 4569 Jadewood Farms + Cambridge + Massachusetts + 02138 + USA + + + Tony McCallum + 617-909-1330 + tony@123.com + 4684 Gerald L. Bates Drive + Cambridge + Massachusetts + 02138 + USA + + + diff --git a/import_processor/tests/res_partner_1.csv b/import_processor/tests/res_partner_1.csv new file mode 100644 index 00000000000..706a5858e59 --- /dev/null +++ b/import_processor/tests/res_partner_1.csv @@ -0,0 +1,4 @@ +phone,email,city,country_id,name,street,street2,zip +(870)-931-0505,max.ryne@example.com,Finton,233,Max Ryne,4311 Owagner Lane,Redmond,98052 +(870)-932-0505,michal.merry@123.com,London,,Michal Merry,Capital Exchange Way,Brentford,TW8 0EX +(870)-932-0505,michal.merry@example.com,London,,Michal Merry,Capital Exchange Way,Brentford,TW8 0EX diff --git a/import_processor/tests/res_partner_1.json b/import_processor/tests/res_partner_1.json new file mode 100644 index 00000000000..325bffb29aa --- /dev/null +++ b/import_processor/tests/res_partner_1.json @@ -0,0 +1,24 @@ +{ + "contact": [ + { + "phone": "(870)-931-0505", + "email": "max.ryne@example.com", + "city": "Finton", + "country_id": 223, + "name": "Max Ryne", + "street": "4311 Owagner Lane", + "street2": "Redmond", + "zip": "98052" + }, + { + "phone": "(870)-935-0505", + "email": "miki.rrryne@example.com", + "city": "Fiinton", + "country_id": 223, + "name": "miki disney", + "street": "4311 Owagner Lane", + "street2": "Redmond", + "zip": "98052" + } + ] +} diff --git a/import_processor/tests/test_import_processor.py b/import_processor/tests/test_import_processor.py new file mode 100644 index 00000000000..98082aea922 --- /dev/null +++ b/import_processor/tests/test_import_processor.py @@ -0,0 +1,271 @@ +import base64 +from pathlib import Path + +from odoo.exceptions import UserError +from odoo.tests import Form +from odoo.tests.common import TransactionCase, tagged +from odoo.tools import file_open + +from odoo.addons.import_processor.models.import_processor import chunking + + +@tagged("post_install", "-at_install") +class ImportProcessorTest(TransactionCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.import_processor_csv = cls.env["import.processor"].create( + { + "name": "Import Res Partner(CSV with chunk size)", + "model_id": cls.env.ref("base.model_res_partner").id, + "active": 1, + "file_type": "csv", + "chunk_size": 2, + "preprocessor": """ +regex = r"^[a-z0-9]+[\\._]?[a-z0-9]+[@][a-zA-Z]+[.][a-zA-Z]{2,3}$" +fields_list = ["name", "email", "phone", "street", "street2", "zip", "city"] +""", + "processor": """ +for contact in entry: + email = contact.get("email") + if re.search(regex, email): + data = {key: value for key, value in contact.items() if key in fields_list} + rec = model.search([("email", "=", email)]) + if rec: + rec.update(data) + else: + rec = model.create(data) + + records |= rec +""", + "postprocessor": "log('Processed the following records: %s', records)", + } + ) + cls.import_processor_xml = cls.env["import.processor"].create( + { + "name": "Import Res Partner (XML)", + "model_id": cls.env.ref("base.model_res_partner").id, + "active": 1, + "file_type": "xml", + "xml_path_entry": "//Customer", + "preprocessor": """ +# Regex Email Validation +regex = r"^[a-z0-9]+[\\._]?[a-z0-9]+[@][a-zA-Z]+[.][a-zA-Z]{2,3}$" + +fields_list = ["name", "email", "phone", "street", "street2", "zip", "city"] + +# Converts XML file to Dictionary +def xml_to_dict(xml_element): + result = {} + for child in xml_element: + if len(child) == 0: + result[child.tag] = child.text + else: + result[child.tag] = xml_to_dict(child) + return result +""", + "processor": """ +entry = xml_to_dict(entry) + +email = entry.get("email") +if re.search(regex, email): + rec = model.search([("email", "=", email)]) + data = {key: value for key, value in entry.items() if key in fields_list} + if rec: + rec.update(data) + else: + rec = model.create(data) + + records |= rec +""", + "postprocessor": "log('Processed the following records: %s', records)", + } + ) + cls.import_processor_json = cls.env["import.processor"].create( + { + "name": "Import Res Partner (JSON)", + "model_id": cls.env.ref("base.model_res_partner").id, + "active": 1, + "file_type": "json", + "json_path_entry": "'contact'", + "preprocessor": """ +# Regex Email Validation +regex = r"^[a-z0-9]+[\\._]?[a-z0-9]+[@][a-zA-Z]+[.][a-zA-Z]{2,3}$" +fields_list = ["name", "email", "phone", "street", "street2", "zip", "city"] +""", + "processor": """ +for contact in entry: + email = contact.get("email") + if re.search(regex, email): + rec = model.search([("email", "=", email)]) + data = {key: value for key, value in contact.items() if key in fields_list} + if rec: + rec.update(data) + else: + rec = model.create(data) + + records |= rec +""", + "postprocessor": "log('Processed the following records: %s', records)", + } + ) + + def _get_file_binary_data(self, file_name): + file_path = Path(__file__).parent / file_name + with file_open(file_path, "rb") as file: + file_contents = file.read() + return file_contents + + def test_check_import_processor_csv_is_active(self): + """Test whether a Import Processor record is active or not""" + self.assertTrue(self.import_processor_csv.active) + + def test_misc(self): + """Test functions to check if there are errors raising""" + self.import_processor_xml._get_default_code() + self.import_processor_xml._compute_help_text() + + def test_import_xml(self): + """Test the import of data from a XML file. + + Preprocess: Defined variable named "regex" to validate email addresses. + Process: Create a new record with a valid email (e.g., email.valid@example.com) + and update record if duplicate email found. + Postprocess: Update the "Job Description" field if it is empty. + """ + file = self._get_file_binary_data("contacts.xml") + record = self.import_processor_xml.process(file) + self.assertEqual(len(record), 2) + + def test_import_json(self): + # Test the import of data from a JSON file. + + file = self._get_file_binary_data("contacts.json") + record = self.import_processor_json.process(file) + self.assertEqual(len(record), 2) + + def test_import_csv_chunk(self): + """Test the import of data from a CSV file with chunk size. + + Preprocess: Use the variable "regex" to validate email addresses. + Process: Create a new record with a valid email (e.g., email.valid@example.com) + and update existing records with the same email. + Postprocess: Update the "Job Description" field if it is empty. + + """ + + file = self._get_file_binary_data("contacts.csv") + record = self.import_processor_csv.process(file) + # Verifies the imported record + self.assertEqual(len(record), 2) + + def test_import_zip_one(self): + """Test the import of data from 'zip_one.zip', which contains + a single CSV file with one res.partner record""" + + # Compression method "Zipped File" + self.import_processor_csv.compression = "zip_one" + file = self._get_file_binary_data("zip_one.zip") + record = self.import_processor_csv.process(file) + + # Verifies the imported record + self.assertEqual(len(record), 1) + + def test_import_zip_all(self): + """Test the import of data from 'zip_all.zip', which contains + two CSV files, each of which has one 'res.partner' record.""" + + # Compression method "Multiple Zipped Files" + self.import_processor_csv.compression = "zip_all" + + file = self._get_file_binary_data("zip_all.zip") + record = self.import_processor_csv.process(file) + # Verifies the imported record + self.assertEqual(len(record), 2) + + def test_import_multi_compression_zip_one(self): + """This case verifies the behavior of the zip_one compression method + when multiple files are compressed into a single ZIP file. The expected + outcome is that the compression process raises a Usererror exception.""" + + # Compression method "Zipped File" + self.import_processor_csv.compression = "zip_one" + file = self._get_file_binary_data("zip_all.zip") + with self.assertRaisesRegex(UserError, "Expected only 1 file."): + self.import_processor_csv.process(file) + + def test_wizard_action_import_csv(self): + file_data = self._get_file_binary_data("contacts.csv") + encoded_data = base64.b64encode(file_data) + count_before = self.env["res.partner"].search_count([]) + wizard = self.env["import.processor.wizard"].create( + { + "model": "res.partner", + "processor_id": self.import_processor_csv.id, + "file_upload": encoded_data, + } + ) + wizard.action_import() + count_after = self.env["res.partner"].search_count([]) + self.assertEqual( + count_before + 2, count_after, "Wizard should have created 2 partners" + ) + + def test_wizard_onchange_model(self): + file_data = self._get_file_binary_data("contacts.csv") + encoded_data = base64.b64encode(file_data) + with Form(self.env["import.processor.wizard"]) as wizard_form: + wizard_form.model = "res.partner" + wizard_form.processor_id = self.import_processor_csv + wizard_form.file_upload = encoded_data + self.assertEqual( + wizard_form.model, + self.import_processor_csv.model_name, + "The onchange should have set the model of the processor.", + ) + + def test_chunking_standard(cls): + items = [1, 2, 3, 4, 5, 6] + result = list(chunking(items, 2)) + expected = [[1, 2], [3, 4], [5, 6]] + cls.assertEqual(result, expected) + + def test_chunking_with_remainder(cls): + items = [1, 2, 3, 4, 5, 6, 7] + result = list(chunking(items, 3)) + expected = [[1, 2, 3], [4, 5, 6], [7]] + cls.assertEqual(result, expected) + + def test_chunking_size_zero_or_less(cls): + items = [1, 2, 3] + result = list(chunking(items, 0)) + cls.assertEqual(result, [1, 2, 3]) + + def test_chunking_empty_list(cls): + items = [] + result = list(chunking(items, 5)) + cls.assertEqual(result, []) + + def test_get_file_types(self): + processor = self.env["import.processor"].browse() + file_types = processor._get_file_types() + expected = [("csv", "CSV"), ("json", "JSON"), ("xlsx", "XLSX"), ("xml", "XML")] + self.assertEqual( + file_types, expected, "File types selection does not match definition." + ) + + def test_get_csv_delimiter(self): + processor = self.env["import.processor"].env["import.processor"].browse() + delimiters = processor._get_csv_delimiter() + keys = [d[0] for d in delimiters] + self.assertListEqual( + keys, ["comma", "semicolon", "tab"], "CSV Delimiter keys are incorrect." + ) + + def test_get_compression(self): + processor = self.env["import.processor"].browse() + compression = processor._get_compression() + keys = [c[0] for c in compression] + self.assertListEqual( + keys, ["zip_one", "zip_all"], "Compression selection keys are incorrect." + ) diff --git a/import_processor/tests/zip_all.zip b/import_processor/tests/zip_all.zip new file mode 100644 index 00000000000..eb486988592 Binary files /dev/null and b/import_processor/tests/zip_all.zip differ diff --git a/import_processor/tests/zip_one.zip b/import_processor/tests/zip_one.zip new file mode 100644 index 00000000000..f340432bf56 Binary files /dev/null and b/import_processor/tests/zip_one.zip differ diff --git a/import_processor/views/import_processor_views.xml b/import_processor/views/import_processor_views.xml new file mode 100644 index 00000000000..8ae2b2a0815 --- /dev/null +++ b/import_processor/views/import_processor_views.xml @@ -0,0 +1,126 @@ + + + + import.processor + + + + + + + + + + + + import.processor + +
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    +

    Help with Python expressions

    +

    Various fields may use Python code or Python expressions. The python environments are passed between the pre-processor, code, and post-processor. The following variables are additionally defined:

    + +
    +     +     +     +     +
    +     +     + + + Import Processors + import.processor + {"active_test": False} + list,form + + + +     diff --git a/import_processor/wizards/__init__.py b/import_processor/wizards/__init__.py new file mode 100644 index 00000000000..75f1de2ce73 --- /dev/null +++ b/import_processor/wizards/__init__.py @@ -0,0 +1,4 @@ +# © 2022 initOS GmbH +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +from . import import_processor_wizard diff --git a/import_processor/wizards/import_processor_wizard.py b/import_processor/wizards/import_processor_wizard.py new file mode 100644 index 00000000000..0fc00a77b91 --- /dev/null +++ b/import_processor/wizards/import_processor_wizard.py @@ -0,0 +1,48 @@ +# © 2022 initOS GmbH +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl). + +import base64 + +from odoo import api, fields, models + + +class ImporterProcessorWizard(models.TransientModel): + _name = "import.processor.wizard" + _description = "Import Processor Wizard" + + model = fields.Char() + processor_id = fields.Many2one( + "import.processor", + domain='[("model_name", "=", model)]', + ) + file_upload = fields.Binary() + message = fields.Text() + + @api.onchange("model") + def onchange_model(self): + processor = ( + self.env["import.processor"] + .sudo() + .search([("model_name", "=", self.model)]) + ) + + if len(processor) == 1: + self.processor_id = processor.id + + def action_import(self): + self.ensure_one() + records = self.processor_id.process(base64.decodebytes(self.file_upload)) + return { + "type": "ir.actions.act_window", + "name": self.env._("Import Processor"), + "target": "new", + "view_mode": "form", + "res_model": self._name, + "context": { + "default_model": self.model, + "default_message": self.env._( + "Imported %(count) record(s)", count=len(records) + ), + "default_processor_id": self.processor_id.id, + }, + } diff --git a/import_processor/wizards/import_processor_wizard_views.xml b/import_processor/wizards/import_processor_wizard_views.xml new file mode 100644 index 00000000000..6fc44a95ed5 --- /dev/null +++ b/import_processor/wizards/import_processor_wizard_views.xml @@ -0,0 +1,31 @@ + + + + import.processor.wizard + +
    + + + + + + + + + +
    +     +     +     diff --git a/requirements.txt b/requirements.txt index dfd6b25509f..e60e2aa02ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ # generated from manifests external_dependencies dataclasses +jsonpath_ng numpy odoorpc openupgradelib