[16.0][l10n_br_fiscal][IMP] l10n_br_fiscal: standardize data+demo load

l10n_br_fiscal/__init__.py

@@ -1,6 +1,100 @@
 # License AGPL-3 - See http://www.gnu.org/licenses/agpl-3.0.html
 
-from .hooks import post_init_hook
-
 from . import models
 from . import wizards
+
+import csv
+from io import StringIO
+import logging
+import os
+from odoo import SUPERUSER_ID, api
+from odoo.tools import convert
+from .demo import DEMO_NCM, DEMO_NBM, DEMO_NBS, DEMO_CEST
+
+_logger = logging.getLogger(__name__)
+
+
+# unlike regular data files: we will force noupdate=True for these files:
+NOUPDATE_FILES = [
+    "data/l10n_br_fiscal.cnae.csv",
+    "data/l10n_br_fiscal.cfop.csv",
+    "data/l10n_br_fiscal_cfop_data.xml",
+    "data/l10n_br_fiscal.tax.ipi.control.seal.csv",
+    "data/l10n_br_fiscal.tax.ipi.guideline.csv",
+    "data/l10n_br_fiscal.tax.ipi.guideline.class.csv",
+    "data/l10n_br_fiscal.tax.pis.cofins.base.csv",
+    "data/l10n_br_fiscal.tax.pis.cofins.credit.csv",
+    "data/l10n_br_fiscal.service.type.csv",
+    "data/simplified_tax_data.xml",
+    "data/operation_data.xml",
+    "data/l10n_br_fiscal_tax_icms_data.xml",
+]
+
+
+def convert_csv_import(
+    cr, module, fname, csvcontent, idref=None, mode="init", noupdate=False
+):
+    """
+    This is a monkey patch allowing filtering for faster demo/test loading
+    and allowing to force noupdate=True for some key fiscal data.
+    """
+    filename, _ext = os.path.splitext(os.path.basename(fname))
+    model = filename.split("-")[0]
+    if fname in NOUPDATE_FILES:
+        noupdate = True
+
+    # is is one of the very large fiscal data files?
+    if model in (
+        "l10n_br_fiscal.ncm",
+        "l10n_br_fiscal.nbm",
+        "l10n_br_fiscal.nbs",
+        "l10n_br_fiscal.cest",
+        # NOTE we could also speed up the cnae
+        # if we also create a simplified demo/test simplified_tax.xml
+    ):
+
+        # now we will figure out if we are in demo/test mode
+        # at this early stage of data module loading, the demo flag of the fiscal module
+        # can still be False, that's why we will consider it's the demo/test mode
+        # if the demo flag is set for the l10n_br_base module.
+        env = api.Environment(cr, SUPERUSER_ID, {})
+        modules = env["ir.module.module"].search(
+            [("name", "in", ("l10n_br_base", "l10n_br_fiscal"))]
+        )
+        if any(module.demo for module in modules):
+            # Then we will filter and load only the demo records:
+            id_column_values = globals().get(f"DEMO_{model.split('.')[-1].upper()}")
+
+            input_stream = StringIO(csvcontent.decode())
+            output_stream = StringIO()
+
+            # Read the CSV content
+            csv_reader = csv.reader(input_stream)
+            csv_writer = csv.writer(output_stream)
+
+            # Write the header to the output
+            header = next(csv_reader)
+            csv_writer.writerow(header)
+
+            # Filter and write rows matching the IDs
+            line_count = 0
+            for row in csv_reader:
+                if row[0].replace('"', "") in id_column_values:
+                    csv_writer.writerow(row)
+                    line_count += 1
+
+            _logger.info(
+                f"(faster demo/test mode: loading only {line_count} demo lines)"
+            )
+
+            # Get the filtered content as a string
+            output_stream.seek(0)
+            csvcontent = output_stream.getvalue().encode()
+
+    return convert.convert_csv_import._original_method(
+        cr, module, fname, csvcontent, idref, mode, noupdate
+    )
+
+
+convert_csv_import._original_method = convert.convert_csv_import
+convert.convert_csv_import = convert_csv_import

l10n_br_fiscal/__manifest__.py

@@ -16,11 +16,11 @@
         "l10n_br_base",
     ],
     "data": [
-        # security
+        # Security
         "security/fiscal_security.xml",
         "security/ir.model.access.csv",
+
         # Data
-        # Some data is being loaded via post_init_hook in hook file
         "data/l10n_br_fiscal_email_template.xml",
         "data/l10n_br_fiscal_data.xml",
         "data/uom_data.xml",
@@ -38,6 +38,25 @@
         "data/l10n_br_fiscal_server_action.xml",
         "data/ir_cron.xml",
         "data/l10n_br_fiscal_comment_data.xml",
+
+        # the following data files will be loaded as noupdate=True
+        "data/l10n_br_fiscal.cnae.csv",
+        "data/l10n_br_fiscal.cfop.csv",
+        "data/l10n_br_fiscal_cfop_data.xml",
+        "data/l10n_br_fiscal.tax.ipi.control.seal.csv",
+        "data/l10n_br_fiscal.tax.ipi.guideline.csv",
+        "data/l10n_br_fiscal.tax.ipi.guideline.class.csv",
+        "data/l10n_br_fiscal.tax.pis.cofins.base.csv",
+        "data/l10n_br_fiscal.tax.pis.cofins.credit.csv",
+        "data/l10n_br_fiscal.service.type.csv",
+        "data/simplified_tax_data.xml",
+        "data/operation_data.xml",
+        "data/l10n_br_fiscal_tax_icms_data.xml",
+        "data/l10n_br_fiscal.ncm.csv",  # (partial load if demo/test)
+        "data/l10n_br_fiscal.nbm.csv",  # (partial load if demo/test)
+        "data/l10n_br_fiscal.nbs.csv",  # (partial load if demo/test)
+        "data/l10n_br_fiscal.cest.csv", # (partial load if demo/test)
+
         # Views
         "views/cnae_view.xml",
         "views/cfop_view.xml",
@@ -84,15 +103,26 @@
         "views/invalidate_number_view.xml",
         "views/city_taxation_code.xml",
         "views/operation_dashboard_view.xml",
+
         # Actions
         "views/l10n_br_fiscal_action.xml",
+
         # Menus
         "views/l10n_br_fiscal_menu.xml",
     ],
     "demo": [
-        # Some demo data is being loaded via post_init_hook in hook file
+        "demo/city_taxation_code_demo.xml",
+        "demo/company_demo.xml",
+        "demo/product_demo.xml",
+        "demo/partner_demo.xml",
+        "demo/fiscal_document_nfse_demo.xml",
+        "demo/fiscal_operation_demo.xml",
+        "demo/subsequent_operation_demo.xml",
+        "demo/l10n_br_fiscal_document_email.xml",
+        "demo/res_users_demo.xml",
+        "demo/icms_tax_definition_demo.xml",
+        "demo/fiscal_document_demo.xml",
     ],
-    "post_init_hook": "post_init_hook",
     "installable": True,
     "application": True,
     "auto_install": False,