From a04aa15191263c6f5f74073737ac4ccefba595c6 Mon Sep 17 00:00:00 2001 From: vrenaville Date: Mon, 10 Mar 2025 10:32:52 +0100 Subject: [PATCH] feat: remove unused modules --- attachment_azure/README.rst | 46 -- attachment_azure/__init__.py | 4 - attachment_azure/__manifest__.py | 22 - attachment_azure/models/__init__.py | 4 - attachment_azure/models/ir_attachment.py | 218 --------- attachment_azure/pyproject.toml | 3 - base_attachment_object_storage/README.rst | 46 -- base_attachment_object_storage/__init__.py | 26 - .../__manifest__.py | 17 - .../data/res_config_settings_data.xml | 11 - .../models/__init__.py | 1 - .../models/ir_attachment.py | 448 ------------------ .../models/strtobool.py | 21 - base_attachment_object_storage/pyproject.toml | 3 - cloud_platform/__manifest__.py | 2 +- cloud_platform/models/cloud_platform.py | 63 +-- cloud_platform_azure/README.md | 5 - cloud_platform_azure/__init__.py | 1 - cloud_platform_azure/__manifest__.py | 24 - cloud_platform_azure/models/__init__.py | 1 - cloud_platform_azure/models/cloud_platform.py | 126 ----- cloud_platform_azure/pyproject.toml | 3 - 22 files changed, 4 insertions(+), 1091 deletions(-) delete mode 100644 attachment_azure/README.rst delete mode 100644 attachment_azure/__init__.py delete mode 100644 attachment_azure/__manifest__.py delete mode 100644 attachment_azure/models/__init__.py delete mode 100644 attachment_azure/models/ir_attachment.py delete mode 100644 attachment_azure/pyproject.toml delete mode 100644 base_attachment_object_storage/README.rst delete mode 100644 base_attachment_object_storage/__init__.py delete mode 100644 base_attachment_object_storage/__manifest__.py delete mode 100644 base_attachment_object_storage/data/res_config_settings_data.xml delete mode 100644 base_attachment_object_storage/models/__init__.py delete mode 100644 base_attachment_object_storage/models/ir_attachment.py delete mode 100644 base_attachment_object_storage/models/strtobool.py delete mode 100644 base_attachment_object_storage/pyproject.toml delete mode 100644 cloud_platform_azure/README.md delete mode 100644 cloud_platform_azure/__init__.py delete mode 100644 cloud_platform_azure/__manifest__.py delete mode 100644 cloud_platform_azure/models/__init__.py delete mode 100644 cloud_platform_azure/models/cloud_platform.py delete mode 100644 cloud_platform_azure/pyproject.toml diff --git a/attachment_azure/README.rst b/attachment_azure/README.rst deleted file mode 100644 index b5c26d2..0000000 --- a/attachment_azure/README.rst +++ /dev/null @@ -1,46 +0,0 @@ -=========================================== -Attachments on Microsoft Azure Blob Storage -=========================================== - -This addon allows to store the attachments (documents and assets) on `Microsoft Azure -Blob Storage `_. - -Configuration -------------- - -Activate Azure Blob storage: - -* Create or set the system parameter with the key ``ir_attachment.location`` - and the value in the form ``azure``. - -Configure accesses with environment variables: - -* ``AZURE_STORAGE_CONNECTION_STRING`` or -* ``AZURE_STORAGE_ACCOUNT_NAME`` -* ``AZURE_STORAGE_ACCOUNT_URL`` -* ``AZURE_STORAGE_ACCOUNT_KEY`` - -One container will be created per database using the `RUNNING_ENV` environment variable -and the name of the database. By default, `RUNNING_ENV` is set to `dev`. - -The container name can be overridden with environment variable ``AZURE_STORAGE_NAME``. -The strings ``{db}`` and ``{env}`` can be used inside that variable and the values -will be replaced respectively by the database name and environment name. - -The container name will also be stored in the database for each attachment, -and will be used to access the right container in the storage. - -This addon must be added in the server wide addons with (``--load`` option): - -``--load=web,attachment_azure`` - -The System Parameter ``ir_attachment.storage.force.database`` can be customized to -force storage of files in the database. See the documentation of the module -``base_attachment_object_storage``. - -Limitations ------------ - -* You need to call ``env['ir.attachment'].force_storage()`` after - having changed the ``ir_attachment.location`` configuration in order to - migrate the existing attachments to Azure Blob Storage. diff --git a/attachment_azure/__init__.py b/attachment_azure/__init__.py deleted file mode 100644 index 49d7105..0000000 --- a/attachment_azure/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2016-2019 Camptocamp SA -# Copyright 2021 Open Source Integrators -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -from . import models diff --git a/attachment_azure/__manifest__.py b/attachment_azure/__manifest__.py deleted file mode 100644 index c69834b..0000000 --- a/attachment_azure/__manifest__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2016-2019 Camptocamp SA -# Copyright 2021 Open Source Integrators -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -{ - "name": "Attachments on Azure storage", - "summary": "Store assets and attachments on a Azure compatible object storage", - "version": "17.0.1.0.0", - "author": "Camptocamp, " - "Open Source Integrators, " - "Serpent Consulting Services, " - "Odoo Community Association (OCA)", - "license": "AGPL-3", - "category": "Knowledge Management", - "depends": ["base_attachment_object_storage"], - "external_dependencies": { - "python": ["azure-storage-blob", "azure-identity"], - }, - "website": "https://github.com/camptocamp/odoo-cloud-platform", - "installable": False, - "development_status": "Beta", - "maintainers": ["max3903"], -} diff --git a/attachment_azure/models/__init__.py b/attachment_azure/models/__init__.py deleted file mode 100644 index cb9b196..0000000 --- a/attachment_azure/models/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2016-2019 Camptocamp SA -# Copyright 2021 Open Source Integrators -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -from . import ir_attachment diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py deleted file mode 100644 index ee0741f..0000000 --- a/attachment_azure/models/ir_attachment.py +++ /dev/null @@ -1,218 +0,0 @@ -# Copyright 2016-2019 Camptocamp SA -# Copyright 2021 Open Source Integrators -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -import io -import logging -import os -import re -from datetime import datetime, timedelta - -from odoo import _, api, exceptions, models - -_logger = logging.getLogger(__name__) - -try: - from azure.core.exceptions import HttpResponseError, ResourceExistsError - from azure.storage.blob import ( - AccountSasPermissions, - BlobServiceClient, - ResourceTypes, - generate_account_sas, - ) -except ImportError: - _logger.debug("Cannot 'import azure-storage-blob'.") - -try: - from azure.identity import DefaultAzureCredential -except ImportError: - _logger.debug("Cannot 'import azure-identity'.") - - -class IrAttachment(models.Model): - _inherit = "ir.attachment" - - def _get_stores(self): - return ["azure"] + super()._get_stores() - - @api.model - def _get_blob_service_client(self): - """Connect to Azure and return the blob service client - - The following environment variables must be set: - * ``AZURE_STORAGE_CONNECTION_STRING`` - or - * ``AZURE_STORAGE_ACCOUNT_NAME`` - * ``AZURE_STORAGE_ACCOUNT_URL`` - * ``AZURE_STORAGE_ACCOUNT_KEY`` - or if you want to use AAD (pod identity), set it to 1 or 0 - * ``AZURE_STORAGE_USE_AAD`` - - """ - connect_str = os.environ.get("AZURE_STORAGE_CONNECTION_STRING") - account_name = os.environ.get("AZURE_STORAGE_ACCOUNT_NAME") - account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL") - account_key = os.environ.get("AZURE_STORAGE_ACCOUNT_KEY") - account_use_aad = os.environ.get("AZURE_STORAGE_USE_AAD") - if not ( - connect_str - or (account_name and account_url and account_key) - or account_use_aad - ): - msg = _( - "If you want to read from the Azure container, you must provide the " - "following environment variables:\n" - "* AZURE_STORAGE_CONNECTION_STRING\n" - "or\n" - "* AZURE_STORAGE_ACCOUNT_NAME\n" - "* AZURE_STORAGE_ACCOUNT_URL\n" - "* AZURE_STORAGE_ACCOUNT_KEY\n" - "or\n" - "* AZURE_STORAGE_USE_AAD\n" - ) - raise exceptions.UserError(msg) - blob_service_client = None - if account_use_aad: - token_credential = DefaultAzureCredential() - blob_service_client = BlobServiceClient( - account_url=account_url, credential=token_credential - ) - elif connect_str: - try: - blob_service_client = BlobServiceClient.from_connection_string( - connect_str - ) - except HttpResponseError as error: - _logger.exception( - "Error during the connection to Azure container using the " - "connection string." - ) - raise exceptions.UserError(str(error)) from None - else: - try: - sas_token = generate_account_sas( - account_name=account_name, - account_key=account_key, - resource_types=ResourceTypes(container=True, object=True), - permission=AccountSasPermissions(read=True, write=True), - expiry=datetime.utcnow() + timedelta(hours=1), - ) - blob_service_client = BlobServiceClient( - account_url=account_url, - credential=sas_token, - ) - except HttpResponseError as error: - _logger.exception( - "Error during the connection to Azure container using the Shared " - "Access Signature (SAS)" - ) - raise exceptions.UserError(str(error)) from None - return blob_service_client - - @api.model - def _get_container_name(self): - # Container naming rules: - # https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names # noqa: E501 - running_env = os.environ.get("RUNNING_ENV", "dev") - storage_name = os.environ.get("AZURE_STORAGE_NAME", r"{env}-{db}") - storage_name = storage_name.format(env=running_env, db=self.env.cr.dbname) - # replace invalid characters by _ - storage_name = re.sub(r"[\W_]+", "-", storage_name) - # lowercase, max 63 chars - return str.lower(storage_name)[:63] - - @api.model - def _get_azure_container(self, container_name=None): - if not container_name: - container_name = self._get_container_name() - try: - blob_service_client = self._get_blob_service_client() - except exceptions.UserError: - _logger.exception( - "error accessing to storage '%s' please check credentials ", - container_name, - ) - return False - container_client = blob_service_client.get_container_client(container_name) - if not container_client.exists(): - try: - # Create the container - container_client.create_container() - except HttpResponseError as error: - _logger.exception("Error during the creation of the Azure container") - raise exceptions.UserError(str(error)) from None - return container_client - - @api.model - def _store_file_read(self, fname, bin_size=False): - if fname.startswith("azure://"): - key = fname.replace("azure://", "", 1).lower() - if "/" in key: - container_name, key = key.split("/", 1) - else: - container_name = None - container_client = self._get_azure_container(container_name) - # if container cannot be retrived, abort reading from azure storage - if not container_client: - return "" - try: - blob_client = container_client.get_blob_client(key) - read = blob_client.download_blob().readall() - except HttpResponseError: - read = "" - _logger.info("Attachment '%s' missing on object storage", fname) - return read - else: - return super()._store_file_read(fname, bin_size) - - @api.model - def _store_file_write(self, key, bin_data): - location = self.env.context.get("storage_location") or self._storage() - if location == "azure": - container_client = self._get_azure_container() - filename = f"azure://{container_client.container_name}/{key}" - with io.BytesIO() as file: - blob_client = container_client.get_blob_client(key.lower()) - file.write(bin_data) - file.seek(0) - try: - blob_client.upload_blob(file, blob_type="BlockBlob") - except ResourceExistsError: - _logger.exception( - "Trying to re create an existing resource %s" % filename - ) - except HttpResponseError as error: - # log verbose error from azure, return short message for user - _logger.exception( - "HTTP Error during storage of the file %s" % filename - ) - raise exceptions.UserError( - _("The file could not be stored: %s") % str(error) - ) from None - else: - _super = super() - filename = _super._store_file_write(key, bin_data) - return filename - - @api.model - def _store_file_delete(self, fname): - if fname.startswith("azure://"): - key = fname.replace("azure://", "", 1).lower() - if "/" in key: - container_name, key = key.split("/", 1) - else: - container_name = None - container_client = self._get_azure_container(container_name) - if not container_client: - return "" - # delete the file only if it is on the current configured container - # otherwise, we might delete files used on a different environment - try: - blob_client = container_client.get_blob_client(key) - blob_client.delete_blob() - _logger.info("File %s deleted on the object storage" % (fname)) - except HttpResponseError: - # log verbose error from azure, return short message for - # user - _logger.exception("Error during deletion of the file %s" % fname) - else: - super()._store_file_delete(fname) diff --git a/attachment_azure/pyproject.toml b/attachment_azure/pyproject.toml deleted file mode 100644 index 4231d0c..0000000 --- a/attachment_azure/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[build-system] -requires = ["whool"] -build-backend = "whool.buildapi" diff --git a/base_attachment_object_storage/README.rst b/base_attachment_object_storage/README.rst deleted file mode 100644 index 0ff25c9..0000000 --- a/base_attachment_object_storage/README.rst +++ /dev/null @@ -1,46 +0,0 @@ -Base class for attachments on external object store -=================================================== - -This is a base addon that regroup common code used by addons targeting specific object store - -Configuration -------------- - -Object storage may be slow, and for this reason, we want to store -some files in the database whatever. - -Small images (128, 256) are used in Odoo in list / kanban views. We -want them to be fast to read. -They are generally < 50KB (default configuration) so they don't take -that much space in database, but they'll be read much faster than from -the object storage. - -The assets (application/javascript, text/css) are stored in database -as well whatever their size is: - -* a database doesn't have thousands of them -* of course better for performance -* better portability of a database: when replicating a production - instance for dev, the assets are included - -This storage configuration can be modified in the system parameter -``ir_attachment.storage.force.database``, as a JSON value, for instance:: - - {"image/": 51200, "application/javascript": 0, "text/css": 0} - -Where the key is the beginning of the mimetype to configure and the -value is the limit in size below which attachments are kept in DB. -0 means no limit. - -Default configuration means: - -* images mimetypes (image/png, image/jpeg, ...) below 50KB are - stored in database -* application/javascript are stored in database whatever their size -* text/css are stored in database whatever their size - -Disable attachment storage I/O ------------------------------- - -Define a environment variable `DISABLE_ATTACHMENT_STORAGE` set to `1` -This will prevent any kind of exceptions and read/write on storage attachments. diff --git a/base_attachment_object_storage/__init__.py b/base_attachment_object_storage/__init__.py deleted file mode 100644 index 6a5f25a..0000000 --- a/base_attachment_object_storage/__init__.py +++ /dev/null @@ -1,26 +0,0 @@ -from . import models -from odoo.http import Stream - - -old_from_attachment = Stream.from_attachment - - -@classmethod -def from_attachment(cls, attachment): - if attachment.store_fname and attachment._is_file_from_a_store( - attachment.store_fname - ): - self = cls( - mimetype=attachment.mimetype, - download_name=attachment.name, - conditional=True, - etag=attachment.checksum, - ) - self.type = "data" - self.data = attachment.raw - self.size = len(self.data) - return self - return old_from_attachment(attachment) - - -Stream.from_attachment = from_attachment diff --git a/base_attachment_object_storage/__manifest__.py b/base_attachment_object_storage/__manifest__.py deleted file mode 100644 index e8ee76c..0000000 --- a/base_attachment_object_storage/__manifest__.py +++ /dev/null @@ -1,17 +0,0 @@ -# Copyright 2017-2021 Camptocamp SA -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - - -{ - "name": "Base Attachment Object Store", - "summary": "Base module for the implementation of external object store.", - "version": "17.0.1.0.0", - "author": "Camptocamp,Odoo Community Association (OCA)", - "license": "AGPL-3", - "category": "Knowledge Management", - "depends": ["base"], - "website": "https://github.com/camptocamp/odoo-cloud-platform", - "data": ["data/res_config_settings_data.xml"], - "installable": False, - "auto_install": True, -} diff --git a/base_attachment_object_storage/data/res_config_settings_data.xml b/base_attachment_object_storage/data/res_config_settings_data.xml deleted file mode 100644 index 4a1b8d4..0000000 --- a/base_attachment_object_storage/data/res_config_settings_data.xml +++ /dev/null @@ -1,11 +0,0 @@ - - - - - ir_attachment.storage.force.database - {"image/": 51200, "application/javascript": 0, "text/css": 0} - - - diff --git a/base_attachment_object_storage/models/__init__.py b/base_attachment_object_storage/models/__init__.py deleted file mode 100644 index aaf38a1..0000000 --- a/base_attachment_object_storage/models/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import ir_attachment diff --git a/base_attachment_object_storage/models/ir_attachment.py b/base_attachment_object_storage/models/ir_attachment.py deleted file mode 100644 index 49c31f6..0000000 --- a/base_attachment_object_storage/models/ir_attachment.py +++ /dev/null @@ -1,448 +0,0 @@ -# Copyright 2017-2019 Camptocamp SA -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - -import inspect -import logging -import os -import time -from contextlib import closing, contextmanager - -import psycopg2 - -import odoo -from odoo import _, api, exceptions, models -from odoo.osv.expression import AND, OR, normalize_domain -from odoo.tools.safe_eval import const_eval - -from .strtobool import strtobool - -_logger = logging.getLogger(__name__) - - -def is_true(strval): - return bool(strtobool(strval or "0")) - - -def clean_fs(files): - _logger.info("cleaning old files from filestore") - for full_path in files: - if os.path.exists(full_path): - try: - os.unlink(full_path) - except OSError: - # Harmless and needed for race conditions - _logger.info( - "_file_delete could not unlink %s", full_path, exc_info=True - ) - - -class IrAttachment(models.Model): - _inherit = "ir.attachment" - - @staticmethod - def is_storage_disabled(storage=None, log=True): - msg = _("Storages are disabled (see environment configuration).") - if storage: - msg = _("Storage '%s' is disabled (see environment configuration).") % ( - storage, - ) - is_disabled = is_true(os.environ.get("DISABLE_ATTACHMENT_STORAGE")) - if is_disabled and log: - _logger.warning(msg) - return is_disabled - - def _register_hook(self): - super()._register_hook() - location = self.env.context.get("storage_location") or self._storage() - # ignore if we are not using an object storage - if location not in self._get_stores(): - return - curframe = inspect.currentframe() - calframe = inspect.getouterframes(curframe, 2) - # the caller of _register_hook is 'load_modules' in - # odoo/modules/loading.py - load_modules_frame = calframe[1][0] - # 'update_module' is an argument that 'load_modules' receives with a - # True-ish value meaning that an install or upgrade of addon has been - # done during the initialization. We need to move the attachments that - # could have been created or updated in other addons before this addon - # was loaded - update_module = load_modules_frame.f_locals.get("update_module") - - # We need to call the migration on the loading of the model because - # when we are upgrading addons, some of them might add attachments. - # To be sure they are migrated to the storage we need to call the - # migration here. - # Typical example is images of ir.ui.menu which are updated in - # ir.attachment at every upgrade of the addons - if update_module: - self.env["ir.attachment"].sudo()._force_storage_to_object_storage() - - @property - def _object_storage_default_force_db_config(self): - return {"image/": 51200, "application/javascript": 0, "text/css": 0} - - def _get_storage_force_db_config(self): - param = ( - self.env["ir.config_parameter"] - .sudo() - .get_param( - "ir_attachment.storage.force.database", - ) - ) - storage_config = None - if param: - try: - storage_config = const_eval(param) - except (SyntaxError, TypeError, ValueError): - _logger.exception( - "Could not parse system parameter" - " 'ir_attachment.storage.force.database', reverting to the" - " default configuration." - ) - - if not storage_config: - storage_config = self._object_storage_default_force_db_config - return storage_config - - def _store_in_db_instead_of_object_storage_domain(self): - """Return a domain for attachments that must be forced to DB - - Read the docstring of ``_store_in_db_instead_of_object_storage`` for - more details. - - Used in ``force_storage_to_db_for_special_fields`` to find records - to move from the object storage to the database. - - The domain must be inline with the conditions in - ``_store_in_db_instead_of_object_storage``. - """ - domain = [] - storage_config = self._get_storage_force_db_config() - for mimetype_key, limit in storage_config.items(): - part = [("mimetype", "=like", f"{mimetype_key}%")] - if limit: - part = AND([part, [("file_size", "<=", limit)]]) - domain = OR([domain, part]) - return domain - - def _store_in_db_instead_of_object_storage(self, data, mimetype): - """Return whether an attachment must be stored in db - - When we are using an Object Storage. This is sometimes required - because the object storage is slower than the database/filesystem. - - Small images (128, 256) are used in Odoo in list / kanban views. We - want them to be fast to read. - They are generally < 50KB (default configuration) so they don't take - that much space in database, but they'll be read much faster than from - the object storage. - - The assets (application/javascript, text/css) are stored in database - as well whatever their size is: - - * a database doesn't have thousands of them - * of course better for performance - * better portability of a database: when replicating a production - instance for dev, the assets are included - - The configuration can be modified in the ir.config_parameter - ``ir_attachment.storage.force.database``, as a dictionary, for - instance:: - - {"image/": 51200, "application/javascript": 0, "text/css": 0} - - Where the key is the beginning of the mimetype to configure and the - value is the limit in size below which attachments are kept in DB. - 0 means no limit. - - Default configuration means: - - * images mimetypes (image/png, image/jpeg, ...) below 51200 bytes are - stored in database - * application/javascript are stored in database whatever their size - * text/css are stored in database whatever their size - - The conditions must be inline with the domain in - ``_store_in_db_instead_of_object_storage_domain``. - - """ - if self.is_storage_disabled(): - return True - storage_config = self._get_storage_force_db_config() - for mimetype_key, limit in storage_config.items(): - if mimetype.startswith(mimetype_key): - if not limit: - return True - bin_data = data - return len(bin_data) <= limit - return False - - def _get_datas_related_values(self, data, mimetype): - storage = self.env.context.get("storage_location") or self._storage() - if data and storage in self._get_stores(): - if self._store_in_db_instead_of_object_storage(data, mimetype): - # compute the fields that depend on datas - bin_data = data - values = { - "file_size": len(bin_data), - "checksum": self._compute_checksum(bin_data), - "index_content": self._index(bin_data, mimetype), - "store_fname": False, - "db_datas": data, - } - return values - return super()._get_datas_related_values(data, mimetype) - - @api.model - def _file_read(self, fname): - if self._is_file_from_a_store(fname): - return self._store_file_read(fname) - else: - return super()._file_read(fname) - - def _store_file_read(self, fname): - storage = fname.partition("://")[0] - raise NotImplementedError(f"No implementation for {storage}") - - def _store_file_write(self, key, bin_data): - storage = self.storage() - raise NotImplementedError(f"No implementation for {storage}") - - def _store_file_delete(self, fname): - storage = fname.partition("://")[0] - raise NotImplementedError(f"No implementation for {storage}") - - @api.model - def _file_write(self, bin_data, checksum): - location = self.env.context.get("storage_location") or self._storage() - if location in self._get_stores(): - key = self.env.context.get("force_storage_key") - if not key: - key = self._compute_checksum(bin_data) - filename = self._store_file_write(key, bin_data) - else: - filename = super()._file_write(bin_data, checksum) - return filename - - @api.model - def _file_delete(self, fname): - if self._is_file_from_a_store(fname): - cr = self.env.cr - # using SQL to include files hidden through unlink or due to record - # rules - cr.execute( - "SELECT COUNT(*) FROM ir_attachment " "WHERE store_fname = %s", (fname,) - ) - count = cr.fetchone()[0] - if not count: - self._store_file_delete(fname) - else: - return super()._file_delete(fname) - - @api.model - def _is_file_from_a_store(self, fname): - for store_name in self._get_stores(): - if self.is_storage_disabled(store_name): - continue - uri = f"{store_name}://" - if fname.startswith(uri): - return True - return False - - @contextmanager - def do_in_new_env(self, new_cr=False): - """Context manager that yields a new environment - - Using a new Odoo Environment thus a new PG transaction. - """ - if new_cr: - registry = odoo.modules.registry.Registry.new(self.env.cr.dbname) - with closing(registry.cursor()) as cr: - try: - yield self.env(cr=cr) - except Exception: - cr.rollback() - raise - else: - # disable pylint error because this is a valid commit, - # we are in a new env - cr.commit() # pylint: disable=invalid-commit - else: - # make a copy - yield self.env() - - def _move_attachment_to_store(self): - self.ensure_one() - _logger.info("inspecting attachment %s (%d)", self.name, self.id) - fname = self.store_fname - storage = fname.partition("://")[0] - if self.is_storage_disabled(storage): - fname = False - if fname: - # migrating from filesystem filestore - # or from the old 'store_fname' without the bucket name - _logger.info("moving %s on the object storage", fname) - self.write( - { - "datas": self.datas, - # this is required otherwise the - # mimetype gets overriden with - # 'application/octet-stream' - # on assets - "mimetype": self.mimetype, - } - ) - _logger.info("moved %s on the object storage", fname) - return self._full_path(fname) - elif self.db_datas: - _logger.info("moving on the object storage from database") - self.write({"datas": self.datas}) - - @api.model - def force_storage(self): - if not self.env["res.users"].browse(self.env.uid)._is_admin(): - raise exceptions.AccessError( - _("Only administrators can execute this action.") - ) - location = self.env.context.get("storage_location") or self._storage() - if location not in self._get_stores(): - return super().force_storage() - self._force_storage_to_object_storage() - - @api.model - def force_storage_to_db_for_special_fields(self, new_cr=False): - """Migrate special attachments from Object Storage back to database - - The access to a file stored on the objects storage is slower - than a local disk or database access. For attachments like - image_small that are accessed in batch for kanban views, this - is too slow. We store this type of attachment in the database. - - This method can be used when migrating a filestore where all the files, - including the special files (assets, image_small, ...) have been pushed - to the Object Storage and we want to write them back in the database. - - It is not called anywhere, but can be called by RPC or scripts. - """ - storage = self._storage() - if self.is_storage_disabled(storage): - return - if storage not in self._get_stores(): - return - - domain = AND( - ( - normalize_domain( - [ - ("store_fname", "=like", f"{storage}://%"), - # for res_field, see comment in - # _force_storage_to_object_storage - "|", - ("res_field", "=", False), - ("res_field", "!=", False), - ] - ), - normalize_domain(self._store_in_db_instead_of_object_storage_domain()), - ) - ) - - with self.do_in_new_env(new_cr=new_cr) as new_env: - model_env = new_env["ir.attachment"].with_context(prefetch_fields=False) - attachment_ids = model_env.search(domain).ids - if not attachment_ids: - return - total = len(attachment_ids) - start_time = time.time() - _logger.info( - "Moving %d attachments from %s to" " DB for fast access", total, storage - ) - current = 0 - for attachment_id in attachment_ids: - current += 1 - # if we browse attachments outside of the loop, the first - # access to 'datas' will compute all the 'datas' fields at - # once, which means reading hundreds or thousands of files at - # once, exhausting memory - attachment = model_env.browse(attachment_id) - # this write will read the datas from the Object Storage and - # write them back in the DB (the logic for location to write is - # in the 'datas' inverse computed field) - attachment.write({"datas": attachment.datas}) - # as the file will potentially be dropped on the bucket, - # we should commit the changes here - new_env.cr.commit() - if current % 100 == 0 or total - current == 0: - _logger.info( - "attachment %s/%s after %.2fs", - current, - total, - time.time() - start_time, - ) - - @api.model - def _force_storage_to_object_storage(self, new_cr=False): - _logger.info("migrating files to the object storage") - storage = self.env.context.get("storage_location") or self._storage() - if self.is_storage_disabled(storage): - return - # The weird "res_field = False OR res_field != False" domain - # is required! It's because of an override of _search in ir.attachment - # which adds ('res_field', '=', False) when the domain does not - # contain 'res_field'. - # https://github.com/odoo/odoo/blob/17.0/odoo/addons/base/models/ir_attachment.py#L523 - - domain = [ - "!", - ("store_fname", "=like", f"{storage}://%"), - "|", - ("res_field", "=", False), - ("res_field", "!=", False), - ] - # We do a copy of the environment so we can workaround the cache issue - # below. We do not create a new cursor by default because it causes - # serialization issues due to concurrent updates on attachments during - # the installation - with self.do_in_new_env(new_cr=new_cr) as new_env: - model_env = new_env["ir.attachment"] - ids = model_env.search(domain).ids - files_to_clean = [] - for attachment_id in ids: - try: - with new_env.cr.savepoint(): - # check that no other transaction has - # locked the row, don't send a file to storage - # in that case - self.env.cr.execute( - "SELECT id " - "FROM ir_attachment " - "WHERE id = %s " - "FOR UPDATE NOWAIT", - (attachment_id,), - log_exceptions=False, - ) - - # This is a trick to avoid having the 'datas' - # function fields computed for every attachment on - # each iteration of the loop. The former issue - # being that it reads the content of the file of - # ALL the attachments on each loop. - new_env.clear() - attachment = model_env.browse(attachment_id) - path = attachment._move_attachment_to_store() - if path: - files_to_clean.append(path) - except psycopg2.OperationalError: - _logger.error( - "Could not migrate attachment %s to S3", attachment_id - ) - - # delete the files from the filesystem once we know the changes - # have been committed in ir.attachment - if files_to_clean: - new_env.cr.commit() - clean_fs(files_to_clean) - - def _get_stores(self): - """To get the list of stores activated in the system""" - return [] diff --git a/base_attachment_object_storage/models/strtobool.py b/base_attachment_object_storage/models/strtobool.py deleted file mode 100644 index 1a7ad55..0000000 --- a/base_attachment_object_storage/models/strtobool.py +++ /dev/null @@ -1,21 +0,0 @@ -_MAP = { - "y": True, - "yes": True, - "t": True, - "true": True, - "on": True, - "1": True, - "n": False, - "no": False, - "f": False, - "false": False, - "off": False, - "0": False, -} - - -def strtobool(value): - try: - return _MAP[str(value).lower()] - except KeyError as error: - raise ValueError(f'"{value}" is not a valid bool value') from error diff --git a/base_attachment_object_storage/pyproject.toml b/base_attachment_object_storage/pyproject.toml deleted file mode 100644 index 4231d0c..0000000 --- a/base_attachment_object_storage/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[build-system] -requires = ["whool"] -build-backend = "whool.buildapi" diff --git a/cloud_platform/__manifest__.py b/cloud_platform/__manifest__.py index 89352d2..1473887 100644 --- a/cloud_platform/__manifest__.py +++ b/cloud_platform/__manifest__.py @@ -17,5 +17,5 @@ ], "website": "https://github.com/camptocamp/odoo-cloud-platform", "data": [], - "installable": False, + "installable": True, } diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py index 5df8dd7..470032e 100644 --- a/cloud_platform/models/cloud_platform.py +++ b/cloud_platform/models/cloud_platform.py @@ -1,4 +1,4 @@ -# Copyright 2016-2019 Camptocamp SA +# Copyright 2016-2025 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) import logging @@ -18,39 +18,10 @@ def is_true(strval): return bool(strtobool(strval or "0")) -PlatformConfig = namedtuple("PlatformConfig", "filestore") - - -FilestoreKind = namedtuple("FilestoreKind", ["name", "location"]) - - class CloudPlatform(models.AbstractModel): _name = "cloud.platform" _description = "cloud.platform" - @api.model - def _default_config(self): - return PlatformConfig(self._filestore_kinds()["db"]) - - @api.model - def _filestore_kinds(self): - return { - "db": FilestoreKind("db", "local"), - "file": FilestoreKind("file", "local"), - } - - @api.model - def _platform_kinds(self): - return [] - - @api.model - def _config_by_server_env(self, platform_kind, environment): - configs_getter = getattr( - self, "_config_by_server_env_for_%s" % platform_kind, None - ) - configs = configs_getter() if configs_getter else {} - return configs.get(environment) or self._default_config() - def _get_running_env(self): environment_name = config["running_env"] if environment_name.startswith("labs"): @@ -60,25 +31,9 @@ class CloudPlatform(models.AbstractModel): return environment_name @api.model - def _install(self, platform_kind): - assert platform_kind in self._platform_kinds() - params = self.env["ir.config_parameter"].sudo() - params.set_param("cloud.platform.kind", platform_kind) - environment_name = self._get_running_env() - configs = self._config_by_server_env(platform_kind, environment_name) - params.set_param("ir_attachment.location", configs.filestore.name) + def _install(self, environment): self.check() - if configs.filestore.location == "remote": - self.env["ir.attachment"].sudo().force_storage() - _logger.info(f"cloud platform configured for {platform_kind}") - - @api.model - def install(self): - raise NotImplementedError - - @api.model - def _check_filestore(self, environment_name): - raise NotImplementedError + _logger.info(f"cloud platform configured for {environment}") @api.model def _check_redis(self, environment_name): @@ -110,19 +65,7 @@ class CloudPlatform(models.AbstractModel): @api.model def check(self): - if is_true(os.environ.get("ODOO_CLOUD_PLATFORM_UNSAFE")): - _logger.warning("cloud platform checks disabled, this is not safe") - return - params = self.env["ir.config_parameter"].sudo() - kind = params.get_param("cloud.platform.kind") - if not kind: - _logger.warning( - "cloud platform not configured, you should " - "probably run 'env['cloud.platform'].install()'" - ) - return environment_name = self._get_running_env() - self._check_filestore(environment_name) self._check_redis(environment_name) def _register_hook(self): diff --git a/cloud_platform_azure/README.md b/cloud_platform_azure/README.md deleted file mode 100644 index 449ab29..0000000 --- a/cloud_platform_azure/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# Cloud Platform Azure - -Install addons specific to the Azure setup. - - * The object storage is Azure blob storage diff --git a/cloud_platform_azure/__init__.py b/cloud_platform_azure/__init__.py deleted file mode 100644 index 0650744..0000000 --- a/cloud_platform_azure/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import models diff --git a/cloud_platform_azure/__manifest__.py b/cloud_platform_azure/__manifest__.py deleted file mode 100644 index 4c9b2ba..0000000 --- a/cloud_platform_azure/__manifest__.py +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright 2017-2021 Camptocamp SA -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - - -{ - "name": "Cloud Platform Azure", - "summary": "Addons required for the Camptocamp Cloud Platform on Azure", - "version": "17.0.1.0.0", - "author": "Camptocamp,Odoo Community Association (OCA)", - "license": "AGPL-3", - "category": "Extra Tools", - "depends": [ - "cloud_platform", - "attachment_azure", - "monitoring_prometheus", - ], - "excludes": [ - "cloud_platform_ovh", - "cloud_platform_exoscale", - ], - "website": "https://github.com/camptocamp/odoo-cloud-platform", - "data": [], - "installable": False, -} diff --git a/cloud_platform_azure/models/__init__.py b/cloud_platform_azure/models/__init__.py deleted file mode 100644 index 5d08f36..0000000 --- a/cloud_platform_azure/models/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import cloud_platform diff --git a/cloud_platform_azure/models/cloud_platform.py b/cloud_platform_azure/models/cloud_platform.py deleted file mode 100644 index 8d8725c..0000000 --- a/cloud_platform_azure/models/cloud_platform.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright 2016-2021 Camptocamp SA -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - -import os -import re - -from odoo import api, models - -from odoo.addons.cloud_platform.models.cloud_platform import ( - FilestoreKind, - PlatformConfig, -) - -AZURE_STORE_KIND = FilestoreKind("azure", "remote") - - -class CloudPlatform(models.AbstractModel): - _inherit = "cloud.platform" - - @api.model - def _filestore_kinds(self): - kinds = super()._filestore_kinds() - kinds["azure"] = AZURE_STORE_KIND - return kinds - - @api.model - def _platform_kinds(self): - kinds = super()._platform_kinds() - kinds.append("azure") - return kinds - - @api.model - def _config_by_server_env_for_azure(self): - fs_kinds = self._filestore_kinds() - configs = { - "prod": PlatformConfig(filestore=fs_kinds["azure"]), - "integration": PlatformConfig(filestore=fs_kinds["azure"]), - "labs": PlatformConfig(filestore=fs_kinds["azure"]), - "test": PlatformConfig(filestore=fs_kinds["db"]), - "dev": PlatformConfig(filestore=fs_kinds["db"]), - } - return configs - - @api.model - def _check_filestore(self, environment_name): - params = self.env["ir.config_parameter"].sudo() - use_azure = params.get_param("ir_attachment.location") == AZURE_STORE_KIND.name - if environment_name in ("prod", "integration"): - # Labs instances use azure by default, but we don't want - # to enforce it in case we want to test something with a different - # storage. At your own risks! - assert use_azure, ( - "azure must be used on production and integration instances. " - "It is activated by setting 'ir_attachment.location.' to 'azure'." - " The 'install()' function sets this option " - "automatically." - ) - if use_azure: - key_sets = [ - ["AZURE_STORAGE_USE_AAD", "AZURE_STORAGE_ACCOUNT_URL"], - ["AZURE_STORAGE_CONNECTION_STRING"], - [ - "AZURE_STORAGE_ACCOUNT_NAME", - "AZURE_STORAGE_ACCOUNT_URL", - "AZURE_STORAGE_ACCOUNT_KEY", - ], - ] - is_valid = False - for key_set in key_sets: - if all([os.environ.get(key) for key in key_set]): - is_valid = True - break - assert is_valid, ( - "When ir_attachment.location is set to 'azure', " - "at least one of the following enviromnent variable set " - "is required : {}".format( - " or ".join( - [" + ".join([key for key in key_set]) for key_set in key_sets] - ) - ) - ) - storage_name = os.environ.get("AZURE_STORAGE_NAME", "") - if environment_name in ("prod", "integration", "labs"): - assert storage_name, ( - "AZURE_STORAGE_NAME environment variable is required when " - "ir_attachment.location is 'azure'.\n" - "Normally, 'azure' is activated on labs, integration " - "and production, but should not be used in dev environment" - " (or using a dedicated dev bucket, never using the " - "integration/prod bucket).\n" - "If you don't actually need a bucket, change the" - " 'ir_attachment.location' parameter." - ) - # A bucket name is defined under the following format - # ^[a-z]+\-[a-z]+\-\d+$ - # Anything other than prod bucket must be suffixed with env name - # - # Use AZURE_STORAGE_NAME_UNSTRUCTURED to by-pass check - # on bucket name structure - if os.environ.get("AZURE_STORAGE_NAME_UNSTRUCTURED"): - return - prod_bucket = bool(re.match(r"^[a-z]+\-[a-z]+\-\d+$", storage_name)) - if environment_name == "prod": - assert prod_bucket, ( - "AZURE_STORAGE_NAME should match '^[a-z]+\\-[a-z]+\\-\\d+$', " - f"we got: '{storage_name}'" - ) - else: - # if we are using the prod bucket on another instance - # such as an integration, we must be sure to be in read only! - assert not prod_bucket, ( - "AZURE_STORAGE_NAME should not match '^[a-z]+\\-[a-z]+\\-\\d+$', " - f"we got: '{storage_name}'" - ) - - elif environment_name == "test": - # store in DB so we don't have files local to the host - assert params.get_param("ir_attachment.location") == "db", ( - "In test instances, files must be stored in the database with " - "'ir_attachment.location' set to 'db'. This is " - "automatically set by the function 'install()'." - ) - - @api.model - def install(self): - self._install("azure") diff --git a/cloud_platform_azure/pyproject.toml b/cloud_platform_azure/pyproject.toml deleted file mode 100644 index 4231d0c..0000000 --- a/cloud_platform_azure/pyproject.toml +++ /dev/null @@ -1,3 +0,0 @@ -[build-system] -requires = ["whool"] -build-backend = "whool.buildapi"