From 937e87220a434860d186845fb4e8f67c34ad5a85 Mon Sep 17 00:00:00 2001 From: vrenaville Date: Thu, 12 May 2022 11:04:33 +0200 Subject: [PATCH] add attachment azure --- attachment_azure/README.rst | 46 +++++ attachment_azure/__init__.py | 4 + attachment_azure/__openerp__.py | 22 +++ attachment_azure/models/__init__.py | 4 + attachment_azure/models/ir_attachment.py | 221 +++++++++++++++++++++++ 5 files changed, 297 insertions(+) create mode 100644 attachment_azure/README.rst create mode 100644 attachment_azure/__init__.py create mode 100644 attachment_azure/__openerp__.py create mode 100644 attachment_azure/models/__init__.py create mode 100644 attachment_azure/models/ir_attachment.py diff --git a/attachment_azure/README.rst b/attachment_azure/README.rst new file mode 100644 index 0000000..b5c26d2 --- /dev/null +++ b/attachment_azure/README.rst @@ -0,0 +1,46 @@ +=========================================== +Attachments on Microsoft Azure Blob Storage +=========================================== + +This addon allows to store the attachments (documents and assets) on `Microsoft Azure +Blob Storage `_. + +Configuration +------------- + +Activate Azure Blob storage: + +* Create or set the system parameter with the key ``ir_attachment.location`` + and the value in the form ``azure``. + +Configure accesses with environment variables: + +* ``AZURE_STORAGE_CONNECTION_STRING`` or +* ``AZURE_STORAGE_ACCOUNT_NAME`` +* ``AZURE_STORAGE_ACCOUNT_URL`` +* ``AZURE_STORAGE_ACCOUNT_KEY`` + +One container will be created per database using the `RUNNING_ENV` environment variable +and the name of the database. By default, `RUNNING_ENV` is set to `dev`. + +The container name can be overridden with environment variable ``AZURE_STORAGE_NAME``. +The strings ``{db}`` and ``{env}`` can be used inside that variable and the values +will be replaced respectively by the database name and environment name. + +The container name will also be stored in the database for each attachment, +and will be used to access the right container in the storage. + +This addon must be added in the server wide addons with (``--load`` option): + +``--load=web,attachment_azure`` + +The System Parameter ``ir_attachment.storage.force.database`` can be customized to +force storage of files in the database. See the documentation of the module +``base_attachment_object_storage``. + +Limitations +----------- + +* You need to call ``env['ir.attachment'].force_storage()`` after + having changed the ``ir_attachment.location`` configuration in order to + migrate the existing attachments to Azure Blob Storage. diff --git a/attachment_azure/__init__.py b/attachment_azure/__init__.py new file mode 100644 index 0000000..49d7105 --- /dev/null +++ b/attachment_azure/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +from . import models diff --git a/attachment_azure/__openerp__.py b/attachment_azure/__openerp__.py new file mode 100644 index 0000000..3570bdc --- /dev/null +++ b/attachment_azure/__openerp__.py @@ -0,0 +1,22 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +{ + "name": "Attachments on Azure storage", + "summary": "Store assets and attachments on a Azure compatible object storage", + "version": "15.0.1.0.0", + "author": "Camptocamp, " + "Open Source Integrators, " + "Serpent Consulting Services, " + "Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Knowledge Management", + "depends": ["base_attachment_object_storage"], + "external_dependencies": { + "python": ["azure-storage-blob", "azure-identity"], + }, + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "installable": True, + "development_status": "Beta", + "maintainers": ["max3903"], +} diff --git a/attachment_azure/models/__init__.py b/attachment_azure/models/__init__.py new file mode 100644 index 0000000..cb9b196 --- /dev/null +++ b/attachment_azure/models/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +from . import ir_attachment diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py new file mode 100644 index 0000000..15fc101 --- /dev/null +++ b/attachment_azure/models/ir_attachment.py @@ -0,0 +1,221 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +import io +import logging +import os +import re +from datetime import datetime, timedelta + +from odoo import _, api, exceptions, models +from openerp.tools.translate import _ +from openerp.osv import osv +from openerp.osv.orm import except_orm + +_logger = logging.getLogger(__name__) + +try: + from azure.storage.blob import ( + BlobServiceClient, + generate_account_sas, + ResourceTypes, + AccountSasPermissions, + ) + from azure.core.exceptions import ResourceExistsError, HttpResponseError +except ImportError: + _logger.debug("Cannot 'import azure-storage-blob'.") + +try: + from azure.identity import DefaultAzureCredential +except ImportError: + _logger.debug("Cannot 'import azure-identity'.") + + +class IrAttachment(osv.osv): + _inherit = "ir.attachment" + + def _get_stores(self): + l = ["azure"] + l += super(IrAttachment, self)._get_stores() + return l + + @api.model + def _get_blob_service_client(self): + """Connect to Azure and return the blob service client + + The following environment variables must be set: + * ``AZURE_STORAGE_CONNECTION_STRING`` + or + * ``AZURE_STORAGE_ACCOUNT_NAME`` + * ``AZURE_STORAGE_ACCOUNT_URL`` + * ``AZURE_STORAGE_ACCOUNT_KEY`` + or if you want to use AAD (pod identity), set it to 1 or 0 + * ``AZURE_STORAGE_USE_AAD`` + + """ + connect_str = os.environ.get("AZURE_STORAGE_CONNECTION_STRING") + account_name = os.environ.get("AZURE_STORAGE_ACCOUNT_NAME") + account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL") + account_key = os.environ.get("AZURE_STORAGE_ACCOUNT_KEY") + account_use_aad = os.environ.get("AZURE_STORAGE_USE_AAD") + if not ( + connect_str + or (account_name and account_url and account_key) + or account_use_aad + ): + msg = _( + "If you want to read from the Azure container, you must provide the " + "following environment variables:\n" + "* AZURE_STORAGE_CONNECTION_STRING\n" + "or\n" + "* AZURE_STORAGE_ACCOUNT_NAME\n" + "* AZURE_STORAGE_ACCOUNT_URL\n" + "* AZURE_STORAGE_ACCOUNT_KEY\n" + "or\n" + "* AZURE_STORAGE_USE_AAD\n" + ) + raise exceptions.UserError(msg) + blob_service_client = None + if account_use_aad: + token_credential = DefaultAzureCredential() + blob_service_client = BlobServiceClient( + account_url=account_url, credential=token_credential + ) + elif connect_str: + try: + blob_service_client = BlobServiceClient.from_connection_string( + connect_str + ) + except HttpResponseError as error: + _logger.exception( + "Error during the connection to Azure container using the " + "connection string." + ) + raise exceptions.UserError(str(error)) + else: + try: + sas_token = generate_account_sas( + account_name=account_name, + account_key=account_key, + resource_types=ResourceTypes(container=True, object=True), + permission=AccountSasPermissions(read=True, write=True), + expiry=datetime.utcnow() + timedelta(hours=1), + ) + blob_service_client = BlobServiceClient( + account_url=account_url, + credential=sas_token, + ) + except HttpResponseError as error: + _logger.exception( + "Error during the connection to Azure container using the Shared " + "Access Signature (SAS)" + ) + raise exceptions.UserError(str(error)) + return blob_service_client + + @api.model + def _get_container_name(self): + """ + Container naming rules: + https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names + """ + running_env = os.environ.get("RUNNING_ENV", "dev") + storage_name = os.environ.get("AZURE_STORAGE_NAME", r"{env}-{db}") + storage_name = storage_name.format(env=running_env, db=self.env.cr.dbname) + # replace invalid characters by _ + storage_name = re.sub(r"[\W_]+", "-", storage_name) + # lowercase, max 63 chars + return str.lower(storage_name)[:63] + + @api.model + def _get_azure_container(self, container_name=None): + if not container_name: + container_name = self._get_container_name() + try: + blob_service_client = self._get_blob_service_client() + except exceptions.UserError: + _logger.exception( + "error accessing to storage '%s' please check credentials ", + container_name, + ) + return False + container_client = blob_service_client.get_container_client(container_name) + if not container_client.exists(): + try: + # Create the container + container_client.create_container() + except HttpResponseError as error: + _logger.exception("Error during the creation of the Azure container") + raise exceptions.UserError(str(error)) + return container_client + + @api.model + def _store_file_read(self, fname, bin_size=False): + if fname.startswith("azure://"): + key = fname.replace("azure://", "", 1).lower() + if "/" in key: + container_name, key = key.split("/", 1) + else: + container_name = None + container_client = self._get_azure_container(container_name) + # if container cannot be retrived, abort reading from azure storage + if not container_client: + return "" + try: + blob_client = container_client.get_blob_client(key) + read = blob_client.download_blob().readall() + except HttpResponseError: + read = "" + _logger.info("Attachment '%s' missing on object storage", fname) + return read + else: + return super(IrAttachment, self)._store_file_read(fname, bin_size) + + @api.model + def _store_file_write(self, key, bin_data): + location = self.env.context.get("storage_location") or self._storage() + if location == "azure": + container_client = self._get_azure_container() + filename = "azure://%s/%s" % (container_client.container_name, key) + with io.BytesIO() as file: + blob_client = container_client.get_blob_client(key.lower()) + file.write(bin_data) + file.seek(0) + try: + blob_client.upload_blob(file, blob_type="BlockBlob") + except ResourceExistsError: + pass + except HttpResponseError as error: + # log verbose error from azure, return short message for user + _logger.exception("Error during storage of the file %s" % filename) + raise exceptions.UserError( + _("The file could not be stored: %s") % str(error) + ) + else: + _super = super(IrAttachment, self) + filename = _super._store_file_write(key, bin_data) + return filename + + @api.model + def _store_file_delete(self, fname): + if fname.startswith("azure://"): + key = fname.replace("azure://", "", 1).lower() + if "/" in key: + container_name, key = key.split("/", 1) + else: + container_name = None + container_client = self._get_azure_container(container_name) + if not container_client: + return "" + # delete the file only if it is on the current configured container + # otherwise, we might delete files used on a different environment + try: + blob_client = container_client.get_blob_client(key) + blob_client.delete_blob() + _logger.info("File %s deleted on the object storage" % (fname)) + except HttpResponseError: + # log verbose error from azure, return short message for + # user + _logger.exception("Error during deletion of the file %s" % fname) + else: + super(IrAttachment, self)._store_file_delete(fname)