diff --git a/attachment_azure/README.rst b/attachment_azure/README.rst new file mode 100644 index 0000000..48e3c99 --- /dev/null +++ b/attachment_azure/README.rst @@ -0,0 +1,43 @@ +=========================================== +Attachments on Microsoft Azure Blob Storage +=========================================== + +This addon allows to store the attachments (documents and assets) on `Microsoft Azure +Blob Storage `_. + +Configuration +------------- + +Activate Azure Blob storage: + +* Create or set the system parameter with the key ``ir_attachment.location`` + and the value in the form ``azure``. + +Configure accesses with environment variables: + +* ``AZURE_STORAGE_CONNECTION_STRING`` or +* ``AZURE_STORAGE_ACCOUNT_NAME`` +* ``AZURE_STORAGE_ACCOUNT_URL`` +* ``AZURE_STORAGE_ACCOUNT_KEY`` + +One container will be created per database using the `RUNNING_ENV` environment variable +and the name of the database. By default, `RUNNING_ENV` is set to `dev`. + +The container name can be overridden with environment variable ``AZURE_STORAGE_NAME``. +The strings ``{db}`` and ``{env}`` can be used inside that variable and the values +will be replaced respectively by the database name and environment name. + +This addon must be added in the server wide addons with (``--load`` option): + +``--load=web,attachment_azure`` + +The System Parameter ``ir_attachment.storage.force.database`` can be customized to +force storage of files in the database. See the documentation of the module +``base_attachment_object_storage``. + +Limitations +----------- + +* You need to call ``env['ir.attachment'].force_storage()`` after + having changed the ``ir_attachment.location`` configuration in order to + migrate the existing attachments to Azure Blob Storage. diff --git a/attachment_azure/__init__.py b/attachment_azure/__init__.py new file mode 100644 index 0000000..0650744 --- /dev/null +++ b/attachment_azure/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/attachment_azure/__manifest__.py b/attachment_azure/__manifest__.py new file mode 100644 index 0000000..ce875a4 --- /dev/null +++ b/attachment_azure/__manifest__.py @@ -0,0 +1,22 @@ +# Copyright 2016-2021 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +{ + "name": "Attachments on Azure storage", + "summary": "Store assets and attachments on a Azure compatible object storage", + "version": "13.0.1.0.0", + "author": "Camptocamp, " + "Open Source Integrators, " + "Serpent Consulting Services, " + "Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Knowledge Management", + "depends": ["base_attachment_object_storage"], + "external_dependencies": { + "python": ["azure-storage-blob", "azure-identity"], + }, + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "installable": True, + "development_status": "Beta", + "maintainers": ["max3903"], +} diff --git a/attachment_azure/models/__init__.py b/attachment_azure/models/__init__.py new file mode 100644 index 0000000..aaf38a1 --- /dev/null +++ b/attachment_azure/models/__init__.py @@ -0,0 +1 @@ +from . import ir_attachment diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py new file mode 100644 index 0000000..247d407 --- /dev/null +++ b/attachment_azure/models/ir_attachment.py @@ -0,0 +1,201 @@ +# Copyright 2016-2021 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +import io +import logging +import os +import re +from datetime import datetime, timedelta + +from odoo import _, api, exceptions, models + +_logger = logging.getLogger(__name__) + +try: + from azure.storage.blob import ( + BlobServiceClient, + generate_account_sas, + ResourceTypes, + AccountSasPermissions, + ) + from azure.core.exceptions import ResourceExistsError, HttpResponseError +except ImportError: + _logger.debug("Cannot 'import azure-storage-blob'.") + +try: + from azure.identity import DefaultAzureCredential +except ImportError: + _logger.debug("Cannot 'import azure-identity'.") + + +class IrAttachment(models.Model): + _inherit = "ir.attachment" + + def _get_stores(self): + l = ["azure"] + l += super(IrAttachment, self)._get_stores() + return l + + @api.model + def _get_blob_service_client(self): + """Connect to Azure and return the blob service client + + The following environment variables must be set: + * ``AZURE_STORAGE_CONNECTION_STRING`` + or + * ``AZURE_STORAGE_ACCOUNT_NAME`` + * ``AZURE_STORAGE_ACCOUNT_URL`` + * ``AZURE_STORAGE_ACCOUNT_KEY`` + or if you want to use AAD (pod identity), set it to 1 or 0 + * ``AZURE_STORAGE_USE_AAD`` + + """ + connect_str = os.environ.get("AZURE_STORAGE_CONNECTION_STRING") + account_name = os.environ.get("AZURE_STORAGE_ACCOUNT_NAME") + account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL") + account_key = os.environ.get("AZURE_STORAGE_ACCOUNT_KEY") + account_use_aad = os.environ.get("AZURE_STORAGE_USE_AAD") + if not ( + connect_str + or (account_name and account_url and account_key) + or account_use_aad + ): + msg = _( + "If you want to read from the Azure container, you must provide the " + "following environment variables:\n" + "* AZURE_STORAGE_CONNECTION_STRING\n" + "or\n" + "* AZURE_STORAGE_ACCOUNT_NAME\n" + "* AZURE_STORAGE_ACCOUNT_URL\n" + "* AZURE_STORAGE_ACCOUNT_KEY\n" + "or\n" + "* AZURE_STORAGE_USE_AAD\n" + ) + raise exceptions.UserError(msg) + blob_service_client = None + if account_use_aad: + token_credential = DefaultAzureCredential() + blob_service_client = BlobServiceClient( + account_url=account_url, credential=token_credential + ) + elif connect_str: + try: + blob_service_client = BlobServiceClient.from_connection_string( + connect_str + ) + except HttpResponseError as error: + _logger.exception( + "Error during the connection to Azure container using the " + "connection string." + ) + raise exceptions.UserError(str(error)) + else: + try: + sas_token = generate_account_sas( + account_name=account_name, + account_key=account_key, + resource_types=ResourceTypes(container=True, object=True), + permission=AccountSasPermissions(read=True, write=True), + expiry=datetime.utcnow() + timedelta(hours=1), + ) + blob_service_client = BlobServiceClient( + account_url=account_url, + credential=sas_token, + ) + except HttpResponseError as error: + _logger.exception( + "Error during the connection to Azure container using the Shared " + "Access Signature (SAS)" + ) + raise exceptions.UserError(str(error)) + return blob_service_client + + @api.model + def _get_container_name(self): + """ + Container naming rules: + https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names + """ + running_env = os.environ.get("RUNNING_ENV", "dev") + storage_name = os.environ.get('AZURE_STORAGE_NAME', r'{env}-{db}') + storage_name = storage_name.format( + env=running_env, + db=self.env.cr.dbname + ) + # replace invalid characters by _ + storage_name = re.sub(r"[\W_]+", "-", storage_name) + # lowercase, max 63 chars + return str.lower(storage_name)[:63] + + @api.model + def _get_azure_container(self): + container_name = self._get_container_name() + blob_service_client = self._get_blob_service_client() + container_client = blob_service_client.get_container_client(container_name) + try: + # Create the container + container_client.create_container() + except ResourceExistsError: + pass + except HttpResponseError as error: + _logger.exception("Error during the creation of the Azure container") + raise exceptions.UserError(str(error)) + return container_client + + @api.model + def _store_file_read(self, fname, bin_size=False): + if fname.startswith("azure://"): + container_client = self._get_azure_container() + key = fname.replace("azure://", "", 1).lower() + try: + blob_client = container_client.get_blob_client(key) + read = blob_client.download_blob().readall() + except HttpResponseError: + read = "" + _logger.info("Attachment '%s' missing on object storage", fname) + return read + else: + return super(IrAttachment, self)._store_file_read(fname, bin_size) + + @api.model + def _store_file_write(self, key, bin_data): + location = self.env.context.get("storage_location") or self._storage() + if location == "azure": + container_client = self._get_azure_container() + with io.BytesIO() as file: + blob_client = container_client.get_blob_client(key.lower()) + file.write(bin_data) + file.seek(0) + filename = "azure://%s" % (key) + try: + blob_client.upload_blob(file, blob_type="BlockBlob") + except ResourceExistsError: + pass + except HttpResponseError as error: + # log verbose error from azure, return short message for user + _logger.exception("Error during storage of the file %s" % filename) + raise exceptions.UserError( + _("The file could not be stored: %s") % str(error) + ) + else: + _super = super(IrAttachment, self) + filename = _super._store_file_write(key, bin_data) + return filename + + @api.model + def _store_file_delete(self, fname): + if fname.startswith("azure://"): + container_client = self._get_azure_container() + key = fname.replace("azure://", "", 1).lower() + # delete the file only if it is on the current configured container + # otherwise, we might delete files used on a different environment + try: + blob_client = container_client.get_blob_client(key) + blob_client.delete_blob() + _logger.info("File %s deleted on the object storage" % (fname)) + except HttpResponseError: + # log verbose error from azure, return short message for + # user + _logger.exception("Error during deletion of the file %s" % fname) + else: + super(IrAttachment, self)._store_file_delete(fname) diff --git a/requirements.txt b/requirements.txt index 556253a..0525441 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ +azure-storage-blob==12.8.1 +azure-identity==1.6.0 boto3==1.9.102 redis==2.10.5 python-json-logger==0.1.5