From 1d8bbcf765bcdda15f38bafe67d39f64d31631ff Mon Sep 17 00:00:00 2001 From: Hiren Pattani Date: Thu, 20 May 2021 09:53:12 -0500 Subject: [PATCH 1/6] [ADD] attachment_azure --- attachment_azure/README.rst | 39 ++++++ attachment_azure/__init__.py | 4 + attachment_azure/__manifest__.py | 22 +++ attachment_azure/models/__init__.py | 4 + attachment_azure/models/ir_attachment.py | 165 +++++++++++++++++++++++ requirements.txt | 1 + 6 files changed, 235 insertions(+) create mode 100644 attachment_azure/README.rst create mode 100644 attachment_azure/__init__.py create mode 100644 attachment_azure/__manifest__.py create mode 100644 attachment_azure/models/__init__.py create mode 100644 attachment_azure/models/ir_attachment.py diff --git a/attachment_azure/README.rst b/attachment_azure/README.rst new file mode 100644 index 0000000..6247ee9 --- /dev/null +++ b/attachment_azure/README.rst @@ -0,0 +1,39 @@ +=========================================== +Attachments on Microsoft Azure Blob Storage +=========================================== + +This addon allows to store the attachments (documents and assets) on `Microsoft Azure +Blob Storage `_. + +Configuration +------------- + +Activate Azure Blob storage: + +* Create or set the system parameter with the key ``ir_attachment.location`` + and the value in the form ``azure``. + +Configure accesses with environment variables: + +* ``AZURE_STORAGE_CONNECTION_STRING`` or +* ``AZURE_STORAGE_ACCOUNT_NAME`` +* ``AZURE_STORAGE_ACCOUNT_URL`` +* ``AZURE_STORAGE_ACCOUNT_KEY`` + +One container will be created per database using the `RUNNING_ENV` environment variable +and the name of the database. By default, `RUNNING_ENV` is set to `dev`. + +This addon must be added in the server wide addons with (``--load`` option): + +``--load=web,attachment_azure`` + +The System Parameter ``ir_attachment.storage.force.database`` can be customized to +force storage of files in the database. See the documentation of the module +``base_attachment_object_storage``. + +Limitations +----------- + +* You need to call ``env['ir.attachment'].force_storage()`` after + having changed the ``ir_attachment.location`` configuration in order to + migrate the existing attachments to Azure Blob Storage. diff --git a/attachment_azure/__init__.py b/attachment_azure/__init__.py new file mode 100644 index 0000000..49d7105 --- /dev/null +++ b/attachment_azure/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +from . import models diff --git a/attachment_azure/__manifest__.py b/attachment_azure/__manifest__.py new file mode 100644 index 0000000..5774934 --- /dev/null +++ b/attachment_azure/__manifest__.py @@ -0,0 +1,22 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +{ + "name": "Attachments on Azure storage", + "summary": "Store assets and attachments on a Azure compatible object storage", + "version": "14.0.1.0.0", + "author": "Camptocamp, " + "Open Source Integrators, " + "Serpent Consulting Services, " + "Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Knowledge Management", + "depends": ["base_attachment_object_storage"], + "external_dependencies": { + "python": ["azure-storage-blob"], + }, + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "installable": True, + "development_status": "Beta", + "maintainers": ["max3903"], +} diff --git a/attachment_azure/models/__init__.py b/attachment_azure/models/__init__.py new file mode 100644 index 0000000..cb9b196 --- /dev/null +++ b/attachment_azure/models/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +from . import ir_attachment diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py new file mode 100644 index 0000000..f0ae764 --- /dev/null +++ b/attachment_azure/models/ir_attachment.py @@ -0,0 +1,165 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +import io +import logging +import os +from datetime import datetime, timedelta + +from odoo import _, api, exceptions, models + +_logger = logging.getLogger(__name__) + +try: + from azure.storage.blob import ( + BlobServiceClient, + generate_account_sas, + ResourceTypes, + AccountSasPermissions, + ) + from azure.core.exceptions import ResourceExistsError, HttpResponseError +except ImportError: + _logger.debug("Cannot 'import azure-storage-blob'.") + + +class IrAttachment(models.Model): + _inherit = "ir.attachment" + + def _get_stores(self): + l = ["azure"] + l += super(IrAttachment, self)._get_stores() + return l + + @api.model + def _get_blob_service_client(self): + """Connect to Azure and return the blob service client + + The following environment variables must be set: + * ``AZURE_STORAGE_CONNECTION_STRING`` + or + * ``AZURE_STORAGE_ACCOUNT_NAME`` + * ``AZURE_STORAGE_ACCOUNT_URL`` + * ``AZURE_STORAGE_ACCOUNT_KEY`` + + """ + connect_str = os.environ.get("AZURE_STORAGE_CONNECTION_STRING") + account_name = os.environ.get("AZURE_STORAGE_ACCOUNT_NAME") + account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL") + account_key = os.environ.get("AZURE_STORAGE_ACCOUNT_KEY") + if not (connect_str or (account_name and account_url and account_key)): + msg = _( + "If you want to read from the Azure container, you must provide the " + "following environment variables:\n" + "* AZURE_STORAGE_CONNECTION_STRING\n" + "or\n" + "* AZURE_STORAGE_ACCOUNT_NAME\n" + "* AZURE_STORAGE_ACCOUNT_URL\n" + "* AZURE_STORAGE_ACCOUNT_KEY\n" + ) + raise exceptions.UserError(msg) + blob_service_client = None + if connect_str: + try: + blob_service_client = BlobServiceClient.from_connection_string( + connect_str + ) + except HttpResponseError as error: + _logger.exception( + "Error during the connection to Azure container using the " + "connection string." + ) + raise exceptions.UserError(str(error)) + else: + try: + sas_token = generate_account_sas( + account_name=account_name, + account_key=account_key, + resource_types=ResourceTypes(service=True), + permission=AccountSasPermissions(read=True), + expiry=datetime.utcnow() + timedelta(hours=1), + ) + blob_service_client = BlobServiceClient( + account_url=account_url, + credential=sas_token, + ) + except HttpResponseError as error: + _logger.exception( + "Error during the connection to Azure container using the Shared " + "Access Signature (SAS)" + ) + raise exceptions.UserError(str(error)) + return blob_service_client + + @api.model + def _get_azure_container(self): + running_env = os.environ.get("RUNNING_ENV", "dev") + container_name = str.lower(running_env + "-" + self.env.cr.dbname) + blob_service_client = self._get_blob_service_client() + container_client = blob_service_client.get_container_client(container_name) + try: + # Create the container + container_client.create_container() + except ResourceExistsError: + pass + except HttpResponseError as error: + _logger.exception("Error during the creation of the Azure container") + raise exceptions.UserError(str(error)) + return container_client + + @api.model + def _store_file_read(self, fname, bin_size=False): + if fname.startswith("azure://"): + container_client = self._get_azure_container() + key = fname.replace("azure://", "", 1).lower() + try: + blob_client = container_client.get_blob_client(key) + read = blob_client.download_blob().readall() + except HttpResponseError: + read = "" + _logger.info("Attachment '%s' missing on object storage", fname) + return read + else: + return super(IrAttachment, self)._store_file_read(fname, bin_size) + + @api.model + def _store_file_write(self, key, bin_data): + location = self.env.context.get("storage_location") or self._storage() + if location == "azure": + container_client = self._get_azure_container() + with io.BytesIO() as file: + blob_client = container_client.get_blob_client(key.lower()) + file.write(bin_data) + file.seek(0) + filename = "azure://%s" % (key) + try: + blob_client.upload_blob(file, blob_type="BlockBlob") + except ResourceExistsError: + pass + except HttpResponseError as error: + # log verbose error from azure, return short message for user + _logger.exception("Error during storage of the file %s" % filename) + raise exceptions.UserError( + _("The file could not be stored: %s") % str(error) + ) + else: + _super = super(IrAttachment, self) + filename = _super._store_file_write(key, bin_data) + return filename + + @api.model + def _store_file_delete(self, fname): + if fname.startswith("azure://"): + container_client = self._get_azure_container() + key = fname.replace("azure://", "", 1).lower() + # delete the file only if it is on the current configured container + # otherwise, we might delete files used on a different environment + try: + blob_client = container_client.get_blob_client(key) + blob_client.delete_blob() + _logger.info("File %s deleted on the object storage" % (fname)) + except HttpResponseError: + # log verbose error from azure, return short message for + # user + _logger.exception("Error during deletion of the file %s" % fname) + else: + super(IrAttachment, self)._store_file_delete(fname) diff --git a/requirements.txt b/requirements.txt index ce3f5e6..babf6fc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +azure-storage-blob==12.8.1 boto3==1.9.102 redis==2.10.5 python-json-logger==0.1.5 From 050ce4918433890770779e8dd14bd5f17607b51d Mon Sep 17 00:00:00 2001 From: Nils Hamerlinck Date: Tue, 20 Jul 2021 22:11:39 +0700 Subject: [PATCH 2/6] [FIX] attachment_azure: higher level of permissions needed to create container and upload blobs when using the SAS token --- attachment_azure/models/ir_attachment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py index f0ae764..e75f1d9 100644 --- a/attachment_azure/models/ir_attachment.py +++ b/attachment_azure/models/ir_attachment.py @@ -74,8 +74,8 @@ class IrAttachment(models.Model): sas_token = generate_account_sas( account_name=account_name, account_key=account_key, - resource_types=ResourceTypes(service=True), - permission=AccountSasPermissions(read=True), + resource_types=ResourceTypes(container=True, object=True), + permission=AccountSasPermissions(read=True, write=True), expiry=datetime.utcnow() + timedelta(hours=1), ) blob_service_client = BlobServiceClient( From e6d2b43f2b8364fe767cb1a1f4ff3a27a6e53e3c Mon Sep 17 00:00:00 2001 From: Nils Hamerlinck Date: Tue, 20 Jul 2021 22:04:17 +0700 Subject: [PATCH 3/6] [FIX] attachment_azure: clean dbname to fit with container naming rules --- attachment_azure/models/ir_attachment.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py index e75f1d9..9576d49 100644 --- a/attachment_azure/models/ir_attachment.py +++ b/attachment_azure/models/ir_attachment.py @@ -4,6 +4,7 @@ import io import logging import os +import re from datetime import datetime, timedelta from odoo import _, api, exceptions, models @@ -91,9 +92,20 @@ class IrAttachment(models.Model): return blob_service_client @api.model - def _get_azure_container(self): + def _get_container_name(self): + """ + Container naming rules: + https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names + """ running_env = os.environ.get("RUNNING_ENV", "dev") - container_name = str.lower(running_env + "-" + self.env.cr.dbname) + # replace invalid characters by _ + dbname_cleaned = re.sub(r"[\W_]+", "-", self.env.cr.dbname) + # lowercase, max 63 chars + return str.lower(running_env + "-" + dbname_cleaned)[:63] + + @api.model + def _get_azure_container(self): + container_name = self._get_container_name() blob_service_client = self._get_blob_service_client() container_client = blob_service_client.get_container_client(container_name) try: From 223a84d6a99a05480b389bc88065003132fbebaf Mon Sep 17 00:00:00 2001 From: Vincent Renaville Date: Thu, 22 Jul 2021 10:26:50 +0200 Subject: [PATCH 4/6] [IMP] Add identity (#238) * [IMP] can use ad identity to access storage --- attachment_azure/__manifest__.py | 2 +- attachment_azure/models/ir_attachment.py | 23 +++++++++++++++++++++-- requirements.txt | 1 + 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/attachment_azure/__manifest__.py b/attachment_azure/__manifest__.py index 5774934..f685f39 100644 --- a/attachment_azure/__manifest__.py +++ b/attachment_azure/__manifest__.py @@ -13,7 +13,7 @@ "category": "Knowledge Management", "depends": ["base_attachment_object_storage"], "external_dependencies": { - "python": ["azure-storage-blob"], + "python": ["azure-storage-blob", "azure-identity"], }, "website": "https://github.com/camptocamp/odoo-cloud-platform", "installable": True, diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py index 9576d49..134d860 100644 --- a/attachment_azure/models/ir_attachment.py +++ b/attachment_azure/models/ir_attachment.py @@ -22,6 +22,11 @@ try: except ImportError: _logger.debug("Cannot 'import azure-storage-blob'.") +try: + from azure.identity import DefaultAzureCredential +except ImportError: + _logger.debug("Cannot 'import azure-identity'.") + class IrAttachment(models.Model): _inherit = "ir.attachment" @@ -41,13 +46,20 @@ class IrAttachment(models.Model): * ``AZURE_STORAGE_ACCOUNT_NAME`` * ``AZURE_STORAGE_ACCOUNT_URL`` * ``AZURE_STORAGE_ACCOUNT_KEY`` + or if you want to use AAD (pod identity), set it to 1 or 0 + * ``AZURE_STORAGE_USE_AAD`` """ connect_str = os.environ.get("AZURE_STORAGE_CONNECTION_STRING") account_name = os.environ.get("AZURE_STORAGE_ACCOUNT_NAME") account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL") account_key = os.environ.get("AZURE_STORAGE_ACCOUNT_KEY") - if not (connect_str or (account_name and account_url and account_key)): + account_use_aad = os.environ.get("AZURE_STORAGE_USE_AAD") + if not ( + connect_str + or (account_name and account_url and account_key) + or account_use_aad + ): msg = _( "If you want to read from the Azure container, you must provide the " "following environment variables:\n" @@ -56,10 +68,17 @@ class IrAttachment(models.Model): "* AZURE_STORAGE_ACCOUNT_NAME\n" "* AZURE_STORAGE_ACCOUNT_URL\n" "* AZURE_STORAGE_ACCOUNT_KEY\n" + "or\n" + "* AZURE_STORAGE_USE_AAD\n" ) raise exceptions.UserError(msg) blob_service_client = None - if connect_str: + if account_use_aad: + token_credential = DefaultAzureCredential() + blob_service_client = BlobServiceClient( + account_url=account_url, credential=token_credential + ) + elif connect_str: try: blob_service_client = BlobServiceClient.from_connection_string( connect_str diff --git a/requirements.txt b/requirements.txt index babf6fc..8d39c30 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ azure-storage-blob==12.8.1 +azure-identity==1.6.0 boto3==1.9.102 redis==2.10.5 python-json-logger==0.1.5 From 51cb97370816c9c657d674ce5ea5853e26dea1c3 Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Wed, 4 Aug 2021 10:50:05 +0200 Subject: [PATCH 5/6] [14.0][IMP] attachment_azure: Allow storage name override --- attachment_azure/README.rst | 4 ++++ attachment_azure/models/ir_attachment.py | 9 +++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/attachment_azure/README.rst b/attachment_azure/README.rst index 6247ee9..48e3c99 100644 --- a/attachment_azure/README.rst +++ b/attachment_azure/README.rst @@ -23,6 +23,10 @@ Configure accesses with environment variables: One container will be created per database using the `RUNNING_ENV` environment variable and the name of the database. By default, `RUNNING_ENV` is set to `dev`. +The container name can be overridden with environment variable ``AZURE_STORAGE_NAME``. +The strings ``{db}`` and ``{env}`` can be used inside that variable and the values +will be replaced respectively by the database name and environment name. + This addon must be added in the server wide addons with (``--load`` option): ``--load=web,attachment_azure`` diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py index 134d860..dcf76fc 100644 --- a/attachment_azure/models/ir_attachment.py +++ b/attachment_azure/models/ir_attachment.py @@ -117,10 +117,15 @@ class IrAttachment(models.Model): https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names """ running_env = os.environ.get("RUNNING_ENV", "dev") + storage_name = os.environ.get('AZURE_STORAGE_NAME', r'{env}-{db}') + storage_name = storage_name.format( + env=running_env, + db=self.env.cr.dbname + ) # replace invalid characters by _ - dbname_cleaned = re.sub(r"[\W_]+", "-", self.env.cr.dbname) + storage_name = re.sub(r"[\W_]+", "-", storage_name) # lowercase, max 63 chars - return str.lower(running_env + "-" + dbname_cleaned)[:63] + return str.lower(storage_name)[:63] @api.model def _get_azure_container(self): From 12a6a06e625ad84f2f6c347637f4300d5fad8e9b Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Mon, 26 Jul 2021 10:27:25 +0200 Subject: [PATCH 6/6] [MIG] attachment_azure: Backport from 14.0 to 13.0 --- attachment_azure/__init__.py | 3 --- attachment_azure/__manifest__.py | 4 ++-- attachment_azure/models/__init__.py | 3 --- attachment_azure/models/ir_attachment.py | 2 +- 4 files changed, 3 insertions(+), 9 deletions(-) diff --git a/attachment_azure/__init__.py b/attachment_azure/__init__.py index 49d7105..0650744 100644 --- a/attachment_azure/__init__.py +++ b/attachment_azure/__init__.py @@ -1,4 +1 @@ -# Copyright 2016-2019 Camptocamp SA -# Copyright 2021 Open Source Integrators -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) from . import models diff --git a/attachment_azure/__manifest__.py b/attachment_azure/__manifest__.py index f685f39..ce875a4 100644 --- a/attachment_azure/__manifest__.py +++ b/attachment_azure/__manifest__.py @@ -1,10 +1,10 @@ -# Copyright 2016-2019 Camptocamp SA +# Copyright 2016-2021 Camptocamp SA # Copyright 2021 Open Source Integrators # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) { "name": "Attachments on Azure storage", "summary": "Store assets and attachments on a Azure compatible object storage", - "version": "14.0.1.0.0", + "version": "13.0.1.0.0", "author": "Camptocamp, " "Open Source Integrators, " "Serpent Consulting Services, " diff --git a/attachment_azure/models/__init__.py b/attachment_azure/models/__init__.py index cb9b196..aaf38a1 100644 --- a/attachment_azure/models/__init__.py +++ b/attachment_azure/models/__init__.py @@ -1,4 +1 @@ -# Copyright 2016-2019 Camptocamp SA -# Copyright 2021 Open Source Integrators -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) from . import ir_attachment diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py index dcf76fc..247d407 100644 --- a/attachment_azure/models/ir_attachment.py +++ b/attachment_azure/models/ir_attachment.py @@ -1,4 +1,4 @@ -# Copyright 2016-2019 Camptocamp SA +# Copyright 2016-2021 Camptocamp SA # Copyright 2021 Open Source Integrators # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) import io