Merge pull request #371 from vrenaville/azure70

[7.0] Backport module for azure storage

attachment_azure/README.rst

@@ -0,0 +1,46 @@
+===========================================
+Attachments on Microsoft Azure Blob Storage
+===========================================
+
+This addon allows to store the attachments (documents and assets) on `Microsoft Azure
+Blob Storage <https://docs.microsoft.com/azure/storage/blobs/>`_.
+
+Configuration
+-------------
+
+Activate Azure Blob storage:
+
+* Create or set the system parameter with the key ``ir_attachment.location``
+  and the value in the form ``azure``.
+
+Configure accesses with environment variables:
+
+* ``AZURE_STORAGE_CONNECTION_STRING`` or
+* ``AZURE_STORAGE_ACCOUNT_NAME``
+* ``AZURE_STORAGE_ACCOUNT_URL``
+* ``AZURE_STORAGE_ACCOUNT_KEY``
+
+One container will be created per database using the `RUNNING_ENV` environment variable
+and the name of the database. By default, `RUNNING_ENV` is set to `dev`.
+
+The container name can be overridden with environment variable ``AZURE_STORAGE_NAME``.
+The strings ``{db}`` and ``{env}`` can be used inside that variable and the values
+will be replaced respectively by the database name and environment name.
+
+The container name will also be stored in the database for each attachment,
+and will be used to access the right container in the storage.
+
+This addon must be added in the server wide addons with (``--load`` option):
+
+``--load=web,attachment_azure``
+
+The System Parameter ``ir_attachment.storage.force.database`` can be customized to
+force storage of files in the database. See the documentation of the module
+``base_attachment_object_storage``.
+
+Limitations
+-----------
+
+* You need to call ``env['ir.attachment'].force_storage()`` after
+  having changed the ``ir_attachment.location`` configuration in order to
+  migrate the existing attachments to Azure Blob Storage.

attachment_azure/__init__.py

@@ -0,0 +1,4 @@
+# Copyright 2016-2019 Camptocamp SA
+# Copyright 2021 Open Source Integrators
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+from . import models

attachment_azure/__openerp__.py

@@ -0,0 +1,19 @@
+# Copyright 2016-2019 Camptocamp SA
+# Copyright 2021 Open Source Integrators
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+{
+    "name": "Attachments on Azure storage",
+    "summary": "Store assets and attachments on a Azure compatible object storage",
+    "version": "15.0.1.0.0",
+    "author": "Camptocamp, "
+    "Open Source Integrators, "
+    "Serpent Consulting Services, "
+    "Odoo Community Association (OCA)",
+    "license": "AGPL-3",
+    "category": "Knowledge Management",
+    "depends": ["base_attachment_object_storage"],
+    "website": "https://github.com/camptocamp/odoo-cloud-platform",
+    "installable": True,
+    "development_status": "Beta",
+    "maintainers": ["max3903"],
+}

attachment_azure/models/__init__.py

@@ -0,0 +1,4 @@
+# Copyright 2016-2019 Camptocamp SA
+# Copyright 2021 Open Source Integrators
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+from . import ir_attachment

attachment_azure/models/ir_attachment.py

@@ -0,0 +1,215 @@
+# Copyright 2016-2019 Camptocamp SA
+# Copyright 2021 Open Source Integrators
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+import io
+import logging
+import os
+import re
+from datetime import datetime, timedelta
+
+from openerp.tools.translate import _
+from openerp.osv import osv
+from openerp.osv.orm import except_orm
+
+_logger = logging.getLogger(__name__)
+
+try:
+    from azure.storage.blob import (
+        BlobServiceClient,
+        generate_account_sas,
+        ResourceTypes,
+        AccountSasPermissions,
+    )
+    from azure.core.exceptions import ResourceExistsError, HttpResponseError
+except ImportError:
+    _logger.debug("Cannot 'import azure-storage-blob'.")
+
+try:
+    from azure.identity import DefaultAzureCredential
+except ImportError:
+    _logger.debug("Cannot 'import azure-identity'.")
+
+
+class IrAttachment(osv.osv):
+    _inherit = "ir.attachment"
+
+    def _get_stores(self):
+        l = ["azure"]
+        l += super(IrAttachment, self)._get_stores()
+        return l
+
+    def _get_blob_service_client(self):
+        """Connect to Azure and return the blob service client
+
+        The following environment variables must be set:
+        * ``AZURE_STORAGE_CONNECTION_STRING``
+        or
+        * ``AZURE_STORAGE_ACCOUNT_NAME``
+        * ``AZURE_STORAGE_ACCOUNT_URL``
+        * ``AZURE_STORAGE_ACCOUNT_KEY``
+        or if you want to use AAD (pod identity), set it to 1 or 0
+        * ``AZURE_STORAGE_USE_AAD``
+
+        """
+        connect_str = os.environ.get("AZURE_STORAGE_CONNECTION_STRING")
+        account_name = os.environ.get("AZURE_STORAGE_ACCOUNT_NAME")
+        account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL")
+        account_key = os.environ.get("AZURE_STORAGE_ACCOUNT_KEY")
+        account_use_aad = os.environ.get("AZURE_STORAGE_USE_AAD")
+        if not (
+            connect_str
+            or (account_name and account_url and account_key)
+            or account_use_aad
+        ):
+            msg = _(
+                "If you want to read from the Azure container, you must provide the "
+                "following environment variables:\n"
+                "* AZURE_STORAGE_CONNECTION_STRING\n"
+                "or\n"
+                "* AZURE_STORAGE_ACCOUNT_NAME\n"
+                "* AZURE_STORAGE_ACCOUNT_URL\n"
+                "* AZURE_STORAGE_ACCOUNT_KEY\n"
+                "or\n"
+                "* AZURE_STORAGE_USE_AAD\n"
+            )
+            raise osv.except_osv(_("UserError"), msg)
+        blob_service_client = None
+        if account_use_aad:
+            token_credential = DefaultAzureCredential()
+            blob_service_client = BlobServiceClient(
+                account_url=account_url, credential=token_credential
+            )
+        elif connect_str:
+            try:
+                blob_service_client = BlobServiceClient.from_connection_string(
+                    connect_str
+                )
+            except HttpResponseError as error:
+                _logger.exception(
+                    "Error during the connection to Azure container using the "
+                    "connection string."
+                )
+                raise osv.except_osv(_("UserError"), str(error))
+        else:
+            try:
+                sas_token = generate_account_sas(
+                    account_name=account_name,
+                    account_key=account_key,
+                    resource_types=ResourceTypes(container=True, object=True),
+                    permission=AccountSasPermissions(read=True, write=True),
+                    expiry=datetime.utcnow() + timedelta(hours=1),
+                )
+                blob_service_client = BlobServiceClient(
+                    account_url=account_url,
+                    credential=sas_token,
+                )
+            except HttpResponseError as error:
+                _logger.exception(
+                    "Error during the connection to Azure container using the Shared "
+                    "Access Signature (SAS)"
+                )
+                raise osv.except_osv(_("UserError"), str(error))
+        return blob_service_client
+
+    def _get_container_name(self):
+        """
+        Container naming rules:
+        https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names
+        """
+        running_env = os.environ.get("RUNNING_ENV", "dev")
+        dbname = os.environ.get("DB_NAME", "odoodb")
+        storage_name = os.environ.get("AZURE_STORAGE_NAME", r"{env}-{db}")
+        storage_name = storage_name.format(env=running_env, db=dbname)
+        # replace invalid characters by _
+        storage_name = re.sub(r"[\W_]+", "-", storage_name)
+        # lowercase, max 63 chars
+        return str.lower(storage_name)[:63]
+
+    def _get_azure_container(self, container_name=None):
+        if not container_name:
+            container_name = self._get_container_name()
+        try:
+            blob_service_client = self._get_blob_service_client()
+        except Exception:
+            _logger.exception(
+                "error accessing to storage '%s' please check credentials ",
+                container_name,
+            )
+            return False
+        container_client = blob_service_client.get_container_client(container_name)
+        if not container_client.exists():
+            try:
+                # Create the container
+                container_client.create_container()
+            except HttpResponseError as error:
+                _logger.exception("Error during the creation of the Azure container")
+                raise osv.except_osv(_("UserError"), str(error))
+        return container_client
+
+    def _store_file_read(self, fname, bin_size=False):
+        if fname.startswith("azure://"):
+            key = fname.replace("azure://", "", 1).lower()
+            if "/" in key:
+                container_name, key = key.split("/", 1)
+            else:
+                container_name = None
+            container_client = self._get_azure_container(container_name)
+            # if container cannot be retrived, abort reading from azure storage
+            if not container_client:
+                return ""
+            try:
+                blob_client = container_client.get_blob_client(key)
+                read = blob_client.download_blob().readall()
+            except HttpResponseError:
+                read = ""
+                _logger.info("Attachment '%s' missing on object storage", fname)
+            return read
+        else:
+            return super(IrAttachment, self)._store_file_read(fname, bin_size)
+
+    def _store_file_write(self, storage, key, bin_data):
+        if storage == "azure":
+            container_client = self._get_azure_container()
+            filename = "azure://%s/%s" % (container_client.container_name, key)
+            with io.BytesIO() as file:
+                blob_client = container_client.get_blob_client(key.lower())
+                file.write(bin_data)
+                file.seek(0)
+                try:
+                    blob_client.upload_blob(file, blob_type="BlockBlob")
+                except ResourceExistsError:
+                    pass
+                except HttpResponseError as error:
+                    # log verbose error from azure, return short message for user
+                    _logger.exception("Error during storage of the file %s" % filename)
+                    raise osv.except_osv(
+                        _("UserError"),
+                        _("The file could not be stored: %s") % str(error),
+                    )
+        else:
+            _super = super(IrAttachment, self)
+            filename = _super._store_file_write(key, bin_data)
+        return filename
+
+    def _store_file_delete(self, fname):
+        if fname.startswith("azure://"):
+            key = fname.replace("azure://", "", 1).lower()
+            if "/" in key:
+                container_name, key = key.split("/", 1)
+            else:
+                container_name = None
+            container_client = self._get_azure_container(container_name)
+            if not container_client:
+                return ""
+            # delete the file only if it is on the current configured container
+            # otherwise, we might delete files used on a different environment
+            try:
+                blob_client = container_client.get_blob_client(key)
+                blob_client.delete_blob()
+                _logger.info("File %s deleted on the object storage" % (fname))
+            except HttpResponseError:
+                # log verbose error from azure, return short message for
+                # user
+                _logger.exception("Error during deletion of the file %s" % fname)
+        else:
+            super(IrAttachment, self)._store_file_delete(fname)

base_attachment_object_storage/models/ir_attachment.py

@@ -20,21 +20,19 @@ _logger = logging.getLogger(__name__)
 
 
 def clean_fs(files):
-    _logger.info('cleaning old files from filestore')
+    _logger.info("cleaning old files from filestore")
     for full_path in files:
         if os.path.exists(full_path):
             try:
                 os.unlink(full_path)
             except OSError:
                 _logger.info(
-                    "_file_delete could not unlink %s",
+                    "_file_delete could not unlink %s", full_path, exc_info=True
-                    full_path, exc_info=True
                 )
             except IOError:
                 # Harmless and needed for race conditions
                 _logger.info(
-                    "_file_delete could not unlink %s",
+                    "_file_delete could not unlink %s", full_path, exc_info=True
-                    full_path, exc_info=True
                 )
 
 
@@ -52,34 +50,32 @@ def savepoint(cursor):
 
 
 class IrAttachment(osv.osv):
-    _inherit = 'ir.attachment'
+    _inherit = "ir.attachment"
 
     @staticmethod
     def _compute_checksum(bin_data):
-        """ compute the checksum for the given datas
+        """compute the checksum for the given datas
         :param bin_data : datas in its binary form
         """
         # an empty file has a checksum too (for caching)
-        return hashlib.sha1(bin_data or '').hexdigest()
+        return hashlib.sha1(bin_data or "").hexdigest()
 
     def _is_user_admin(self, cr, uid):
         if uid == SUPERUSER_ID:
             return True
         else:
-            return self.pool.get('res.users').has_group(
+            return self.pool.get("res.users").has_group(
-                cr, uid, 'base.group_erp_manager'
+                cr, uid, "base.group_erp_manager"
             )
 
     def _storage(self, cr, uid, context=None):
-        return self.pool['ir.config_parameter'].get_param(
+        return self.pool["ir.config_parameter"].get_param(
-            cr, SUPERUSER_ID, 'ir_attachment.location', 'file'
+            cr, SUPERUSER_ID, "ir_attachment.location", "file"
         )
 
     def _full_path(self, cr, uid, location, path):
         # Hack to allow filestore migration from local filesystem to any remote
-        return super(IrAttachment, self)._full_path(
+        return super(IrAttachment, self)._full_path(cr, uid, "file://filestore", path)
-            cr, uid, 'file://filestore', path
-        )
 
     def _register_hook(self, cr):
         super(IrAttachment, self)._register_hook(cr)
@@ -101,7 +97,7 @@ class IrAttachment(osv.osv):
         # done during the initialization. We need to move the attachments that
         # could have been created or updated in other addons before this addon
         # was loaded
-        update_module = load_modules_frame.f_locals.get('update_module')
+        update_module = load_modules_frame.f_locals.get("update_module")
 
         # We need to call the migration on the loading of the model because
         # when we are upgrading addons, some of them might add attachments.
@@ -110,12 +106,12 @@ class IrAttachment(osv.osv):
         # Typical example is images of ir.ui.menu which are updated in
         # ir.attachment at every upgrade of the addons
         if update_module:
-            self.pool.get('ir.attachment')._force_storage_to_object_storage(
+            self.pool.get("ir.attachment")._force_storage_to_object_storage(
                 cr, SUPERUSER_ID
             )
 
     def _save_in_db_anyway(self, cr, uid, ids, context=None):
-        """ Return whether an attachment must be stored in db
+        """Return whether an attachment must be stored in db
 
         When we are using an Object Store. This is sometimes required
         because the object storage is slower than the database/filesystem.
@@ -130,12 +126,11 @@ class IrAttachment(osv.osv):
         an old database with attachments pointing to deleted assets.
 
         """
-        assert (isinstance(ids, int) or
+        assert isinstance(ids, int) or len(ids) == 1, "Expecting only one record"
-                len(ids) == 1), 'Expecting only one record'
         rec = self.browse(cr, uid, ids, context=context)
 
         # assets
-        if rec.res_model == 'ir.ui.view':
+        if rec.res_model == "ir.ui.view":
             # assets are stored in 'ir.ui.view'
             return True
 
@@ -146,58 +141,50 @@ class IrAttachment(osv.osv):
         # we keep them in the database instead of the object storage
         location = self._storage(cr, uid)
         for attach in self.browse(cr, uid, id, context):
-            if (location in self._get_stores() and
+            if location in self._get_stores() and self._save_in_db_anyway(
-                    self._save_in_db_anyway(cr, uid, [id], context)):
+                cr, uid, [id], context
+            ):
                 # compute the fields that depend on datas
-                bin_data = value and value.decode('base64') or ''
+                bin_data = value and value.decode("base64") or ""
                 vals = {
-                    'file_size': len(bin_data),
+                    "file_size": len(bin_data),
-                    'checksum': self._compute_checksum(bin_data),
+                    "checksum": self._compute_checksum(bin_data),
-                    'db_datas': value,
+                    "db_datas": value,
                     # we seriously don't need index content on those fields
-                    'index_content': False,
+                    "index_content": False,
-                    'store_fname': False,
+                    "store_fname": False,
                 }
                 fname = attach.store_fname
                 # write as superuser, as user probably does not
                 # have write access
-                super(IrAttachment, self).write(
+                super(IrAttachment, self).write(cr, SUPERUSER_ID, id, vals, context)
-                    cr, SUPERUSER_ID, id, vals, context
-                )
                 if fname:
                     self._file_delete(cr, uid, fname)
                 continue
-            self._data_set(cr, uid, id, 'datas', value, None, context)
+            self._data_set(cr, uid, id, "datas", value, None, context)
 
     def _store_file_read(self, fname, bin_size=False):
-        storage = fname.partition('://')[0]
+        storage = fname.partition("://")[0]
-        raise NotImplementedError(
+        raise NotImplementedError("No implementation for %s" % (storage,))
-            'No implementation for %s' % (storage,)
-        )
 
     def _store_file_write(self, storage, key, bin_data):
-        raise NotImplementedError(
+        raise NotImplementedError("No implementation for %s" % (storage,))
-            'No implementation for %s' % (storage,)
-        )
 
     def _store_file_delete(self, fname):
-        storage = fname.partition('://')[0]
+        storage = fname.partition("://")[0]
-        raise NotImplementedError(
+        raise NotImplementedError("No implementation for %s" % (storage,))
-            'No implementation for %s' % (storage,)
-        )
 
     def _file_read(self, cr, uid, location, fname, bin_size=False):
         if self._is_file_from_a_store(fname):
             return self._store_file_read(fname, bin_size=bin_size)
         else:
             _super = super(IrAttachment, self)
-            return _super._file_read(cr, uid, location,
+            return _super._file_read(cr, uid, location, fname, bin_size=bin_size)
-                                     fname, bin_size=bin_size)
 
     def _file_write(self, cr, uid, location, value):
         storage = self._storage(cr, uid)
         if storage in self._get_stores():
-            bin_data = value.decode('base64')
+            bin_data = value.decode("base64")
             key = self._compute_checksum(bin_data)
             filename = self._store_file_write(storage, key, bin_data)
         else:
@@ -209,8 +196,9 @@ class IrAttachment(osv.osv):
         if self._is_file_from_a_store(fname):
             # using SQL to include files hidden through unlink or due to record
             # rules
-            cr.execute("SELECT COUNT(*) FROM ir_attachment "
+            cr.execute(
-                       "WHERE store_fname = %s", (fname,))
+                "SELECT COUNT(*) FROM ir_attachment " "WHERE store_fname = %s", (fname,)
+            )
             count = cr.fetchone()[0]
             if int(count) == 1:
                 self._store_file_delete(fname)
@@ -219,33 +207,31 @@ class IrAttachment(osv.osv):
 
     def _is_file_from_a_store(self, fname):
         for store_name in self._get_stores():
-            uri = '{}://'.format(store_name)
+            uri = "{}://".format(store_name)
             if fname.startswith(uri):
                 return True
         return False
 
     def _move_attachment_to_store(self, cr, uid, ids, context=None):
-        assert (isinstance(ids, int) or
+        assert isinstance(ids, int) or len(ids) == 1, "Expecting only one record"
-                len(ids) == 1), 'Expecting only one record'
         rec = self.browse(cr, uid, ids, context)
-        _logger.info('inspecting attachment %s (%d)', rec.name, rec.id)
+        _logger.info("inspecting attachment %s (%d)", rec.name, rec.id)
         fname = rec.store_fname
         if fname:
             # migrating from filesystem filestore
             # or from the old 'store_fname' without the bucket name
-            _logger.info('moving %s on the object storage', fname)
+            _logger.info("moving %s on the object storage", fname)
-            self.write(cr, uid, ids, {'datas': rec.datas}, context)
+            self.write(cr, uid, ids, {"datas": rec.datas}, context)
-            _logger.info('moved %s on the object storage', fname)
+            _logger.info("moved %s on the object storage", fname)
             return self._full_path(cr, uid, None, fname)
         elif rec.db_datas:
-            _logger.info('moving on the object storage from database')
+            _logger.info("moving on the object storage from database")
-            self.write(cr, uid, ids, {'datas': rec.datas}, context)
+            self.write(cr, uid, ids, {"datas": rec.datas}, context)
 
     def force_storage(self, cr, uid, context=None):
         if not self._is_user_admin(cr, uid):
             raise except_orm(
-                _('Error'),
+                _("Error"), _("Only administrators can execute this action.")
-                _('Only administrators can execute this action.')
             )
         storage = self._storage(cr, uid)
         if storage not in self._get_stores():
@@ -253,10 +239,10 @@ class IrAttachment(osv.osv):
         self._force_storage_to_object_storage(cr, uid, context)
 
     def _force_storage_to_object_storage(self, cr, uid, context=None):
-        _logger.info('migrating files to the object storage')
+        _logger.info("migrating files to the object storage")
         storage = self._storage(cr, uid)
 
-        domain = [('store_fname', 'not like', '{}://%'.format(storage))]
+        domain = [("store_fname", "not like", "{}://%".format(storage))]
 
         ids = self.search(cr, uid, domain, context=context)
         files_to_clean = []
@@ -273,7 +259,7 @@ class IrAttachment(osv.osv):
                         "WHERE id = %s "
                         "FOR UPDATE NOWAIT",
                         (attachment_id,),
-                        log_exceptions=False
+                        log_exceptions=False,
                     )
 
                     path = self._move_attachment_to_store(
@@ -282,8 +268,9 @@ class IrAttachment(osv.osv):
                     if path:
                         files_to_clean.append(path)
             except psycopg2.OperationalError:
-                _logger.error('Could not migrate attachment %s to %s' %
+                _logger.error(
-                              (attachment_id, storage))
+                    "Could not migrate attachment %s to %s" % (attachment_id, storage)
+                )
 
         def clean():
             clean_fs(files_to_clean)
@@ -291,8 +278,8 @@ class IrAttachment(osv.osv):
         # delete the files from the filesystem once we know the changes
         # have been committed in ir.attachment
         if files_to_clean:
-            cr.after('commit', clean)
+            cr.commit()
 
     def _get_stores(self):
-        """ To get the list of stores activated in the system  """
+        """To get the list of stores activated in the system"""
         return []

cloud_platform/models/cloud_platform.py

@@ -19,93 +19,95 @@ _logger = logging.getLogger(__name__)
 
 
 def is_true(strval):
-    return bool(strtobool(strval or '0'.lower()))
+    return bool(strtobool(strval or "0".lower()))
 
 
-PlatformConfig = namedtuple(
+PlatformConfig = namedtuple("PlatformConfig", "filestore")
-    'PlatformConfig',
-    'filestore'
-)
 
 
 class FilestoreKind(object):
-    db = 'db'
+    db = "db"
-    s3 = 's3'  # or compatible s3 object storage
+    s3 = "s3"  # or compatible s3 object storage
-    swift = 'swift'
+    swift = "swift"
-    file = 'file'
+    file = "file"
+    azure = "azure"
 
 
 class CloudPlatform(osv.osv_abstract):
-    _name = 'cloud.platform'
+    _name = "cloud.platform"
 
     def _platform_kinds(self):
         # XXX for backward compatibility, we need this one here, move
         # it in cloud_platform_exoscale in V11
-        return ['exoscale']
+        return ["exoscale"]
+
+    def _filestore_kinds(self):
+        # XXX for backward compatibility, we need this one here, move
+        # it in cloud_platform_exoscale in V11
+        return ["exoscale"]
 
     # XXX for backward compatibility, we need this one here, move
     # it in cloud_platform_exoscale in V11
     def _config_by_server_env_for_exoscale(self):
         configs = {
-            'prod': PlatformConfig(filestore=FilestoreKind.s3),
+            "prod": PlatformConfig(filestore=FilestoreKind.s3),
-            'integration': PlatformConfig(filestore=FilestoreKind.s3),
+            "integration": PlatformConfig(filestore=FilestoreKind.s3),
-            'test': PlatformConfig(filestore=FilestoreKind.db),
+            "test": PlatformConfig(filestore=FilestoreKind.db),
-            'dev': PlatformConfig(filestore=FilestoreKind.db),
+            "dev": PlatformConfig(filestore=FilestoreKind.db),
         }
         return configs
 
     def _config_by_server_env(self, platform_kind, environment):
         configs_getter = getattr(
-            self,
+            self, "_config_by_server_env_for_%s" % platform_kind, None
-            '_config_by_server_env_for_%s' % platform_kind,
-            None
         )
         configs = configs_getter() if configs_getter else {}
         return configs.get(environment) or FilestoreKind.db
 
     def _get_running_env(self):
-        environment_name = config['running_env']
+        environment_name = config["running_env"]
-        if environment_name.startswith('labs'):
+        if environment_name.startswith("labs"):
             # We allow to have environments such as 'labs-logistics'
             # or 'labs-finance', in order to have the matching ribbon.
-            environment_name = 'labs'
+            environment_name = "labs"
         return environment_name
 
     # Due to the addition of the ovh cloud platform
     # This will be moved to cloud_platform_exoscale on v11
     def install_exoscale(self, cr, uid, context=None):
-        self.install(cr, uid, 'exoscale', context)
+        self.install(cr, uid, "exoscale", context)
 
     def install(self, cr, uid, platform_kind, context=None):
         assert platform_kind in self._platform_kinds()
-        params = self.pool.get('ir.config_parameter')
+        params = self.pool.get("ir.config_parameter")
         params.set_param(
-            cr, SUPERUSER_ID,
+            cr, SUPERUSER_ID, "cloud.platform.kind", platform_kind, context=context
-            'cloud.platform.kind', platform_kind,
-            context=context
         )
         environment_name = self._get_running_env()
         configs = self._config_by_server_env(platform_kind, environment_name)
         params.set_param(
-            cr, SUPERUSER_ID,
+            cr,
-            'ir_attachment.location', configs.filestore,
+            SUPERUSER_ID,
-            context=context
+            "ir_attachment.location",
+            configs.filestore,
+            context=context,
         )
         self.check(cr, uid, context)
         if configs.filestore in [FilestoreKind.swift, FilestoreKind.s3]:
-            self.pool.get('ir.attachment').force_storage(
+            self.pool.get("ir.attachment").force_storage(
                 cr, SUPERUSER_ID, context=context
             )
-        _logger.info('cloud platform configured for {}'.format(platform_kind))
+        _logger.info("cloud platform configured for {}".format(platform_kind))
 
     def _check_swift(self, cr, uid, environment_name, context=None):
-        params = self.pool.get('ir.config_parameter')
+        params = self.pool.get("ir.config_parameter")
         use_swift = (
             params.get_param(
-                cr, SUPERUSER_ID, 'ir_attachment.location', context=context
+                cr, SUPERUSER_ID, "ir_attachment.location", context=context
-            ) == FilestoreKind.swift
             )
-        if environment_name in ('prod', 'integration'):
+            == FilestoreKind.swift
+        )
+        if environment_name in ("prod", "integration"):
             # Labs instances use swift or s3 by default, but we don't want
             # to enforce it in case we want to test something with a different
             # storage. At your own risks!
@@ -116,19 +118,19 @@ class CloudPlatform(osv.osv_abstract):
                 "automatically."
             )
         if use_swift:
-            assert os.environ.get('SWIFT_AUTH_URL'), (
+            assert os.environ.get("SWIFT_AUTH_URL"), (
                 "SWIFT_AUTH_URL environment variable is required when "
                 "ir_attachment.location is 'swift'."
             )
-            assert os.environ.get('SWIFT_ACCOUNT'), (
+            assert os.environ.get("SWIFT_ACCOUNT"), (
                 "SWIFT_ACCOUNT environment variable is required when "
                 "ir_attachment.location is 'swift'."
             )
-            assert os.environ.get('SWIFT_PASSWORD'), (
+            assert os.environ.get("SWIFT_PASSWORD"), (
                 "SWIFT_PASSWORD environment variable is required when "
                 "ir_attachment.location is 'swift'."
             )
-            container_name = os.environ.get('SWIFT_WRITE_CONTAINER')
+            container_name = os.environ.get("SWIFT_WRITE_CONTAINER")
             assert container_name, (
                 "SWIFT_WRITE_CONTAINER environment variable is required when "
                 "ir_attachment.location is 'swift'.\n"
@@ -139,9 +141,8 @@ class CloudPlatform(osv.osv_abstract):
                 "If you don't actually need a bucket, change the"
                 " 'ir_attachment.location' parameter."
             )
-            prod_container = bool(re.match(r'[a-z0-9-]+-odoo-prod',
+            prod_container = bool(re.match(r"[a-z0-9-]+-odoo-prod", container_name))
-                                           container_name))
+            if environment_name == "prod":
-            if environment_name == 'prod':
                 assert prod_container, (
                     "SWIFT_WRITE_CONTAINER should match '<client>-odoo-prod', "
                     "we got: '%s'" % (container_name,)
@@ -153,21 +154,28 @@ class CloudPlatform(osv.osv_abstract):
                     "SWIFT_WRITE_CONTAINER should not match "
                     "'<client>-odoo-prod', we got: '%s'" % (container_name,)
                 )
-        elif environment_name == 'test':
+        elif environment_name == "test":
             # store in DB so we don't have files local to the host
-            assert params.get_param(cr, SUPERUSER_ID, 'ir_attachment.location',
+            assert (
-                                    context=context) == 'db', (
+                params.get_param(
+                    cr, SUPERUSER_ID, "ir_attachment.location", context=context
+                )
+                == "db"
+            ), (
                 "In test instances, files must be stored in the database with "
                 "'ir_attachment.location' set to 'db'. This is "
                 "automatically set by the function 'install_ovh()'."
             )
 
     def _check_s3(self, cr, uid, environment_name, context=None):
-        params = self.pool.get('ir.config_parameter')
+        params = self.pool.get("ir.config_parameter")
-        use_s3 = params.get_param(
+        use_s3 = (
-            cr, SUPERUSER_ID, 'ir_attachment.location', context=context
+            params.get_param(
-        ) == FilestoreKind.s3
+                cr, SUPERUSER_ID, "ir_attachment.location", context=context
-        if environment_name in ('prod', 'integration'):
+            )
+            == FilestoreKind.s3
+        )
+        if environment_name in ("prod", "integration"):
             # Labs instances use swift or s3 by default, but we don't want
             # to enforce it in case we want to test something with a different
             # storage. At your own risks!
@@ -178,15 +186,15 @@ class CloudPlatform(osv.osv_abstract):
                 "automatically."
             )
         if use_s3:
-            assert os.environ.get('AWS_ACCESS_KEY_ID'), (
+            assert os.environ.get("AWS_ACCESS_KEY_ID"), (
                 "AWS_ACCESS_KEY_ID environment variable is required when "
                 "ir_attachment.location is 's3'."
             )
-            assert os.environ.get('AWS_SECRET_ACCESS_KEY'), (
+            assert os.environ.get("AWS_SECRET_ACCESS_KEY"), (
                 "AWS_SECRET_ACCESS_KEY environment variable is required when "
                 "ir_attachment.location is 's3'."
             )
-            bucket_name = os.environ.get('AWS_BUCKETNAME')
+            bucket_name = os.environ.get("AWS_BUCKETNAME")
             assert bucket_name, (
                 "AWS_BUCKETNAME environment variable is required when "
                 "ir_attachment.location is 's3'.\n"
@@ -197,8 +205,8 @@ class CloudPlatform(osv.osv_abstract):
                 "If you don't actually need a bucket, change the"
                 " 'ir_attachment.location' parameter."
             )
-            prod_bucket = bool(re.match(r'[a-z-0-9]+-odoo-prod', bucket_name))
+            prod_bucket = bool(re.match(r"[a-z-0-9]+-odoo-prod", bucket_name))
-            if environment_name == 'prod':
+            if environment_name == "prod":
                 assert prod_bucket, (
                     "AWS_BUCKETNAME should match '<client>-odoo-prod', "
                     "we got: '%s'" % (bucket_name,)
@@ -211,46 +219,137 @@ class CloudPlatform(osv.osv_abstract):
                     "we got: '%s'" % (bucket_name,)
                 )
 
-        elif environment_name == 'test':
+        elif environment_name == "test":
             # store in DB so we don't have files local to the host
-            assert params.get_param(cr, SUPERUSER_ID, 'ir_attachment.location',
+            assert (
-                                    context=context) == 'db', (
+                params.get_param(
+                    cr, SUPERUSER_ID, "ir_attachment.location", context=context
+                )
+                == "db"
+            ), (
                 "In test instances, files must be stored in the database with "
                 "'ir_attachment.location' set to 'db'. This is "
                 "automatically set by the function 'install_exoscale()'."
             )
 
+    def _check_azure(self, cr, uid, environment_name, context=None):
+        params = self.pool.get("ir.config_parameter")
+        use_azure = (
+            params.get_param(
+                cr, SUPERUSER_ID, "ir_attachment.location", context=context
+            )
+            == FilestoreKind.azure
+        )
+        if environment_name in ("prod", "integration"):
+            # Labs instances use azure by default, but we don't want
+            # to enforce it in case we want to test something with a different
+            # storage. At your own risks!
+            assert use_azure, (
+                "azure must be used on production and integration instances. "
+                "It is activated by setting 'ir_attachment.location.' to 'azure'."
+                " The 'install()' function sets this option "
+                "automatically."
+            )
+        if use_azure:
+            key_sets = [
+                ["AZURE_STORAGE_USE_AAD", "AZURE_STORAGE_ACCOUNT_URL"],
+                ["AZURE_STORAGE_CONNECTION_STRING"],
+                [
+                    "AZURE_STORAGE_ACCOUNT_NAME",
+                    "AZURE_STORAGE_ACCOUNT_URL",
+                    "AZURE_STORAGE_ACCOUNT_KEY",
+                ],
+            ]
+            is_valid = False
+            for key_set in key_sets:
+                if all([os.environ.get(key) for key in key_set]):
+                    is_valid = True
+                    break
+            assert is_valid, (
+                "When ir_attachment.location is set to 'azure', "
+                "at least one of the following enviromnent variable set "
+                "is required : {}".format(
+                    " or ".join(
+                        [" + ".join([key for key in key_set]) for key_set in key_sets]
+                    )
+                )
+            )
+            storage_name = os.environ.get("AZURE_STORAGE_NAME", "")
+            if environment_name in ("prod", "integration", "labs"):
+                assert storage_name, (
+                    "AZURE_STORAGE_NAME environment variable is required when "
+                    "ir_attachment.location is 'azure'.\n"
+                    "Normally, 'azure' is activated on labs, integration "
+                    "and production, but should not be used in dev environment"
+                    " (or using a dedicated dev bucket, never using the "
+                    "integration/prod bucket).\n"
+                    "If you don't actually need a bucket, change the"
+                    " 'ir_attachment.location' parameter."
+                )
+            # A bucket name is defined under the following format
+            # ^[a-z]+\-[a-z]+\-\d+$
+            # Anything other than prod bucket must be suffixed with env name
+            #
+            # Use AZURE_STORAGE_NAME_UNSTRUCTURED to by-pass check
+            # on bucket name structure
+            if os.environ.get("AZURE_STORAGE_NAME_UNSTRUCTURED"):
+                return
+            prod_bucket = bool(re.match(r"^[a-z]+\-[a-z]+\-\d+$", storage_name))
+            if environment_name == "prod":
+                assert prod_bucket, (
+                    "AZURE_STORAGE_NAME should match '^[a-z]+\\-[a-z]+\\-\\d+$', "
+                    "we got: '%s'" % (storage_name,)
+                )
+            else:
+                # if we are using the prod bucket on another instance
+                # such as an integration, we must be sure to be in read only!
+                assert not prod_bucket, (
+                    "AZURE_STORAGE_NAME should not match '^[a-z]+\\-[a-z]+\\-\\d+$', "
+                    "we got: '%s'" % (storage_name,)
+                )
+
+        elif environment_name == "test":
+            # store in DB so we don't have files local to the host
+            assert (
+                params.get_param(
+                    cr, SUPERUSER_ID, "ir_attachment.location", context=context
+                )
+                == "db"
+            ), (
+                "In test instances, files must be stored in the database with "
+                "'ir_attachment.location' set to 'db'. This is "
+                "automatically set by the function 'install()'."
+            )
+
     def _check_redis(self, cr, uid, environment_name, context=None):
-        if environment_name in ('prod', 'integration', 'labs', 'test'):
+        if environment_name in ("prod", "integration", "labs", "test"):
-            assert is_true(os.environ.get('ODOO_SESSION_REDIS')), (
+            assert is_true(os.environ.get("ODOO_SESSION_REDIS")), (
                 "Redis must be activated on prod, integration, labs,"
                 " test instances. This is done by setting ODOO_SESSION_REDIS=1."
             )
-            assert (os.environ.get('ODOO_SESSION_REDIS_HOST') or
+            assert os.environ.get("ODOO_SESSION_REDIS_URL") or os.environ.get(
-                    os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST')), (
+                "ODOO_SESSION_REDIS_SENTINEL_URL"
-                "ODOO_SESSION_REDIS_HOST or ODOO_SESSION_REDIS_SENTINEL_HOST "
+            ), (
+                "ODOO_SESSION_REDIS_URL or ODOO_SESSION_REDIS_SENTINEL_URL "
                 "environment variable is required to connect on Redis"
             )
-            assert os.environ.get('ODOO_SESSION_REDIS_PREFIX'), (
+            assert os.environ.get("ODOO_SESSION_REDIS_PREFIX"), (
                 "ODOO_SESSION_REDIS_PREFIX environment variable is required "
                 "to store sessions on Redis"
             )
 
-            prefix = os.environ['ODOO_SESSION_REDIS_PREFIX']
+            prefix = os.environ["ODOO_SESSION_REDIS_PREFIX"]
-            assert re.match(r'^[a-z-0-9]+-odoo-[a-z-0-9]+$', prefix), (
+            assert re.match(r"^[a-z-0-9]+-odoo-[a-z-0-9]+$", prefix), (
                 "ODOO_SESSION_REDIS_PREFIX must match '<client>-odoo-<env>'"
                 ", we got: '%s'" % (prefix,)
             )
 
     def check(self, cr, uid, context=None):
-        if is_true(os.environ.get('ODOO_CLOUD_PLATFORM_UNSAFE')):
+        if is_true(os.environ.get("ODOO_CLOUD_PLATFORM_UNSAFE")):
-            _logger.warning(
+            _logger.warning("cloud platform checks disabled, this is not safe")
-                "cloud platform checks disabled, this is not safe"
-            )
             return
-        params = self.pool.get('ir.config_parameter')
+        params = self.pool.get("ir.config_parameter")
-        kind = params.get_param(cr, SUPERUSER_ID,
+        kind = params.get_param(cr, SUPERUSER_ID, "cloud.platform.kind", context=None)
-                                'cloud.platform.kind', context=None)
         if not kind:
             _logger.warning(
                 "cloud platform not configured, you should "
@@ -258,12 +357,14 @@ class CloudPlatform(osv.osv_abstract):
             )
             return
         environment_name = self._get_running_env()
-        if kind == 'exoscale':
+        if kind == "exoscale":
             self._check_s3(cr, uid, environment_name, context)
-        elif kind == 'ovh':
+        elif kind == "ovh":
             self._check_swift(cr, uid, environment_name, context)
+        elif kind == "azure":
+            self._check_azure(cr, uid, environment_name, context)
         self._check_redis(cr, uid, environment_name, context)
 
     def _register_hook(self, cr):
         super(CloudPlatform, self)._register_hook(cr)
-        self.pool.get('cloud.platform').check(cr, SUPERUSER_ID)
+        self.pool.get("cloud.platform").check(cr, SUPERUSER_ID)

cloud_platform_azure/README.md

@@ -0,0 +1,6 @@
+Cloud Platform Azure
+====================
+
+Install addons specific to the Azure setup.
+
+    * The object storage is Azure blob storage

cloud_platform_azure/__init__.py

@@ -0,0 +1 @@
+from . import models

cloud_platform_azure/__openerp__.py

@@ -0,0 +1,23 @@
+# Copyright 2017-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{
+    "name": "Cloud Platform Azure",
+    "summary": "Addons required for the Camptocamp Cloud Platform on Azure",
+    "version": "15.0.1.0.0",
+    "author": "Camptocamp,Odoo Community Association (OCA)",
+    "license": "AGPL-3",
+    "category": "Extra Tools",
+    "depends": [
+        "cloud_platform",
+        "attachment_azure",
+    ],
+    "excludes": [
+        "cloud_platform_ovh",
+        "cloud_platform_exoscale",
+    ],
+    "website": "https://www.camptocamp.com",
+    "data": [],
+    "installable": True,
+}

cloud_platform_azure/models/__init__.py

@@ -0,0 +1 @@
+from . import cloud_platform

cloud_platform_azure/models/cloud_platform.py

@@ -0,0 +1,37 @@
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import re
+import os
+
+from openerp.osv import osv
+from openerp.addons.cloud_platform.models.cloud_platform import FilestoreKind
+from openerp.addons.cloud_platform.models.cloud_platform import PlatformConfig
+
+
+class CloudPlatform(osv.osv):
+    _inherit = "cloud.platform"
+
+    def _filestore_kinds(self):
+        kinds = super(CloudPlatform, self)._filestore_kinds()
+        kinds.append("azure")
+        return kinds
+
+    def _platform_kinds(self):
+        kinds = super(CloudPlatform, self)._platform_kinds()
+        kinds.append("azure")
+        return kinds
+
+    def _config_by_server_env_for_azure(self):
+        fs_kinds = self._filestore_kinds()
+        configs = {
+            "prod": PlatformConfig(filestore=fs_kinds["azure"]),
+            "integration": PlatformConfig(filestore=fs_kinds["azure"]),
+            "labs": PlatformConfig(filestore=fs_kinds["azure"]),
+            "test": PlatformConfig(filestore=fs_kinds["db"]),
+            "dev": PlatformConfig(filestore=fs_kinds["db"]),
+        }
+        return configs
+
+    def install_azure(self, cr, uid, context=None):
+        self.install(cr, uid, "azure", context)

session_redis/http.py

@@ -24,45 +24,45 @@ except ImportError:
 
 
 def is_true(strval):
-    return bool(strtobool(strval or '0'.lower()))
+    return bool(strtobool(strval or "0".lower()))
 
 
-sentinel_host = os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST')
+sentinel_host = os.environ.get("ODOO_SESSION_REDIS_SENTINEL_HOST")
-sentinel_master_name = os.environ.get(
+sentinel_master_name = os.environ.get("ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME")
-    'ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME'
-)
 if sentinel_host and not sentinel_master_name:
     raise Exception(
         "ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME must be defined "
         "when using session_redis"
     )
-sentinel_port = int(os.environ.get('ODOO_SESSION_REDIS_SENTINEL_PORT', 26379))
+sentinel_port = int(os.environ.get("ODOO_SESSION_REDIS_SENTINEL_PORT", 26379))
-host = os.environ.get('ODOO_SESSION_REDIS_HOST', 'localhost')
+host = os.environ.get("ODOO_SESSION_REDIS_URL", "localhost")
-port = int(os.environ.get('ODOO_SESSION_REDIS_PORT', 6379))
+port = int(os.environ.get("ODOO_SESSION_REDIS_PORT", 6379))
-prefix = os.environ.get('ODOO_SESSION_REDIS_PREFIX')
+prefix = os.environ.get("ODOO_SESSION_REDIS_PREFIX")
-url = os.environ.get('ODOO_SESSION_REDIS_URL')
+url = os.environ.get("ODOO_SESSION_REDIS_URL")
-password = os.environ.get('ODOO_SESSION_REDIS_PASSWORD')
+password = os.environ.get("ODOO_SESSION_REDIS_PASSWORD")
-expiration = os.environ.get('ODOO_SESSION_REDIS_EXPIRATION')
+expiration = os.environ.get("ODOO_SESSION_REDIS_EXPIRATION")
-anon_expiration = os.environ.get('ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS')
+anon_expiration = os.environ.get("ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS")
 
 
 def session_store():
     if sentinel_host:
-        sentinel = Sentinel([(sentinel_host, sentinel_port)],
+        sentinel = Sentinel([(sentinel_host, sentinel_port)], password=password)
-                            password=password)
         redis_client = sentinel.master_for(sentinel_master_name)
     elif url:
         redis_client = redis.from_url(url)
     else:
         redis_client = redis.Redis(host=host, port=port, password=password)
-    return RedisSessionStore(redis=redis_client, prefix=prefix,
+    return RedisSessionStore(
+        redis=redis_client,
+        prefix=prefix,
         expiration=expiration,
         anon_expiration=anon_expiration,
-                             session_class=Session)
+        session_class=Session,
+    )
 
 
 def session_gc(session_store):
-    """ Do not garbage collect the sessions
+    """Do not garbage collect the sessions
 
     Redis keys are automatically cleaned at the end of their
     expiration.
@@ -79,18 +79,26 @@ def purge_fs_sessions(path):
             pass
 
 
-if is_true(os.environ.get('ODOO_SESSION_REDIS')):
+if is_true(os.environ.get("ODOO_SESSION_REDIS")):
     if sentinel_host:
-        _logger.debug("HTTP sessions stored in Redis with prefix '%s'. "
+        _logger.debug(
+            "HTTP sessions stored in Redis with prefix '%s'. "
             "Using Sentinel on %s:%s",
-                      sentinel_host, sentinel_port, prefix or '')
+            sentinel_host,
+            sentinel_port,
+            prefix or "",
+        )
     else:
-        _logger.debug("HTTP sessions stored in Redis with prefix '%s' on "
+        _logger.debug(
-                      "%s:%s", host, port, prefix or '')
+            "HTTP sessions stored in Redis with prefix '%s' on " "%s:%s",
+            host,
+            port,
+            prefix or "",
+        )
 
     store = session_store()
     for handler in openerp.service.wsgi_server.module_handlers:
-        if hasattr(handler, 'session_store'):
+        if hasattr(handler, "session_store"):
             handler.session_store = store
     http.session_gc = session_gc
     # clean the existing sessions on the file system