diff --git a/attachment_azure/README.rst b/attachment_azure/README.rst
new file mode 100644
index 0000000..b5c26d2
--- /dev/null
+++ b/attachment_azure/README.rst
@@ -0,0 +1,46 @@
+===========================================
+Attachments on Microsoft Azure Blob Storage
+===========================================
+
+This addon allows to store the attachments (documents and assets) on `Microsoft Azure
+Blob Storage `_.
+
+Configuration
+-------------
+
+Activate Azure Blob storage:
+
+* Create or set the system parameter with the key ``ir_attachment.location``
+ and the value in the form ``azure``.
+
+Configure accesses with environment variables:
+
+* ``AZURE_STORAGE_CONNECTION_STRING`` or
+* ``AZURE_STORAGE_ACCOUNT_NAME``
+* ``AZURE_STORAGE_ACCOUNT_URL``
+* ``AZURE_STORAGE_ACCOUNT_KEY``
+
+One container will be created per database using the `RUNNING_ENV` environment variable
+and the name of the database. By default, `RUNNING_ENV` is set to `dev`.
+
+The container name can be overridden with environment variable ``AZURE_STORAGE_NAME``.
+The strings ``{db}`` and ``{env}`` can be used inside that variable and the values
+will be replaced respectively by the database name and environment name.
+
+The container name will also be stored in the database for each attachment,
+and will be used to access the right container in the storage.
+
+This addon must be added in the server wide addons with (``--load`` option):
+
+``--load=web,attachment_azure``
+
+The System Parameter ``ir_attachment.storage.force.database`` can be customized to
+force storage of files in the database. See the documentation of the module
+``base_attachment_object_storage``.
+
+Limitations
+-----------
+
+* You need to call ``env['ir.attachment'].force_storage()`` after
+ having changed the ``ir_attachment.location`` configuration in order to
+ migrate the existing attachments to Azure Blob Storage.
diff --git a/attachment_azure/__init__.py b/attachment_azure/__init__.py
new file mode 100644
index 0000000..49d7105
--- /dev/null
+++ b/attachment_azure/__init__.py
@@ -0,0 +1,4 @@
+# Copyright 2016-2019 Camptocamp SA
+# Copyright 2021 Open Source Integrators
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+from . import models
diff --git a/attachment_azure/__openerp__.py b/attachment_azure/__openerp__.py
new file mode 100644
index 0000000..7ada22d
--- /dev/null
+++ b/attachment_azure/__openerp__.py
@@ -0,0 +1,19 @@
+# Copyright 2016-2019 Camptocamp SA
+# Copyright 2021 Open Source Integrators
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+{
+ "name": "Attachments on Azure storage",
+ "summary": "Store assets and attachments on a Azure compatible object storage",
+ "version": "15.0.1.0.0",
+ "author": "Camptocamp, "
+ "Open Source Integrators, "
+ "Serpent Consulting Services, "
+ "Odoo Community Association (OCA)",
+ "license": "AGPL-3",
+ "category": "Knowledge Management",
+ "depends": ["base_attachment_object_storage"],
+ "website": "https://github.com/camptocamp/odoo-cloud-platform",
+ "installable": True,
+ "development_status": "Beta",
+ "maintainers": ["max3903"],
+}
diff --git a/attachment_azure/models/__init__.py b/attachment_azure/models/__init__.py
new file mode 100644
index 0000000..cb9b196
--- /dev/null
+++ b/attachment_azure/models/__init__.py
@@ -0,0 +1,4 @@
+# Copyright 2016-2019 Camptocamp SA
+# Copyright 2021 Open Source Integrators
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+from . import ir_attachment
diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py
new file mode 100644
index 0000000..81415cd
--- /dev/null
+++ b/attachment_azure/models/ir_attachment.py
@@ -0,0 +1,215 @@
+# Copyright 2016-2019 Camptocamp SA
+# Copyright 2021 Open Source Integrators
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+import io
+import logging
+import os
+import re
+from datetime import datetime, timedelta
+
+from openerp.tools.translate import _
+from openerp.osv import osv
+from openerp.osv.orm import except_orm
+
+_logger = logging.getLogger(__name__)
+
+try:
+ from azure.storage.blob import (
+ BlobServiceClient,
+ generate_account_sas,
+ ResourceTypes,
+ AccountSasPermissions,
+ )
+ from azure.core.exceptions import ResourceExistsError, HttpResponseError
+except ImportError:
+ _logger.debug("Cannot 'import azure-storage-blob'.")
+
+try:
+ from azure.identity import DefaultAzureCredential
+except ImportError:
+ _logger.debug("Cannot 'import azure-identity'.")
+
+
+class IrAttachment(osv.osv):
+ _inherit = "ir.attachment"
+
+ def _get_stores(self):
+ l = ["azure"]
+ l += super(IrAttachment, self)._get_stores()
+ return l
+
+ def _get_blob_service_client(self):
+ """Connect to Azure and return the blob service client
+
+ The following environment variables must be set:
+ * ``AZURE_STORAGE_CONNECTION_STRING``
+ or
+ * ``AZURE_STORAGE_ACCOUNT_NAME``
+ * ``AZURE_STORAGE_ACCOUNT_URL``
+ * ``AZURE_STORAGE_ACCOUNT_KEY``
+ or if you want to use AAD (pod identity), set it to 1 or 0
+ * ``AZURE_STORAGE_USE_AAD``
+
+ """
+ connect_str = os.environ.get("AZURE_STORAGE_CONNECTION_STRING")
+ account_name = os.environ.get("AZURE_STORAGE_ACCOUNT_NAME")
+ account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL")
+ account_key = os.environ.get("AZURE_STORAGE_ACCOUNT_KEY")
+ account_use_aad = os.environ.get("AZURE_STORAGE_USE_AAD")
+ if not (
+ connect_str
+ or (account_name and account_url and account_key)
+ or account_use_aad
+ ):
+ msg = _(
+ "If you want to read from the Azure container, you must provide the "
+ "following environment variables:\n"
+ "* AZURE_STORAGE_CONNECTION_STRING\n"
+ "or\n"
+ "* AZURE_STORAGE_ACCOUNT_NAME\n"
+ "* AZURE_STORAGE_ACCOUNT_URL\n"
+ "* AZURE_STORAGE_ACCOUNT_KEY\n"
+ "or\n"
+ "* AZURE_STORAGE_USE_AAD\n"
+ )
+ raise osv.except_osv(_("UserError"), msg)
+ blob_service_client = None
+ if account_use_aad:
+ token_credential = DefaultAzureCredential()
+ blob_service_client = BlobServiceClient(
+ account_url=account_url, credential=token_credential
+ )
+ elif connect_str:
+ try:
+ blob_service_client = BlobServiceClient.from_connection_string(
+ connect_str
+ )
+ except HttpResponseError as error:
+ _logger.exception(
+ "Error during the connection to Azure container using the "
+ "connection string."
+ )
+ raise osv.except_osv(_("UserError"), str(error))
+ else:
+ try:
+ sas_token = generate_account_sas(
+ account_name=account_name,
+ account_key=account_key,
+ resource_types=ResourceTypes(container=True, object=True),
+ permission=AccountSasPermissions(read=True, write=True),
+ expiry=datetime.utcnow() + timedelta(hours=1),
+ )
+ blob_service_client = BlobServiceClient(
+ account_url=account_url,
+ credential=sas_token,
+ )
+ except HttpResponseError as error:
+ _logger.exception(
+ "Error during the connection to Azure container using the Shared "
+ "Access Signature (SAS)"
+ )
+ raise osv.except_osv(_("UserError"), str(error))
+ return blob_service_client
+
+ def _get_container_name(self):
+ """
+ Container naming rules:
+ https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names
+ """
+ running_env = os.environ.get("RUNNING_ENV", "dev")
+ dbname = os.environ.get("DB_NAME", "odoodb")
+ storage_name = os.environ.get("AZURE_STORAGE_NAME", r"{env}-{db}")
+ storage_name = storage_name.format(env=running_env, db=dbname)
+ # replace invalid characters by _
+ storage_name = re.sub(r"[\W_]+", "-", storage_name)
+ # lowercase, max 63 chars
+ return str.lower(storage_name)[:63]
+
+ def _get_azure_container(self, container_name=None):
+ if not container_name:
+ container_name = self._get_container_name()
+ try:
+ blob_service_client = self._get_blob_service_client()
+ except Exception:
+ _logger.exception(
+ "error accessing to storage '%s' please check credentials ",
+ container_name,
+ )
+ return False
+ container_client = blob_service_client.get_container_client(container_name)
+ if not container_client.exists():
+ try:
+ # Create the container
+ container_client.create_container()
+ except HttpResponseError as error:
+ _logger.exception("Error during the creation of the Azure container")
+ raise osv.except_osv(_("UserError"), str(error))
+ return container_client
+
+ def _store_file_read(self, fname, bin_size=False):
+ if fname.startswith("azure://"):
+ key = fname.replace("azure://", "", 1).lower()
+ if "/" in key:
+ container_name, key = key.split("/", 1)
+ else:
+ container_name = None
+ container_client = self._get_azure_container(container_name)
+ # if container cannot be retrived, abort reading from azure storage
+ if not container_client:
+ return ""
+ try:
+ blob_client = container_client.get_blob_client(key)
+ read = blob_client.download_blob().readall()
+ except HttpResponseError:
+ read = ""
+ _logger.info("Attachment '%s' missing on object storage", fname)
+ return read
+ else:
+ return super(IrAttachment, self)._store_file_read(fname, bin_size)
+
+ def _store_file_write(self, storage, key, bin_data):
+ if storage == "azure":
+ container_client = self._get_azure_container()
+ filename = "azure://%s/%s" % (container_client.container_name, key)
+ with io.BytesIO() as file:
+ blob_client = container_client.get_blob_client(key.lower())
+ file.write(bin_data)
+ file.seek(0)
+ try:
+ blob_client.upload_blob(file, blob_type="BlockBlob")
+ except ResourceExistsError:
+ pass
+ except HttpResponseError as error:
+ # log verbose error from azure, return short message for user
+ _logger.exception("Error during storage of the file %s" % filename)
+ raise osv.except_osv(
+ _("UserError"),
+ _("The file could not be stored: %s") % str(error),
+ )
+ else:
+ _super = super(IrAttachment, self)
+ filename = _super._store_file_write(key, bin_data)
+ return filename
+
+ def _store_file_delete(self, fname):
+ if fname.startswith("azure://"):
+ key = fname.replace("azure://", "", 1).lower()
+ if "/" in key:
+ container_name, key = key.split("/", 1)
+ else:
+ container_name = None
+ container_client = self._get_azure_container(container_name)
+ if not container_client:
+ return ""
+ # delete the file only if it is on the current configured container
+ # otherwise, we might delete files used on a different environment
+ try:
+ blob_client = container_client.get_blob_client(key)
+ blob_client.delete_blob()
+ _logger.info("File %s deleted on the object storage" % (fname))
+ except HttpResponseError:
+ # log verbose error from azure, return short message for
+ # user
+ _logger.exception("Error during deletion of the file %s" % fname)
+ else:
+ super(IrAttachment, self)._store_file_delete(fname)
diff --git a/base_attachment_object_storage/models/ir_attachment.py b/base_attachment_object_storage/models/ir_attachment.py
index 8ada85f..a857407 100644
--- a/base_attachment_object_storage/models/ir_attachment.py
+++ b/base_attachment_object_storage/models/ir_attachment.py
@@ -20,21 +20,19 @@ _logger = logging.getLogger(__name__)
def clean_fs(files):
- _logger.info('cleaning old files from filestore')
+ _logger.info("cleaning old files from filestore")
for full_path in files:
if os.path.exists(full_path):
try:
os.unlink(full_path)
except OSError:
_logger.info(
- "_file_delete could not unlink %s",
- full_path, exc_info=True
+ "_file_delete could not unlink %s", full_path, exc_info=True
)
except IOError:
# Harmless and needed for race conditions
_logger.info(
- "_file_delete could not unlink %s",
- full_path, exc_info=True
+ "_file_delete could not unlink %s", full_path, exc_info=True
)
@@ -52,34 +50,32 @@ def savepoint(cursor):
class IrAttachment(osv.osv):
- _inherit = 'ir.attachment'
+ _inherit = "ir.attachment"
@staticmethod
def _compute_checksum(bin_data):
- """ compute the checksum for the given datas
- :param bin_data : datas in its binary form
+ """compute the checksum for the given datas
+ :param bin_data : datas in its binary form
"""
# an empty file has a checksum too (for caching)
- return hashlib.sha1(bin_data or '').hexdigest()
+ return hashlib.sha1(bin_data or "").hexdigest()
def _is_user_admin(self, cr, uid):
if uid == SUPERUSER_ID:
return True
else:
- return self.pool.get('res.users').has_group(
- cr, uid, 'base.group_erp_manager'
+ return self.pool.get("res.users").has_group(
+ cr, uid, "base.group_erp_manager"
)
def _storage(self, cr, uid, context=None):
- return self.pool['ir.config_parameter'].get_param(
- cr, SUPERUSER_ID, 'ir_attachment.location', 'file'
+ return self.pool["ir.config_parameter"].get_param(
+ cr, SUPERUSER_ID, "ir_attachment.location", "file"
)
def _full_path(self, cr, uid, location, path):
# Hack to allow filestore migration from local filesystem to any remote
- return super(IrAttachment, self)._full_path(
- cr, uid, 'file://filestore', path
- )
+ return super(IrAttachment, self)._full_path(cr, uid, "file://filestore", path)
def _register_hook(self, cr):
super(IrAttachment, self)._register_hook(cr)
@@ -101,7 +97,7 @@ class IrAttachment(osv.osv):
# done during the initialization. We need to move the attachments that
# could have been created or updated in other addons before this addon
# was loaded
- update_module = load_modules_frame.f_locals.get('update_module')
+ update_module = load_modules_frame.f_locals.get("update_module")
# We need to call the migration on the loading of the model because
# when we are upgrading addons, some of them might add attachments.
@@ -110,12 +106,12 @@ class IrAttachment(osv.osv):
# Typical example is images of ir.ui.menu which are updated in
# ir.attachment at every upgrade of the addons
if update_module:
- self.pool.get('ir.attachment')._force_storage_to_object_storage(
+ self.pool.get("ir.attachment")._force_storage_to_object_storage(
cr, SUPERUSER_ID
)
def _save_in_db_anyway(self, cr, uid, ids, context=None):
- """ Return whether an attachment must be stored in db
+ """Return whether an attachment must be stored in db
When we are using an Object Store. This is sometimes required
because the object storage is slower than the database/filesystem.
@@ -130,12 +126,11 @@ class IrAttachment(osv.osv):
an old database with attachments pointing to deleted assets.
"""
- assert (isinstance(ids, int) or
- len(ids) == 1), 'Expecting only one record'
+ assert isinstance(ids, int) or len(ids) == 1, "Expecting only one record"
rec = self.browse(cr, uid, ids, context=context)
# assets
- if rec.res_model == 'ir.ui.view':
+ if rec.res_model == "ir.ui.view":
# assets are stored in 'ir.ui.view'
return True
@@ -146,58 +141,50 @@ class IrAttachment(osv.osv):
# we keep them in the database instead of the object storage
location = self._storage(cr, uid)
for attach in self.browse(cr, uid, id, context):
- if (location in self._get_stores() and
- self._save_in_db_anyway(cr, uid, [id], context)):
+ if location in self._get_stores() and self._save_in_db_anyway(
+ cr, uid, [id], context
+ ):
# compute the fields that depend on datas
- bin_data = value and value.decode('base64') or ''
+ bin_data = value and value.decode("base64") or ""
vals = {
- 'file_size': len(bin_data),
- 'checksum': self._compute_checksum(bin_data),
- 'db_datas': value,
+ "file_size": len(bin_data),
+ "checksum": self._compute_checksum(bin_data),
+ "db_datas": value,
# we seriously don't need index content on those fields
- 'index_content': False,
- 'store_fname': False,
+ "index_content": False,
+ "store_fname": False,
}
fname = attach.store_fname
# write as superuser, as user probably does not
# have write access
- super(IrAttachment, self).write(
- cr, SUPERUSER_ID, id, vals, context
- )
+ super(IrAttachment, self).write(cr, SUPERUSER_ID, id, vals, context)
if fname:
self._file_delete(cr, uid, fname)
continue
- self._data_set(cr, uid, id, 'datas', value, None, context)
+ self._data_set(cr, uid, id, "datas", value, None, context)
def _store_file_read(self, fname, bin_size=False):
- storage = fname.partition('://')[0]
- raise NotImplementedError(
- 'No implementation for %s' % (storage,)
- )
+ storage = fname.partition("://")[0]
+ raise NotImplementedError("No implementation for %s" % (storage,))
def _store_file_write(self, storage, key, bin_data):
- raise NotImplementedError(
- 'No implementation for %s' % (storage,)
- )
+ raise NotImplementedError("No implementation for %s" % (storage,))
def _store_file_delete(self, fname):
- storage = fname.partition('://')[0]
- raise NotImplementedError(
- 'No implementation for %s' % (storage,)
- )
+ storage = fname.partition("://")[0]
+ raise NotImplementedError("No implementation for %s" % (storage,))
def _file_read(self, cr, uid, location, fname, bin_size=False):
if self._is_file_from_a_store(fname):
return self._store_file_read(fname, bin_size=bin_size)
else:
_super = super(IrAttachment, self)
- return _super._file_read(cr, uid, location,
- fname, bin_size=bin_size)
+ return _super._file_read(cr, uid, location, fname, bin_size=bin_size)
def _file_write(self, cr, uid, location, value):
storage = self._storage(cr, uid)
if storage in self._get_stores():
- bin_data = value.decode('base64')
+ bin_data = value.decode("base64")
key = self._compute_checksum(bin_data)
filename = self._store_file_write(storage, key, bin_data)
else:
@@ -209,8 +196,9 @@ class IrAttachment(osv.osv):
if self._is_file_from_a_store(fname):
# using SQL to include files hidden through unlink or due to record
# rules
- cr.execute("SELECT COUNT(*) FROM ir_attachment "
- "WHERE store_fname = %s", (fname,))
+ cr.execute(
+ "SELECT COUNT(*) FROM ir_attachment " "WHERE store_fname = %s", (fname,)
+ )
count = cr.fetchone()[0]
if int(count) == 1:
self._store_file_delete(fname)
@@ -219,33 +207,31 @@ class IrAttachment(osv.osv):
def _is_file_from_a_store(self, fname):
for store_name in self._get_stores():
- uri = '{}://'.format(store_name)
+ uri = "{}://".format(store_name)
if fname.startswith(uri):
return True
return False
def _move_attachment_to_store(self, cr, uid, ids, context=None):
- assert (isinstance(ids, int) or
- len(ids) == 1), 'Expecting only one record'
+ assert isinstance(ids, int) or len(ids) == 1, "Expecting only one record"
rec = self.browse(cr, uid, ids, context)
- _logger.info('inspecting attachment %s (%d)', rec.name, rec.id)
+ _logger.info("inspecting attachment %s (%d)", rec.name, rec.id)
fname = rec.store_fname
if fname:
# migrating from filesystem filestore
# or from the old 'store_fname' without the bucket name
- _logger.info('moving %s on the object storage', fname)
- self.write(cr, uid, ids, {'datas': rec.datas}, context)
- _logger.info('moved %s on the object storage', fname)
+ _logger.info("moving %s on the object storage", fname)
+ self.write(cr, uid, ids, {"datas": rec.datas}, context)
+ _logger.info("moved %s on the object storage", fname)
return self._full_path(cr, uid, None, fname)
elif rec.db_datas:
- _logger.info('moving on the object storage from database')
- self.write(cr, uid, ids, {'datas': rec.datas}, context)
+ _logger.info("moving on the object storage from database")
+ self.write(cr, uid, ids, {"datas": rec.datas}, context)
def force_storage(self, cr, uid, context=None):
if not self._is_user_admin(cr, uid):
raise except_orm(
- _('Error'),
- _('Only administrators can execute this action.')
+ _("Error"), _("Only administrators can execute this action.")
)
storage = self._storage(cr, uid)
if storage not in self._get_stores():
@@ -253,10 +239,10 @@ class IrAttachment(osv.osv):
self._force_storage_to_object_storage(cr, uid, context)
def _force_storage_to_object_storage(self, cr, uid, context=None):
- _logger.info('migrating files to the object storage')
+ _logger.info("migrating files to the object storage")
storage = self._storage(cr, uid)
- domain = [('store_fname', 'not like', '{}://%'.format(storage))]
+ domain = [("store_fname", "not like", "{}://%".format(storage))]
ids = self.search(cr, uid, domain, context=context)
files_to_clean = []
@@ -273,7 +259,7 @@ class IrAttachment(osv.osv):
"WHERE id = %s "
"FOR UPDATE NOWAIT",
(attachment_id,),
- log_exceptions=False
+ log_exceptions=False,
)
path = self._move_attachment_to_store(
@@ -282,8 +268,9 @@ class IrAttachment(osv.osv):
if path:
files_to_clean.append(path)
except psycopg2.OperationalError:
- _logger.error('Could not migrate attachment %s to %s' %
- (attachment_id, storage))
+ _logger.error(
+ "Could not migrate attachment %s to %s" % (attachment_id, storage)
+ )
def clean():
clean_fs(files_to_clean)
@@ -291,8 +278,8 @@ class IrAttachment(osv.osv):
# delete the files from the filesystem once we know the changes
# have been committed in ir.attachment
if files_to_clean:
- cr.after('commit', clean)
+ cr.commit()
def _get_stores(self):
- """ To get the list of stores activated in the system """
+ """To get the list of stores activated in the system"""
return []
diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py
index 06de6d3..c92200b 100644
--- a/cloud_platform/models/cloud_platform.py
+++ b/cloud_platform/models/cloud_platform.py
@@ -19,93 +19,95 @@ _logger = logging.getLogger(__name__)
def is_true(strval):
- return bool(strtobool(strval or '0'.lower()))
+ return bool(strtobool(strval or "0".lower()))
-PlatformConfig = namedtuple(
- 'PlatformConfig',
- 'filestore'
-)
+PlatformConfig = namedtuple("PlatformConfig", "filestore")
class FilestoreKind(object):
- db = 'db'
- s3 = 's3' # or compatible s3 object storage
- swift = 'swift'
- file = 'file'
+ db = "db"
+ s3 = "s3" # or compatible s3 object storage
+ swift = "swift"
+ file = "file"
+ azure = "azure"
class CloudPlatform(osv.osv_abstract):
- _name = 'cloud.platform'
+ _name = "cloud.platform"
def _platform_kinds(self):
# XXX for backward compatibility, we need this one here, move
# it in cloud_platform_exoscale in V11
- return ['exoscale']
+ return ["exoscale"]
+
+ def _filestore_kinds(self):
+ # XXX for backward compatibility, we need this one here, move
+ # it in cloud_platform_exoscale in V11
+ return ["exoscale"]
# XXX for backward compatibility, we need this one here, move
# it in cloud_platform_exoscale in V11
def _config_by_server_env_for_exoscale(self):
configs = {
- 'prod': PlatformConfig(filestore=FilestoreKind.s3),
- 'integration': PlatformConfig(filestore=FilestoreKind.s3),
- 'test': PlatformConfig(filestore=FilestoreKind.db),
- 'dev': PlatformConfig(filestore=FilestoreKind.db),
+ "prod": PlatformConfig(filestore=FilestoreKind.s3),
+ "integration": PlatformConfig(filestore=FilestoreKind.s3),
+ "test": PlatformConfig(filestore=FilestoreKind.db),
+ "dev": PlatformConfig(filestore=FilestoreKind.db),
}
return configs
def _config_by_server_env(self, platform_kind, environment):
configs_getter = getattr(
- self,
- '_config_by_server_env_for_%s' % platform_kind,
- None
+ self, "_config_by_server_env_for_%s" % platform_kind, None
)
configs = configs_getter() if configs_getter else {}
return configs.get(environment) or FilestoreKind.db
def _get_running_env(self):
- environment_name = config['running_env']
- if environment_name.startswith('labs'):
+ environment_name = config["running_env"]
+ if environment_name.startswith("labs"):
# We allow to have environments such as 'labs-logistics'
# or 'labs-finance', in order to have the matching ribbon.
- environment_name = 'labs'
+ environment_name = "labs"
return environment_name
# Due to the addition of the ovh cloud platform
# This will be moved to cloud_platform_exoscale on v11
def install_exoscale(self, cr, uid, context=None):
- self.install(cr, uid, 'exoscale', context)
+ self.install(cr, uid, "exoscale", context)
def install(self, cr, uid, platform_kind, context=None):
assert platform_kind in self._platform_kinds()
- params = self.pool.get('ir.config_parameter')
+ params = self.pool.get("ir.config_parameter")
params.set_param(
- cr, SUPERUSER_ID,
- 'cloud.platform.kind', platform_kind,
- context=context
+ cr, SUPERUSER_ID, "cloud.platform.kind", platform_kind, context=context
)
environment_name = self._get_running_env()
configs = self._config_by_server_env(platform_kind, environment_name)
params.set_param(
- cr, SUPERUSER_ID,
- 'ir_attachment.location', configs.filestore,
- context=context
+ cr,
+ SUPERUSER_ID,
+ "ir_attachment.location",
+ configs.filestore,
+ context=context,
)
self.check(cr, uid, context)
if configs.filestore in [FilestoreKind.swift, FilestoreKind.s3]:
- self.pool.get('ir.attachment').force_storage(
+ self.pool.get("ir.attachment").force_storage(
cr, SUPERUSER_ID, context=context
)
- _logger.info('cloud platform configured for {}'.format(platform_kind))
+ _logger.info("cloud platform configured for {}".format(platform_kind))
def _check_swift(self, cr, uid, environment_name, context=None):
- params = self.pool.get('ir.config_parameter')
+ params = self.pool.get("ir.config_parameter")
use_swift = (
params.get_param(
- cr, SUPERUSER_ID, 'ir_attachment.location', context=context
- ) == FilestoreKind.swift
+ cr, SUPERUSER_ID, "ir_attachment.location", context=context
+ )
+ == FilestoreKind.swift
)
- if environment_name in ('prod', 'integration'):
+ if environment_name in ("prod", "integration"):
# Labs instances use swift or s3 by default, but we don't want
# to enforce it in case we want to test something with a different
# storage. At your own risks!
@@ -116,19 +118,19 @@ class CloudPlatform(osv.osv_abstract):
"automatically."
)
if use_swift:
- assert os.environ.get('SWIFT_AUTH_URL'), (
+ assert os.environ.get("SWIFT_AUTH_URL"), (
"SWIFT_AUTH_URL environment variable is required when "
"ir_attachment.location is 'swift'."
)
- assert os.environ.get('SWIFT_ACCOUNT'), (
+ assert os.environ.get("SWIFT_ACCOUNT"), (
"SWIFT_ACCOUNT environment variable is required when "
"ir_attachment.location is 'swift'."
)
- assert os.environ.get('SWIFT_PASSWORD'), (
+ assert os.environ.get("SWIFT_PASSWORD"), (
"SWIFT_PASSWORD environment variable is required when "
"ir_attachment.location is 'swift'."
)
- container_name = os.environ.get('SWIFT_WRITE_CONTAINER')
+ container_name = os.environ.get("SWIFT_WRITE_CONTAINER")
assert container_name, (
"SWIFT_WRITE_CONTAINER environment variable is required when "
"ir_attachment.location is 'swift'.\n"
@@ -139,9 +141,8 @@ class CloudPlatform(osv.osv_abstract):
"If you don't actually need a bucket, change the"
" 'ir_attachment.location' parameter."
)
- prod_container = bool(re.match(r'[a-z0-9-]+-odoo-prod',
- container_name))
- if environment_name == 'prod':
+ prod_container = bool(re.match(r"[a-z0-9-]+-odoo-prod", container_name))
+ if environment_name == "prod":
assert prod_container, (
"SWIFT_WRITE_CONTAINER should match '-odoo-prod', "
"we got: '%s'" % (container_name,)
@@ -153,21 +154,28 @@ class CloudPlatform(osv.osv_abstract):
"SWIFT_WRITE_CONTAINER should not match "
"'-odoo-prod', we got: '%s'" % (container_name,)
)
- elif environment_name == 'test':
+ elif environment_name == "test":
# store in DB so we don't have files local to the host
- assert params.get_param(cr, SUPERUSER_ID, 'ir_attachment.location',
- context=context) == 'db', (
+ assert (
+ params.get_param(
+ cr, SUPERUSER_ID, "ir_attachment.location", context=context
+ )
+ == "db"
+ ), (
"In test instances, files must be stored in the database with "
"'ir_attachment.location' set to 'db'. This is "
"automatically set by the function 'install_ovh()'."
)
def _check_s3(self, cr, uid, environment_name, context=None):
- params = self.pool.get('ir.config_parameter')
- use_s3 = params.get_param(
- cr, SUPERUSER_ID, 'ir_attachment.location', context=context
- ) == FilestoreKind.s3
- if environment_name in ('prod', 'integration'):
+ params = self.pool.get("ir.config_parameter")
+ use_s3 = (
+ params.get_param(
+ cr, SUPERUSER_ID, "ir_attachment.location", context=context
+ )
+ == FilestoreKind.s3
+ )
+ if environment_name in ("prod", "integration"):
# Labs instances use swift or s3 by default, but we don't want
# to enforce it in case we want to test something with a different
# storage. At your own risks!
@@ -178,15 +186,15 @@ class CloudPlatform(osv.osv_abstract):
"automatically."
)
if use_s3:
- assert os.environ.get('AWS_ACCESS_KEY_ID'), (
+ assert os.environ.get("AWS_ACCESS_KEY_ID"), (
"AWS_ACCESS_KEY_ID environment variable is required when "
"ir_attachment.location is 's3'."
)
- assert os.environ.get('AWS_SECRET_ACCESS_KEY'), (
+ assert os.environ.get("AWS_SECRET_ACCESS_KEY"), (
"AWS_SECRET_ACCESS_KEY environment variable is required when "
"ir_attachment.location is 's3'."
)
- bucket_name = os.environ.get('AWS_BUCKETNAME')
+ bucket_name = os.environ.get("AWS_BUCKETNAME")
assert bucket_name, (
"AWS_BUCKETNAME environment variable is required when "
"ir_attachment.location is 's3'.\n"
@@ -197,8 +205,8 @@ class CloudPlatform(osv.osv_abstract):
"If you don't actually need a bucket, change the"
" 'ir_attachment.location' parameter."
)
- prod_bucket = bool(re.match(r'[a-z-0-9]+-odoo-prod', bucket_name))
- if environment_name == 'prod':
+ prod_bucket = bool(re.match(r"[a-z-0-9]+-odoo-prod", bucket_name))
+ if environment_name == "prod":
assert prod_bucket, (
"AWS_BUCKETNAME should match '-odoo-prod', "
"we got: '%s'" % (bucket_name,)
@@ -211,46 +219,137 @@ class CloudPlatform(osv.osv_abstract):
"we got: '%s'" % (bucket_name,)
)
- elif environment_name == 'test':
+ elif environment_name == "test":
# store in DB so we don't have files local to the host
- assert params.get_param(cr, SUPERUSER_ID, 'ir_attachment.location',
- context=context) == 'db', (
+ assert (
+ params.get_param(
+ cr, SUPERUSER_ID, "ir_attachment.location", context=context
+ )
+ == "db"
+ ), (
"In test instances, files must be stored in the database with "
"'ir_attachment.location' set to 'db'. This is "
"automatically set by the function 'install_exoscale()'."
)
+ def _check_azure(self, cr, uid, environment_name, context=None):
+ params = self.pool.get("ir.config_parameter")
+ use_azure = (
+ params.get_param(
+ cr, SUPERUSER_ID, "ir_attachment.location", context=context
+ )
+ == FilestoreKind.azure
+ )
+ if environment_name in ("prod", "integration"):
+ # Labs instances use azure by default, but we don't want
+ # to enforce it in case we want to test something with a different
+ # storage. At your own risks!
+ assert use_azure, (
+ "azure must be used on production and integration instances. "
+ "It is activated by setting 'ir_attachment.location.' to 'azure'."
+ " The 'install()' function sets this option "
+ "automatically."
+ )
+ if use_azure:
+ key_sets = [
+ ["AZURE_STORAGE_USE_AAD", "AZURE_STORAGE_ACCOUNT_URL"],
+ ["AZURE_STORAGE_CONNECTION_STRING"],
+ [
+ "AZURE_STORAGE_ACCOUNT_NAME",
+ "AZURE_STORAGE_ACCOUNT_URL",
+ "AZURE_STORAGE_ACCOUNT_KEY",
+ ],
+ ]
+ is_valid = False
+ for key_set in key_sets:
+ if all([os.environ.get(key) for key in key_set]):
+ is_valid = True
+ break
+ assert is_valid, (
+ "When ir_attachment.location is set to 'azure', "
+ "at least one of the following enviromnent variable set "
+ "is required : {}".format(
+ " or ".join(
+ [" + ".join([key for key in key_set]) for key_set in key_sets]
+ )
+ )
+ )
+ storage_name = os.environ.get("AZURE_STORAGE_NAME", "")
+ if environment_name in ("prod", "integration", "labs"):
+ assert storage_name, (
+ "AZURE_STORAGE_NAME environment variable is required when "
+ "ir_attachment.location is 'azure'.\n"
+ "Normally, 'azure' is activated on labs, integration "
+ "and production, but should not be used in dev environment"
+ " (or using a dedicated dev bucket, never using the "
+ "integration/prod bucket).\n"
+ "If you don't actually need a bucket, change the"
+ " 'ir_attachment.location' parameter."
+ )
+ # A bucket name is defined under the following format
+ # ^[a-z]+\-[a-z]+\-\d+$
+ # Anything other than prod bucket must be suffixed with env name
+ #
+ # Use AZURE_STORAGE_NAME_UNSTRUCTURED to by-pass check
+ # on bucket name structure
+ if os.environ.get("AZURE_STORAGE_NAME_UNSTRUCTURED"):
+ return
+ prod_bucket = bool(re.match(r"^[a-z]+\-[a-z]+\-\d+$", storage_name))
+ if environment_name == "prod":
+ assert prod_bucket, (
+ "AZURE_STORAGE_NAME should match '^[a-z]+\\-[a-z]+\\-\\d+$', "
+ "we got: '%s'" % (storage_name,)
+ )
+ else:
+ # if we are using the prod bucket on another instance
+ # such as an integration, we must be sure to be in read only!
+ assert not prod_bucket, (
+ "AZURE_STORAGE_NAME should not match '^[a-z]+\\-[a-z]+\\-\\d+$', "
+ "we got: '%s'" % (storage_name,)
+ )
+
+ elif environment_name == "test":
+ # store in DB so we don't have files local to the host
+ assert (
+ params.get_param(
+ cr, SUPERUSER_ID, "ir_attachment.location", context=context
+ )
+ == "db"
+ ), (
+ "In test instances, files must be stored in the database with "
+ "'ir_attachment.location' set to 'db'. This is "
+ "automatically set by the function 'install()'."
+ )
+
def _check_redis(self, cr, uid, environment_name, context=None):
- if environment_name in ('prod', 'integration', 'labs', 'test'):
- assert is_true(os.environ.get('ODOO_SESSION_REDIS')), (
+ if environment_name in ("prod", "integration", "labs", "test"):
+ assert is_true(os.environ.get("ODOO_SESSION_REDIS")), (
"Redis must be activated on prod, integration, labs,"
" test instances. This is done by setting ODOO_SESSION_REDIS=1."
)
- assert (os.environ.get('ODOO_SESSION_REDIS_HOST') or
- os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST')), (
- "ODOO_SESSION_REDIS_HOST or ODOO_SESSION_REDIS_SENTINEL_HOST "
+ assert os.environ.get("ODOO_SESSION_REDIS_URL") or os.environ.get(
+ "ODOO_SESSION_REDIS_SENTINEL_URL"
+ ), (
+ "ODOO_SESSION_REDIS_URL or ODOO_SESSION_REDIS_SENTINEL_URL "
"environment variable is required to connect on Redis"
)
- assert os.environ.get('ODOO_SESSION_REDIS_PREFIX'), (
+ assert os.environ.get("ODOO_SESSION_REDIS_PREFIX"), (
"ODOO_SESSION_REDIS_PREFIX environment variable is required "
"to store sessions on Redis"
)
- prefix = os.environ['ODOO_SESSION_REDIS_PREFIX']
- assert re.match(r'^[a-z-0-9]+-odoo-[a-z-0-9]+$', prefix), (
+ prefix = os.environ["ODOO_SESSION_REDIS_PREFIX"]
+ assert re.match(r"^[a-z-0-9]+-odoo-[a-z-0-9]+$", prefix), (
"ODOO_SESSION_REDIS_PREFIX must match '-odoo-'"
", we got: '%s'" % (prefix,)
)
def check(self, cr, uid, context=None):
- if is_true(os.environ.get('ODOO_CLOUD_PLATFORM_UNSAFE')):
- _logger.warning(
- "cloud platform checks disabled, this is not safe"
- )
+ if is_true(os.environ.get("ODOO_CLOUD_PLATFORM_UNSAFE")):
+ _logger.warning("cloud platform checks disabled, this is not safe")
return
- params = self.pool.get('ir.config_parameter')
- kind = params.get_param(cr, SUPERUSER_ID,
- 'cloud.platform.kind', context=None)
+ params = self.pool.get("ir.config_parameter")
+ kind = params.get_param(cr, SUPERUSER_ID, "cloud.platform.kind", context=None)
if not kind:
_logger.warning(
"cloud platform not configured, you should "
@@ -258,12 +357,14 @@ class CloudPlatform(osv.osv_abstract):
)
return
environment_name = self._get_running_env()
- if kind == 'exoscale':
+ if kind == "exoscale":
self._check_s3(cr, uid, environment_name, context)
- elif kind == 'ovh':
+ elif kind == "ovh":
self._check_swift(cr, uid, environment_name, context)
+ elif kind == "azure":
+ self._check_azure(cr, uid, environment_name, context)
self._check_redis(cr, uid, environment_name, context)
def _register_hook(self, cr):
super(CloudPlatform, self)._register_hook(cr)
- self.pool.get('cloud.platform').check(cr, SUPERUSER_ID)
+ self.pool.get("cloud.platform").check(cr, SUPERUSER_ID)
diff --git a/cloud_platform_azure/README.md b/cloud_platform_azure/README.md
new file mode 100644
index 0000000..1f7bd5d
--- /dev/null
+++ b/cloud_platform_azure/README.md
@@ -0,0 +1,6 @@
+Cloud Platform Azure
+====================
+
+Install addons specific to the Azure setup.
+
+ * The object storage is Azure blob storage
diff --git a/cloud_platform_azure/__init__.py b/cloud_platform_azure/__init__.py
new file mode 100644
index 0000000..0650744
--- /dev/null
+++ b/cloud_platform_azure/__init__.py
@@ -0,0 +1 @@
+from . import models
diff --git a/cloud_platform_azure/__openerp__.py b/cloud_platform_azure/__openerp__.py
new file mode 100644
index 0000000..d6c2263
--- /dev/null
+++ b/cloud_platform_azure/__openerp__.py
@@ -0,0 +1,23 @@
+# Copyright 2017-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{
+ "name": "Cloud Platform Azure",
+ "summary": "Addons required for the Camptocamp Cloud Platform on Azure",
+ "version": "15.0.1.0.0",
+ "author": "Camptocamp,Odoo Community Association (OCA)",
+ "license": "AGPL-3",
+ "category": "Extra Tools",
+ "depends": [
+ "cloud_platform",
+ "attachment_azure",
+ ],
+ "excludes": [
+ "cloud_platform_ovh",
+ "cloud_platform_exoscale",
+ ],
+ "website": "https://www.camptocamp.com",
+ "data": [],
+ "installable": True,
+}
diff --git a/cloud_platform_azure/models/__init__.py b/cloud_platform_azure/models/__init__.py
new file mode 100644
index 0000000..5d08f36
--- /dev/null
+++ b/cloud_platform_azure/models/__init__.py
@@ -0,0 +1 @@
+from . import cloud_platform
diff --git a/cloud_platform_azure/models/cloud_platform.py b/cloud_platform_azure/models/cloud_platform.py
new file mode 100644
index 0000000..8fd0233
--- /dev/null
+++ b/cloud_platform_azure/models/cloud_platform.py
@@ -0,0 +1,37 @@
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import re
+import os
+
+from openerp.osv import osv
+from openerp.addons.cloud_platform.models.cloud_platform import FilestoreKind
+from openerp.addons.cloud_platform.models.cloud_platform import PlatformConfig
+
+
+class CloudPlatform(osv.osv):
+ _inherit = "cloud.platform"
+
+ def _filestore_kinds(self):
+ kinds = super(CloudPlatform, self)._filestore_kinds()
+ kinds.append("azure")
+ return kinds
+
+ def _platform_kinds(self):
+ kinds = super(CloudPlatform, self)._platform_kinds()
+ kinds.append("azure")
+ return kinds
+
+ def _config_by_server_env_for_azure(self):
+ fs_kinds = self._filestore_kinds()
+ configs = {
+ "prod": PlatformConfig(filestore=fs_kinds["azure"]),
+ "integration": PlatformConfig(filestore=fs_kinds["azure"]),
+ "labs": PlatformConfig(filestore=fs_kinds["azure"]),
+ "test": PlatformConfig(filestore=fs_kinds["db"]),
+ "dev": PlatformConfig(filestore=fs_kinds["db"]),
+ }
+ return configs
+
+ def install_azure(self, cr, uid, context=None):
+ self.install(cr, uid, "azure", context)
diff --git a/session_redis/http.py b/session_redis/http.py
index 7da42ee..6a77af1 100644
--- a/session_redis/http.py
+++ b/session_redis/http.py
@@ -24,45 +24,45 @@ except ImportError:
def is_true(strval):
- return bool(strtobool(strval or '0'.lower()))
+ return bool(strtobool(strval or "0".lower()))
-sentinel_host = os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST')
-sentinel_master_name = os.environ.get(
- 'ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME'
-)
+sentinel_host = os.environ.get("ODOO_SESSION_REDIS_SENTINEL_HOST")
+sentinel_master_name = os.environ.get("ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME")
if sentinel_host and not sentinel_master_name:
raise Exception(
"ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME must be defined "
"when using session_redis"
)
-sentinel_port = int(os.environ.get('ODOO_SESSION_REDIS_SENTINEL_PORT', 26379))
-host = os.environ.get('ODOO_SESSION_REDIS_HOST', 'localhost')
-port = int(os.environ.get('ODOO_SESSION_REDIS_PORT', 6379))
-prefix = os.environ.get('ODOO_SESSION_REDIS_PREFIX')
-url = os.environ.get('ODOO_SESSION_REDIS_URL')
-password = os.environ.get('ODOO_SESSION_REDIS_PASSWORD')
-expiration = os.environ.get('ODOO_SESSION_REDIS_EXPIRATION')
-anon_expiration = os.environ.get('ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS')
+sentinel_port = int(os.environ.get("ODOO_SESSION_REDIS_SENTINEL_PORT", 26379))
+host = os.environ.get("ODOO_SESSION_REDIS_URL", "localhost")
+port = int(os.environ.get("ODOO_SESSION_REDIS_PORT", 6379))
+prefix = os.environ.get("ODOO_SESSION_REDIS_PREFIX")
+url = os.environ.get("ODOO_SESSION_REDIS_URL")
+password = os.environ.get("ODOO_SESSION_REDIS_PASSWORD")
+expiration = os.environ.get("ODOO_SESSION_REDIS_EXPIRATION")
+anon_expiration = os.environ.get("ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS")
def session_store():
if sentinel_host:
- sentinel = Sentinel([(sentinel_host, sentinel_port)],
- password=password)
+ sentinel = Sentinel([(sentinel_host, sentinel_port)], password=password)
redis_client = sentinel.master_for(sentinel_master_name)
elif url:
redis_client = redis.from_url(url)
else:
redis_client = redis.Redis(host=host, port=port, password=password)
- return RedisSessionStore(redis=redis_client, prefix=prefix,
- expiration=expiration,
- anon_expiration=anon_expiration,
- session_class=Session)
+ return RedisSessionStore(
+ redis=redis_client,
+ prefix=prefix,
+ expiration=expiration,
+ anon_expiration=anon_expiration,
+ session_class=Session,
+ )
def session_gc(session_store):
- """ Do not garbage collect the sessions
+ """Do not garbage collect the sessions
Redis keys are automatically cleaned at the end of their
expiration.
@@ -79,18 +79,26 @@ def purge_fs_sessions(path):
pass
-if is_true(os.environ.get('ODOO_SESSION_REDIS')):
+if is_true(os.environ.get("ODOO_SESSION_REDIS")):
if sentinel_host:
- _logger.debug("HTTP sessions stored in Redis with prefix '%s'. "
- "Using Sentinel on %s:%s",
- sentinel_host, sentinel_port, prefix or '')
+ _logger.debug(
+ "HTTP sessions stored in Redis with prefix '%s'. "
+ "Using Sentinel on %s:%s",
+ sentinel_host,
+ sentinel_port,
+ prefix or "",
+ )
else:
- _logger.debug("HTTP sessions stored in Redis with prefix '%s' on "
- "%s:%s", host, port, prefix or '')
+ _logger.debug(
+ "HTTP sessions stored in Redis with prefix '%s' on " "%s:%s",
+ host,
+ port,
+ prefix or "",
+ )
store = session_store()
for handler in openerp.service.wsgi_server.module_handlers:
- if hasattr(handler, 'session_store'):
+ if hasattr(handler, "session_store"):
handler.session_store = store
http.session_gc = session_gc
# clean the existing sessions on the file system