diff --git a/attachment_azure/README.rst b/attachment_azure/README.rst new file mode 100644 index 0000000..b5c26d2 --- /dev/null +++ b/attachment_azure/README.rst @@ -0,0 +1,46 @@ +=========================================== +Attachments on Microsoft Azure Blob Storage +=========================================== + +This addon allows to store the attachments (documents and assets) on `Microsoft Azure +Blob Storage `_. + +Configuration +------------- + +Activate Azure Blob storage: + +* Create or set the system parameter with the key ``ir_attachment.location`` + and the value in the form ``azure``. + +Configure accesses with environment variables: + +* ``AZURE_STORAGE_CONNECTION_STRING`` or +* ``AZURE_STORAGE_ACCOUNT_NAME`` +* ``AZURE_STORAGE_ACCOUNT_URL`` +* ``AZURE_STORAGE_ACCOUNT_KEY`` + +One container will be created per database using the `RUNNING_ENV` environment variable +and the name of the database. By default, `RUNNING_ENV` is set to `dev`. + +The container name can be overridden with environment variable ``AZURE_STORAGE_NAME``. +The strings ``{db}`` and ``{env}`` can be used inside that variable and the values +will be replaced respectively by the database name and environment name. + +The container name will also be stored in the database for each attachment, +and will be used to access the right container in the storage. + +This addon must be added in the server wide addons with (``--load`` option): + +``--load=web,attachment_azure`` + +The System Parameter ``ir_attachment.storage.force.database`` can be customized to +force storage of files in the database. See the documentation of the module +``base_attachment_object_storage``. + +Limitations +----------- + +* You need to call ``env['ir.attachment'].force_storage()`` after + having changed the ``ir_attachment.location`` configuration in order to + migrate the existing attachments to Azure Blob Storage. diff --git a/attachment_azure/__init__.py b/attachment_azure/__init__.py new file mode 100644 index 0000000..49d7105 --- /dev/null +++ b/attachment_azure/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +from . import models diff --git a/attachment_azure/__openerp__.py b/attachment_azure/__openerp__.py new file mode 100644 index 0000000..7ada22d --- /dev/null +++ b/attachment_azure/__openerp__.py @@ -0,0 +1,19 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +{ + "name": "Attachments on Azure storage", + "summary": "Store assets and attachments on a Azure compatible object storage", + "version": "15.0.1.0.0", + "author": "Camptocamp, " + "Open Source Integrators, " + "Serpent Consulting Services, " + "Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Knowledge Management", + "depends": ["base_attachment_object_storage"], + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "installable": True, + "development_status": "Beta", + "maintainers": ["max3903"], +} diff --git a/attachment_azure/models/__init__.py b/attachment_azure/models/__init__.py new file mode 100644 index 0000000..cb9b196 --- /dev/null +++ b/attachment_azure/models/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +from . import ir_attachment diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py new file mode 100644 index 0000000..81415cd --- /dev/null +++ b/attachment_azure/models/ir_attachment.py @@ -0,0 +1,215 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +import io +import logging +import os +import re +from datetime import datetime, timedelta + +from openerp.tools.translate import _ +from openerp.osv import osv +from openerp.osv.orm import except_orm + +_logger = logging.getLogger(__name__) + +try: + from azure.storage.blob import ( + BlobServiceClient, + generate_account_sas, + ResourceTypes, + AccountSasPermissions, + ) + from azure.core.exceptions import ResourceExistsError, HttpResponseError +except ImportError: + _logger.debug("Cannot 'import azure-storage-blob'.") + +try: + from azure.identity import DefaultAzureCredential +except ImportError: + _logger.debug("Cannot 'import azure-identity'.") + + +class IrAttachment(osv.osv): + _inherit = "ir.attachment" + + def _get_stores(self): + l = ["azure"] + l += super(IrAttachment, self)._get_stores() + return l + + def _get_blob_service_client(self): + """Connect to Azure and return the blob service client + + The following environment variables must be set: + * ``AZURE_STORAGE_CONNECTION_STRING`` + or + * ``AZURE_STORAGE_ACCOUNT_NAME`` + * ``AZURE_STORAGE_ACCOUNT_URL`` + * ``AZURE_STORAGE_ACCOUNT_KEY`` + or if you want to use AAD (pod identity), set it to 1 or 0 + * ``AZURE_STORAGE_USE_AAD`` + + """ + connect_str = os.environ.get("AZURE_STORAGE_CONNECTION_STRING") + account_name = os.environ.get("AZURE_STORAGE_ACCOUNT_NAME") + account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL") + account_key = os.environ.get("AZURE_STORAGE_ACCOUNT_KEY") + account_use_aad = os.environ.get("AZURE_STORAGE_USE_AAD") + if not ( + connect_str + or (account_name and account_url and account_key) + or account_use_aad + ): + msg = _( + "If you want to read from the Azure container, you must provide the " + "following environment variables:\n" + "* AZURE_STORAGE_CONNECTION_STRING\n" + "or\n" + "* AZURE_STORAGE_ACCOUNT_NAME\n" + "* AZURE_STORAGE_ACCOUNT_URL\n" + "* AZURE_STORAGE_ACCOUNT_KEY\n" + "or\n" + "* AZURE_STORAGE_USE_AAD\n" + ) + raise osv.except_osv(_("UserError"), msg) + blob_service_client = None + if account_use_aad: + token_credential = DefaultAzureCredential() + blob_service_client = BlobServiceClient( + account_url=account_url, credential=token_credential + ) + elif connect_str: + try: + blob_service_client = BlobServiceClient.from_connection_string( + connect_str + ) + except HttpResponseError as error: + _logger.exception( + "Error during the connection to Azure container using the " + "connection string." + ) + raise osv.except_osv(_("UserError"), str(error)) + else: + try: + sas_token = generate_account_sas( + account_name=account_name, + account_key=account_key, + resource_types=ResourceTypes(container=True, object=True), + permission=AccountSasPermissions(read=True, write=True), + expiry=datetime.utcnow() + timedelta(hours=1), + ) + blob_service_client = BlobServiceClient( + account_url=account_url, + credential=sas_token, + ) + except HttpResponseError as error: + _logger.exception( + "Error during the connection to Azure container using the Shared " + "Access Signature (SAS)" + ) + raise osv.except_osv(_("UserError"), str(error)) + return blob_service_client + + def _get_container_name(self): + """ + Container naming rules: + https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names + """ + running_env = os.environ.get("RUNNING_ENV", "dev") + dbname = os.environ.get("DB_NAME", "odoodb") + storage_name = os.environ.get("AZURE_STORAGE_NAME", r"{env}-{db}") + storage_name = storage_name.format(env=running_env, db=dbname) + # replace invalid characters by _ + storage_name = re.sub(r"[\W_]+", "-", storage_name) + # lowercase, max 63 chars + return str.lower(storage_name)[:63] + + def _get_azure_container(self, container_name=None): + if not container_name: + container_name = self._get_container_name() + try: + blob_service_client = self._get_blob_service_client() + except Exception: + _logger.exception( + "error accessing to storage '%s' please check credentials ", + container_name, + ) + return False + container_client = blob_service_client.get_container_client(container_name) + if not container_client.exists(): + try: + # Create the container + container_client.create_container() + except HttpResponseError as error: + _logger.exception("Error during the creation of the Azure container") + raise osv.except_osv(_("UserError"), str(error)) + return container_client + + def _store_file_read(self, fname, bin_size=False): + if fname.startswith("azure://"): + key = fname.replace("azure://", "", 1).lower() + if "/" in key: + container_name, key = key.split("/", 1) + else: + container_name = None + container_client = self._get_azure_container(container_name) + # if container cannot be retrived, abort reading from azure storage + if not container_client: + return "" + try: + blob_client = container_client.get_blob_client(key) + read = blob_client.download_blob().readall() + except HttpResponseError: + read = "" + _logger.info("Attachment '%s' missing on object storage", fname) + return read + else: + return super(IrAttachment, self)._store_file_read(fname, bin_size) + + def _store_file_write(self, storage, key, bin_data): + if storage == "azure": + container_client = self._get_azure_container() + filename = "azure://%s/%s" % (container_client.container_name, key) + with io.BytesIO() as file: + blob_client = container_client.get_blob_client(key.lower()) + file.write(bin_data) + file.seek(0) + try: + blob_client.upload_blob(file, blob_type="BlockBlob") + except ResourceExistsError: + pass + except HttpResponseError as error: + # log verbose error from azure, return short message for user + _logger.exception("Error during storage of the file %s" % filename) + raise osv.except_osv( + _("UserError"), + _("The file could not be stored: %s") % str(error), + ) + else: + _super = super(IrAttachment, self) + filename = _super._store_file_write(key, bin_data) + return filename + + def _store_file_delete(self, fname): + if fname.startswith("azure://"): + key = fname.replace("azure://", "", 1).lower() + if "/" in key: + container_name, key = key.split("/", 1) + else: + container_name = None + container_client = self._get_azure_container(container_name) + if not container_client: + return "" + # delete the file only if it is on the current configured container + # otherwise, we might delete files used on a different environment + try: + blob_client = container_client.get_blob_client(key) + blob_client.delete_blob() + _logger.info("File %s deleted on the object storage" % (fname)) + except HttpResponseError: + # log verbose error from azure, return short message for + # user + _logger.exception("Error during deletion of the file %s" % fname) + else: + super(IrAttachment, self)._store_file_delete(fname) diff --git a/base_attachment_object_storage/models/ir_attachment.py b/base_attachment_object_storage/models/ir_attachment.py index 8ada85f..a857407 100644 --- a/base_attachment_object_storage/models/ir_attachment.py +++ b/base_attachment_object_storage/models/ir_attachment.py @@ -20,21 +20,19 @@ _logger = logging.getLogger(__name__) def clean_fs(files): - _logger.info('cleaning old files from filestore') + _logger.info("cleaning old files from filestore") for full_path in files: if os.path.exists(full_path): try: os.unlink(full_path) except OSError: _logger.info( - "_file_delete could not unlink %s", - full_path, exc_info=True + "_file_delete could not unlink %s", full_path, exc_info=True ) except IOError: # Harmless and needed for race conditions _logger.info( - "_file_delete could not unlink %s", - full_path, exc_info=True + "_file_delete could not unlink %s", full_path, exc_info=True ) @@ -52,34 +50,32 @@ def savepoint(cursor): class IrAttachment(osv.osv): - _inherit = 'ir.attachment' + _inherit = "ir.attachment" @staticmethod def _compute_checksum(bin_data): - """ compute the checksum for the given datas - :param bin_data : datas in its binary form + """compute the checksum for the given datas + :param bin_data : datas in its binary form """ # an empty file has a checksum too (for caching) - return hashlib.sha1(bin_data or '').hexdigest() + return hashlib.sha1(bin_data or "").hexdigest() def _is_user_admin(self, cr, uid): if uid == SUPERUSER_ID: return True else: - return self.pool.get('res.users').has_group( - cr, uid, 'base.group_erp_manager' + return self.pool.get("res.users").has_group( + cr, uid, "base.group_erp_manager" ) def _storage(self, cr, uid, context=None): - return self.pool['ir.config_parameter'].get_param( - cr, SUPERUSER_ID, 'ir_attachment.location', 'file' + return self.pool["ir.config_parameter"].get_param( + cr, SUPERUSER_ID, "ir_attachment.location", "file" ) def _full_path(self, cr, uid, location, path): # Hack to allow filestore migration from local filesystem to any remote - return super(IrAttachment, self)._full_path( - cr, uid, 'file://filestore', path - ) + return super(IrAttachment, self)._full_path(cr, uid, "file://filestore", path) def _register_hook(self, cr): super(IrAttachment, self)._register_hook(cr) @@ -101,7 +97,7 @@ class IrAttachment(osv.osv): # done during the initialization. We need to move the attachments that # could have been created or updated in other addons before this addon # was loaded - update_module = load_modules_frame.f_locals.get('update_module') + update_module = load_modules_frame.f_locals.get("update_module") # We need to call the migration on the loading of the model because # when we are upgrading addons, some of them might add attachments. @@ -110,12 +106,12 @@ class IrAttachment(osv.osv): # Typical example is images of ir.ui.menu which are updated in # ir.attachment at every upgrade of the addons if update_module: - self.pool.get('ir.attachment')._force_storage_to_object_storage( + self.pool.get("ir.attachment")._force_storage_to_object_storage( cr, SUPERUSER_ID ) def _save_in_db_anyway(self, cr, uid, ids, context=None): - """ Return whether an attachment must be stored in db + """Return whether an attachment must be stored in db When we are using an Object Store. This is sometimes required because the object storage is slower than the database/filesystem. @@ -130,12 +126,11 @@ class IrAttachment(osv.osv): an old database with attachments pointing to deleted assets. """ - assert (isinstance(ids, int) or - len(ids) == 1), 'Expecting only one record' + assert isinstance(ids, int) or len(ids) == 1, "Expecting only one record" rec = self.browse(cr, uid, ids, context=context) # assets - if rec.res_model == 'ir.ui.view': + if rec.res_model == "ir.ui.view": # assets are stored in 'ir.ui.view' return True @@ -146,58 +141,50 @@ class IrAttachment(osv.osv): # we keep them in the database instead of the object storage location = self._storage(cr, uid) for attach in self.browse(cr, uid, id, context): - if (location in self._get_stores() and - self._save_in_db_anyway(cr, uid, [id], context)): + if location in self._get_stores() and self._save_in_db_anyway( + cr, uid, [id], context + ): # compute the fields that depend on datas - bin_data = value and value.decode('base64') or '' + bin_data = value and value.decode("base64") or "" vals = { - 'file_size': len(bin_data), - 'checksum': self._compute_checksum(bin_data), - 'db_datas': value, + "file_size": len(bin_data), + "checksum": self._compute_checksum(bin_data), + "db_datas": value, # we seriously don't need index content on those fields - 'index_content': False, - 'store_fname': False, + "index_content": False, + "store_fname": False, } fname = attach.store_fname # write as superuser, as user probably does not # have write access - super(IrAttachment, self).write( - cr, SUPERUSER_ID, id, vals, context - ) + super(IrAttachment, self).write(cr, SUPERUSER_ID, id, vals, context) if fname: self._file_delete(cr, uid, fname) continue - self._data_set(cr, uid, id, 'datas', value, None, context) + self._data_set(cr, uid, id, "datas", value, None, context) def _store_file_read(self, fname, bin_size=False): - storage = fname.partition('://')[0] - raise NotImplementedError( - 'No implementation for %s' % (storage,) - ) + storage = fname.partition("://")[0] + raise NotImplementedError("No implementation for %s" % (storage,)) def _store_file_write(self, storage, key, bin_data): - raise NotImplementedError( - 'No implementation for %s' % (storage,) - ) + raise NotImplementedError("No implementation for %s" % (storage,)) def _store_file_delete(self, fname): - storage = fname.partition('://')[0] - raise NotImplementedError( - 'No implementation for %s' % (storage,) - ) + storage = fname.partition("://")[0] + raise NotImplementedError("No implementation for %s" % (storage,)) def _file_read(self, cr, uid, location, fname, bin_size=False): if self._is_file_from_a_store(fname): return self._store_file_read(fname, bin_size=bin_size) else: _super = super(IrAttachment, self) - return _super._file_read(cr, uid, location, - fname, bin_size=bin_size) + return _super._file_read(cr, uid, location, fname, bin_size=bin_size) def _file_write(self, cr, uid, location, value): storage = self._storage(cr, uid) if storage in self._get_stores(): - bin_data = value.decode('base64') + bin_data = value.decode("base64") key = self._compute_checksum(bin_data) filename = self._store_file_write(storage, key, bin_data) else: @@ -209,8 +196,9 @@ class IrAttachment(osv.osv): if self._is_file_from_a_store(fname): # using SQL to include files hidden through unlink or due to record # rules - cr.execute("SELECT COUNT(*) FROM ir_attachment " - "WHERE store_fname = %s", (fname,)) + cr.execute( + "SELECT COUNT(*) FROM ir_attachment " "WHERE store_fname = %s", (fname,) + ) count = cr.fetchone()[0] if int(count) == 1: self._store_file_delete(fname) @@ -219,33 +207,31 @@ class IrAttachment(osv.osv): def _is_file_from_a_store(self, fname): for store_name in self._get_stores(): - uri = '{}://'.format(store_name) + uri = "{}://".format(store_name) if fname.startswith(uri): return True return False def _move_attachment_to_store(self, cr, uid, ids, context=None): - assert (isinstance(ids, int) or - len(ids) == 1), 'Expecting only one record' + assert isinstance(ids, int) or len(ids) == 1, "Expecting only one record" rec = self.browse(cr, uid, ids, context) - _logger.info('inspecting attachment %s (%d)', rec.name, rec.id) + _logger.info("inspecting attachment %s (%d)", rec.name, rec.id) fname = rec.store_fname if fname: # migrating from filesystem filestore # or from the old 'store_fname' without the bucket name - _logger.info('moving %s on the object storage', fname) - self.write(cr, uid, ids, {'datas': rec.datas}, context) - _logger.info('moved %s on the object storage', fname) + _logger.info("moving %s on the object storage", fname) + self.write(cr, uid, ids, {"datas": rec.datas}, context) + _logger.info("moved %s on the object storage", fname) return self._full_path(cr, uid, None, fname) elif rec.db_datas: - _logger.info('moving on the object storage from database') - self.write(cr, uid, ids, {'datas': rec.datas}, context) + _logger.info("moving on the object storage from database") + self.write(cr, uid, ids, {"datas": rec.datas}, context) def force_storage(self, cr, uid, context=None): if not self._is_user_admin(cr, uid): raise except_orm( - _('Error'), - _('Only administrators can execute this action.') + _("Error"), _("Only administrators can execute this action.") ) storage = self._storage(cr, uid) if storage not in self._get_stores(): @@ -253,10 +239,10 @@ class IrAttachment(osv.osv): self._force_storage_to_object_storage(cr, uid, context) def _force_storage_to_object_storage(self, cr, uid, context=None): - _logger.info('migrating files to the object storage') + _logger.info("migrating files to the object storage") storage = self._storage(cr, uid) - domain = [('store_fname', 'not like', '{}://%'.format(storage))] + domain = [("store_fname", "not like", "{}://%".format(storage))] ids = self.search(cr, uid, domain, context=context) files_to_clean = [] @@ -273,7 +259,7 @@ class IrAttachment(osv.osv): "WHERE id = %s " "FOR UPDATE NOWAIT", (attachment_id,), - log_exceptions=False + log_exceptions=False, ) path = self._move_attachment_to_store( @@ -282,8 +268,9 @@ class IrAttachment(osv.osv): if path: files_to_clean.append(path) except psycopg2.OperationalError: - _logger.error('Could not migrate attachment %s to %s' % - (attachment_id, storage)) + _logger.error( + "Could not migrate attachment %s to %s" % (attachment_id, storage) + ) def clean(): clean_fs(files_to_clean) @@ -291,8 +278,8 @@ class IrAttachment(osv.osv): # delete the files from the filesystem once we know the changes # have been committed in ir.attachment if files_to_clean: - cr.after('commit', clean) + cr.commit() def _get_stores(self): - """ To get the list of stores activated in the system """ + """To get the list of stores activated in the system""" return [] diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py index 06de6d3..c92200b 100644 --- a/cloud_platform/models/cloud_platform.py +++ b/cloud_platform/models/cloud_platform.py @@ -19,93 +19,95 @@ _logger = logging.getLogger(__name__) def is_true(strval): - return bool(strtobool(strval or '0'.lower())) + return bool(strtobool(strval or "0".lower())) -PlatformConfig = namedtuple( - 'PlatformConfig', - 'filestore' -) +PlatformConfig = namedtuple("PlatformConfig", "filestore") class FilestoreKind(object): - db = 'db' - s3 = 's3' # or compatible s3 object storage - swift = 'swift' - file = 'file' + db = "db" + s3 = "s3" # or compatible s3 object storage + swift = "swift" + file = "file" + azure = "azure" class CloudPlatform(osv.osv_abstract): - _name = 'cloud.platform' + _name = "cloud.platform" def _platform_kinds(self): # XXX for backward compatibility, we need this one here, move # it in cloud_platform_exoscale in V11 - return ['exoscale'] + return ["exoscale"] + + def _filestore_kinds(self): + # XXX for backward compatibility, we need this one here, move + # it in cloud_platform_exoscale in V11 + return ["exoscale"] # XXX for backward compatibility, we need this one here, move # it in cloud_platform_exoscale in V11 def _config_by_server_env_for_exoscale(self): configs = { - 'prod': PlatformConfig(filestore=FilestoreKind.s3), - 'integration': PlatformConfig(filestore=FilestoreKind.s3), - 'test': PlatformConfig(filestore=FilestoreKind.db), - 'dev': PlatformConfig(filestore=FilestoreKind.db), + "prod": PlatformConfig(filestore=FilestoreKind.s3), + "integration": PlatformConfig(filestore=FilestoreKind.s3), + "test": PlatformConfig(filestore=FilestoreKind.db), + "dev": PlatformConfig(filestore=FilestoreKind.db), } return configs def _config_by_server_env(self, platform_kind, environment): configs_getter = getattr( - self, - '_config_by_server_env_for_%s' % platform_kind, - None + self, "_config_by_server_env_for_%s" % platform_kind, None ) configs = configs_getter() if configs_getter else {} return configs.get(environment) or FilestoreKind.db def _get_running_env(self): - environment_name = config['running_env'] - if environment_name.startswith('labs'): + environment_name = config["running_env"] + if environment_name.startswith("labs"): # We allow to have environments such as 'labs-logistics' # or 'labs-finance', in order to have the matching ribbon. - environment_name = 'labs' + environment_name = "labs" return environment_name # Due to the addition of the ovh cloud platform # This will be moved to cloud_platform_exoscale on v11 def install_exoscale(self, cr, uid, context=None): - self.install(cr, uid, 'exoscale', context) + self.install(cr, uid, "exoscale", context) def install(self, cr, uid, platform_kind, context=None): assert platform_kind in self._platform_kinds() - params = self.pool.get('ir.config_parameter') + params = self.pool.get("ir.config_parameter") params.set_param( - cr, SUPERUSER_ID, - 'cloud.platform.kind', platform_kind, - context=context + cr, SUPERUSER_ID, "cloud.platform.kind", platform_kind, context=context ) environment_name = self._get_running_env() configs = self._config_by_server_env(platform_kind, environment_name) params.set_param( - cr, SUPERUSER_ID, - 'ir_attachment.location', configs.filestore, - context=context + cr, + SUPERUSER_ID, + "ir_attachment.location", + configs.filestore, + context=context, ) self.check(cr, uid, context) if configs.filestore in [FilestoreKind.swift, FilestoreKind.s3]: - self.pool.get('ir.attachment').force_storage( + self.pool.get("ir.attachment").force_storage( cr, SUPERUSER_ID, context=context ) - _logger.info('cloud platform configured for {}'.format(platform_kind)) + _logger.info("cloud platform configured for {}".format(platform_kind)) def _check_swift(self, cr, uid, environment_name, context=None): - params = self.pool.get('ir.config_parameter') + params = self.pool.get("ir.config_parameter") use_swift = ( params.get_param( - cr, SUPERUSER_ID, 'ir_attachment.location', context=context - ) == FilestoreKind.swift + cr, SUPERUSER_ID, "ir_attachment.location", context=context + ) + == FilestoreKind.swift ) - if environment_name in ('prod', 'integration'): + if environment_name in ("prod", "integration"): # Labs instances use swift or s3 by default, but we don't want # to enforce it in case we want to test something with a different # storage. At your own risks! @@ -116,19 +118,19 @@ class CloudPlatform(osv.osv_abstract): "automatically." ) if use_swift: - assert os.environ.get('SWIFT_AUTH_URL'), ( + assert os.environ.get("SWIFT_AUTH_URL"), ( "SWIFT_AUTH_URL environment variable is required when " "ir_attachment.location is 'swift'." ) - assert os.environ.get('SWIFT_ACCOUNT'), ( + assert os.environ.get("SWIFT_ACCOUNT"), ( "SWIFT_ACCOUNT environment variable is required when " "ir_attachment.location is 'swift'." ) - assert os.environ.get('SWIFT_PASSWORD'), ( + assert os.environ.get("SWIFT_PASSWORD"), ( "SWIFT_PASSWORD environment variable is required when " "ir_attachment.location is 'swift'." ) - container_name = os.environ.get('SWIFT_WRITE_CONTAINER') + container_name = os.environ.get("SWIFT_WRITE_CONTAINER") assert container_name, ( "SWIFT_WRITE_CONTAINER environment variable is required when " "ir_attachment.location is 'swift'.\n" @@ -139,9 +141,8 @@ class CloudPlatform(osv.osv_abstract): "If you don't actually need a bucket, change the" " 'ir_attachment.location' parameter." ) - prod_container = bool(re.match(r'[a-z0-9-]+-odoo-prod', - container_name)) - if environment_name == 'prod': + prod_container = bool(re.match(r"[a-z0-9-]+-odoo-prod", container_name)) + if environment_name == "prod": assert prod_container, ( "SWIFT_WRITE_CONTAINER should match '-odoo-prod', " "we got: '%s'" % (container_name,) @@ -153,21 +154,28 @@ class CloudPlatform(osv.osv_abstract): "SWIFT_WRITE_CONTAINER should not match " "'-odoo-prod', we got: '%s'" % (container_name,) ) - elif environment_name == 'test': + elif environment_name == "test": # store in DB so we don't have files local to the host - assert params.get_param(cr, SUPERUSER_ID, 'ir_attachment.location', - context=context) == 'db', ( + assert ( + params.get_param( + cr, SUPERUSER_ID, "ir_attachment.location", context=context + ) + == "db" + ), ( "In test instances, files must be stored in the database with " "'ir_attachment.location' set to 'db'. This is " "automatically set by the function 'install_ovh()'." ) def _check_s3(self, cr, uid, environment_name, context=None): - params = self.pool.get('ir.config_parameter') - use_s3 = params.get_param( - cr, SUPERUSER_ID, 'ir_attachment.location', context=context - ) == FilestoreKind.s3 - if environment_name in ('prod', 'integration'): + params = self.pool.get("ir.config_parameter") + use_s3 = ( + params.get_param( + cr, SUPERUSER_ID, "ir_attachment.location", context=context + ) + == FilestoreKind.s3 + ) + if environment_name in ("prod", "integration"): # Labs instances use swift or s3 by default, but we don't want # to enforce it in case we want to test something with a different # storage. At your own risks! @@ -178,15 +186,15 @@ class CloudPlatform(osv.osv_abstract): "automatically." ) if use_s3: - assert os.environ.get('AWS_ACCESS_KEY_ID'), ( + assert os.environ.get("AWS_ACCESS_KEY_ID"), ( "AWS_ACCESS_KEY_ID environment variable is required when " "ir_attachment.location is 's3'." ) - assert os.environ.get('AWS_SECRET_ACCESS_KEY'), ( + assert os.environ.get("AWS_SECRET_ACCESS_KEY"), ( "AWS_SECRET_ACCESS_KEY environment variable is required when " "ir_attachment.location is 's3'." ) - bucket_name = os.environ.get('AWS_BUCKETNAME') + bucket_name = os.environ.get("AWS_BUCKETNAME") assert bucket_name, ( "AWS_BUCKETNAME environment variable is required when " "ir_attachment.location is 's3'.\n" @@ -197,8 +205,8 @@ class CloudPlatform(osv.osv_abstract): "If you don't actually need a bucket, change the" " 'ir_attachment.location' parameter." ) - prod_bucket = bool(re.match(r'[a-z-0-9]+-odoo-prod', bucket_name)) - if environment_name == 'prod': + prod_bucket = bool(re.match(r"[a-z-0-9]+-odoo-prod", bucket_name)) + if environment_name == "prod": assert prod_bucket, ( "AWS_BUCKETNAME should match '-odoo-prod', " "we got: '%s'" % (bucket_name,) @@ -211,46 +219,137 @@ class CloudPlatform(osv.osv_abstract): "we got: '%s'" % (bucket_name,) ) - elif environment_name == 'test': + elif environment_name == "test": # store in DB so we don't have files local to the host - assert params.get_param(cr, SUPERUSER_ID, 'ir_attachment.location', - context=context) == 'db', ( + assert ( + params.get_param( + cr, SUPERUSER_ID, "ir_attachment.location", context=context + ) + == "db" + ), ( "In test instances, files must be stored in the database with " "'ir_attachment.location' set to 'db'. This is " "automatically set by the function 'install_exoscale()'." ) + def _check_azure(self, cr, uid, environment_name, context=None): + params = self.pool.get("ir.config_parameter") + use_azure = ( + params.get_param( + cr, SUPERUSER_ID, "ir_attachment.location", context=context + ) + == FilestoreKind.azure + ) + if environment_name in ("prod", "integration"): + # Labs instances use azure by default, but we don't want + # to enforce it in case we want to test something with a different + # storage. At your own risks! + assert use_azure, ( + "azure must be used on production and integration instances. " + "It is activated by setting 'ir_attachment.location.' to 'azure'." + " The 'install()' function sets this option " + "automatically." + ) + if use_azure: + key_sets = [ + ["AZURE_STORAGE_USE_AAD", "AZURE_STORAGE_ACCOUNT_URL"], + ["AZURE_STORAGE_CONNECTION_STRING"], + [ + "AZURE_STORAGE_ACCOUNT_NAME", + "AZURE_STORAGE_ACCOUNT_URL", + "AZURE_STORAGE_ACCOUNT_KEY", + ], + ] + is_valid = False + for key_set in key_sets: + if all([os.environ.get(key) for key in key_set]): + is_valid = True + break + assert is_valid, ( + "When ir_attachment.location is set to 'azure', " + "at least one of the following enviromnent variable set " + "is required : {}".format( + " or ".join( + [" + ".join([key for key in key_set]) for key_set in key_sets] + ) + ) + ) + storage_name = os.environ.get("AZURE_STORAGE_NAME", "") + if environment_name in ("prod", "integration", "labs"): + assert storage_name, ( + "AZURE_STORAGE_NAME environment variable is required when " + "ir_attachment.location is 'azure'.\n" + "Normally, 'azure' is activated on labs, integration " + "and production, but should not be used in dev environment" + " (or using a dedicated dev bucket, never using the " + "integration/prod bucket).\n" + "If you don't actually need a bucket, change the" + " 'ir_attachment.location' parameter." + ) + # A bucket name is defined under the following format + # ^[a-z]+\-[a-z]+\-\d+$ + # Anything other than prod bucket must be suffixed with env name + # + # Use AZURE_STORAGE_NAME_UNSTRUCTURED to by-pass check + # on bucket name structure + if os.environ.get("AZURE_STORAGE_NAME_UNSTRUCTURED"): + return + prod_bucket = bool(re.match(r"^[a-z]+\-[a-z]+\-\d+$", storage_name)) + if environment_name == "prod": + assert prod_bucket, ( + "AZURE_STORAGE_NAME should match '^[a-z]+\\-[a-z]+\\-\\d+$', " + "we got: '%s'" % (storage_name,) + ) + else: + # if we are using the prod bucket on another instance + # such as an integration, we must be sure to be in read only! + assert not prod_bucket, ( + "AZURE_STORAGE_NAME should not match '^[a-z]+\\-[a-z]+\\-\\d+$', " + "we got: '%s'" % (storage_name,) + ) + + elif environment_name == "test": + # store in DB so we don't have files local to the host + assert ( + params.get_param( + cr, SUPERUSER_ID, "ir_attachment.location", context=context + ) + == "db" + ), ( + "In test instances, files must be stored in the database with " + "'ir_attachment.location' set to 'db'. This is " + "automatically set by the function 'install()'." + ) + def _check_redis(self, cr, uid, environment_name, context=None): - if environment_name in ('prod', 'integration', 'labs', 'test'): - assert is_true(os.environ.get('ODOO_SESSION_REDIS')), ( + if environment_name in ("prod", "integration", "labs", "test"): + assert is_true(os.environ.get("ODOO_SESSION_REDIS")), ( "Redis must be activated on prod, integration, labs," " test instances. This is done by setting ODOO_SESSION_REDIS=1." ) - assert (os.environ.get('ODOO_SESSION_REDIS_HOST') or - os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST')), ( - "ODOO_SESSION_REDIS_HOST or ODOO_SESSION_REDIS_SENTINEL_HOST " + assert os.environ.get("ODOO_SESSION_REDIS_URL") or os.environ.get( + "ODOO_SESSION_REDIS_SENTINEL_URL" + ), ( + "ODOO_SESSION_REDIS_URL or ODOO_SESSION_REDIS_SENTINEL_URL " "environment variable is required to connect on Redis" ) - assert os.environ.get('ODOO_SESSION_REDIS_PREFIX'), ( + assert os.environ.get("ODOO_SESSION_REDIS_PREFIX"), ( "ODOO_SESSION_REDIS_PREFIX environment variable is required " "to store sessions on Redis" ) - prefix = os.environ['ODOO_SESSION_REDIS_PREFIX'] - assert re.match(r'^[a-z-0-9]+-odoo-[a-z-0-9]+$', prefix), ( + prefix = os.environ["ODOO_SESSION_REDIS_PREFIX"] + assert re.match(r"^[a-z-0-9]+-odoo-[a-z-0-9]+$", prefix), ( "ODOO_SESSION_REDIS_PREFIX must match '-odoo-'" ", we got: '%s'" % (prefix,) ) def check(self, cr, uid, context=None): - if is_true(os.environ.get('ODOO_CLOUD_PLATFORM_UNSAFE')): - _logger.warning( - "cloud platform checks disabled, this is not safe" - ) + if is_true(os.environ.get("ODOO_CLOUD_PLATFORM_UNSAFE")): + _logger.warning("cloud platform checks disabled, this is not safe") return - params = self.pool.get('ir.config_parameter') - kind = params.get_param(cr, SUPERUSER_ID, - 'cloud.platform.kind', context=None) + params = self.pool.get("ir.config_parameter") + kind = params.get_param(cr, SUPERUSER_ID, "cloud.platform.kind", context=None) if not kind: _logger.warning( "cloud platform not configured, you should " @@ -258,12 +357,14 @@ class CloudPlatform(osv.osv_abstract): ) return environment_name = self._get_running_env() - if kind == 'exoscale': + if kind == "exoscale": self._check_s3(cr, uid, environment_name, context) - elif kind == 'ovh': + elif kind == "ovh": self._check_swift(cr, uid, environment_name, context) + elif kind == "azure": + self._check_azure(cr, uid, environment_name, context) self._check_redis(cr, uid, environment_name, context) def _register_hook(self, cr): super(CloudPlatform, self)._register_hook(cr) - self.pool.get('cloud.platform').check(cr, SUPERUSER_ID) + self.pool.get("cloud.platform").check(cr, SUPERUSER_ID) diff --git a/cloud_platform_azure/README.md b/cloud_platform_azure/README.md new file mode 100644 index 0000000..1f7bd5d --- /dev/null +++ b/cloud_platform_azure/README.md @@ -0,0 +1,6 @@ +Cloud Platform Azure +==================== + +Install addons specific to the Azure setup. + + * The object storage is Azure blob storage diff --git a/cloud_platform_azure/__init__.py b/cloud_platform_azure/__init__.py new file mode 100644 index 0000000..0650744 --- /dev/null +++ b/cloud_platform_azure/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/cloud_platform_azure/__openerp__.py b/cloud_platform_azure/__openerp__.py new file mode 100644 index 0000000..d6c2263 --- /dev/null +++ b/cloud_platform_azure/__openerp__.py @@ -0,0 +1,23 @@ +# Copyright 2017-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{ + "name": "Cloud Platform Azure", + "summary": "Addons required for the Camptocamp Cloud Platform on Azure", + "version": "15.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Extra Tools", + "depends": [ + "cloud_platform", + "attachment_azure", + ], + "excludes": [ + "cloud_platform_ovh", + "cloud_platform_exoscale", + ], + "website": "https://www.camptocamp.com", + "data": [], + "installable": True, +} diff --git a/cloud_platform_azure/models/__init__.py b/cloud_platform_azure/models/__init__.py new file mode 100644 index 0000000..5d08f36 --- /dev/null +++ b/cloud_platform_azure/models/__init__.py @@ -0,0 +1 @@ +from . import cloud_platform diff --git a/cloud_platform_azure/models/cloud_platform.py b/cloud_platform_azure/models/cloud_platform.py new file mode 100644 index 0000000..8fd0233 --- /dev/null +++ b/cloud_platform_azure/models/cloud_platform.py @@ -0,0 +1,37 @@ +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import re +import os + +from openerp.osv import osv +from openerp.addons.cloud_platform.models.cloud_platform import FilestoreKind +from openerp.addons.cloud_platform.models.cloud_platform import PlatformConfig + + +class CloudPlatform(osv.osv): + _inherit = "cloud.platform" + + def _filestore_kinds(self): + kinds = super(CloudPlatform, self)._filestore_kinds() + kinds.append("azure") + return kinds + + def _platform_kinds(self): + kinds = super(CloudPlatform, self)._platform_kinds() + kinds.append("azure") + return kinds + + def _config_by_server_env_for_azure(self): + fs_kinds = self._filestore_kinds() + configs = { + "prod": PlatformConfig(filestore=fs_kinds["azure"]), + "integration": PlatformConfig(filestore=fs_kinds["azure"]), + "labs": PlatformConfig(filestore=fs_kinds["azure"]), + "test": PlatformConfig(filestore=fs_kinds["db"]), + "dev": PlatformConfig(filestore=fs_kinds["db"]), + } + return configs + + def install_azure(self, cr, uid, context=None): + self.install(cr, uid, "azure", context) diff --git a/session_redis/http.py b/session_redis/http.py index 7da42ee..6a77af1 100644 --- a/session_redis/http.py +++ b/session_redis/http.py @@ -24,45 +24,45 @@ except ImportError: def is_true(strval): - return bool(strtobool(strval or '0'.lower())) + return bool(strtobool(strval or "0".lower())) -sentinel_host = os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST') -sentinel_master_name = os.environ.get( - 'ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME' -) +sentinel_host = os.environ.get("ODOO_SESSION_REDIS_SENTINEL_HOST") +sentinel_master_name = os.environ.get("ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME") if sentinel_host and not sentinel_master_name: raise Exception( "ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME must be defined " "when using session_redis" ) -sentinel_port = int(os.environ.get('ODOO_SESSION_REDIS_SENTINEL_PORT', 26379)) -host = os.environ.get('ODOO_SESSION_REDIS_HOST', 'localhost') -port = int(os.environ.get('ODOO_SESSION_REDIS_PORT', 6379)) -prefix = os.environ.get('ODOO_SESSION_REDIS_PREFIX') -url = os.environ.get('ODOO_SESSION_REDIS_URL') -password = os.environ.get('ODOO_SESSION_REDIS_PASSWORD') -expiration = os.environ.get('ODOO_SESSION_REDIS_EXPIRATION') -anon_expiration = os.environ.get('ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS') +sentinel_port = int(os.environ.get("ODOO_SESSION_REDIS_SENTINEL_PORT", 26379)) +host = os.environ.get("ODOO_SESSION_REDIS_URL", "localhost") +port = int(os.environ.get("ODOO_SESSION_REDIS_PORT", 6379)) +prefix = os.environ.get("ODOO_SESSION_REDIS_PREFIX") +url = os.environ.get("ODOO_SESSION_REDIS_URL") +password = os.environ.get("ODOO_SESSION_REDIS_PASSWORD") +expiration = os.environ.get("ODOO_SESSION_REDIS_EXPIRATION") +anon_expiration = os.environ.get("ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS") def session_store(): if sentinel_host: - sentinel = Sentinel([(sentinel_host, sentinel_port)], - password=password) + sentinel = Sentinel([(sentinel_host, sentinel_port)], password=password) redis_client = sentinel.master_for(sentinel_master_name) elif url: redis_client = redis.from_url(url) else: redis_client = redis.Redis(host=host, port=port, password=password) - return RedisSessionStore(redis=redis_client, prefix=prefix, - expiration=expiration, - anon_expiration=anon_expiration, - session_class=Session) + return RedisSessionStore( + redis=redis_client, + prefix=prefix, + expiration=expiration, + anon_expiration=anon_expiration, + session_class=Session, + ) def session_gc(session_store): - """ Do not garbage collect the sessions + """Do not garbage collect the sessions Redis keys are automatically cleaned at the end of their expiration. @@ -79,18 +79,26 @@ def purge_fs_sessions(path): pass -if is_true(os.environ.get('ODOO_SESSION_REDIS')): +if is_true(os.environ.get("ODOO_SESSION_REDIS")): if sentinel_host: - _logger.debug("HTTP sessions stored in Redis with prefix '%s'. " - "Using Sentinel on %s:%s", - sentinel_host, sentinel_port, prefix or '') + _logger.debug( + "HTTP sessions stored in Redis with prefix '%s'. " + "Using Sentinel on %s:%s", + sentinel_host, + sentinel_port, + prefix or "", + ) else: - _logger.debug("HTTP sessions stored in Redis with prefix '%s' on " - "%s:%s", host, port, prefix or '') + _logger.debug( + "HTTP sessions stored in Redis with prefix '%s' on " "%s:%s", + host, + port, + prefix or "", + ) store = session_store() for handler in openerp.service.wsgi_server.module_handlers: - if hasattr(handler, 'session_store'): + if hasattr(handler, "session_store"): handler.session_store = store http.session_gc = session_gc # clean the existing sessions on the file system