From 8109c8def4a754a54c365efb8622151b8d36eb56 Mon Sep 17 00:00:00 2001 From: Stephane Mangin Date: Wed, 13 Apr 2022 12:43:20 +0200 Subject: [PATCH 1/4] Object Storage - inactive mode --- README.md | 5 +++ base_attachment_object_storage/README.rst | 8 ++++- .../models/ir_attachment.py | 31 ++++++++++++++++++- 3 files changed, 42 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b5e11d3..e69c69b 100644 --- a/README.md +++ b/README.md @@ -162,3 +162,8 @@ environment. It will refuse to start if anything is badly configured. The checks can be bypassed with the environment variable `ODOO_CLOUD_PLATFORM_UNSAFE` set to `1`. + +### Attachment storage inactivation + +To prevent object storage to be accessed while failing for any kind of reason +set this environment variable `ATTACHMENT_STORAGE_INACTIVE` set to `1`. diff --git a/base_attachment_object_storage/README.rst b/base_attachment_object_storage/README.rst index 176459f..48dfdd3 100644 --- a/base_attachment_object_storage/README.rst +++ b/base_attachment_object_storage/README.rst @@ -1,7 +1,7 @@ Base class for attachments on external object store =================================================== -This is a base addon that regroup common code used by addons targeting specific object store +This is a base addon that regroup common code used by addons targeting specific object store Configuration ------------- @@ -38,3 +38,9 @@ Default configuration means: stored in database * application/javascript are stored in database whatever their size * text/css are stored in database whatever their size + +Inactivate attachment storage I/O +--------------------------------- + +Define a environment variable `ATTACHMENT_STORAGE_INACTIVE` set to `1` +This will prevent any kind of exceptions and read/write on storage attachments. diff --git a/base_attachment_object_storage/models/ir_attachment.py b/base_attachment_object_storage/models/ir_attachment.py index ea9283f..8813ea4 100644 --- a/base_attachment_object_storage/models/ir_attachment.py +++ b/base_attachment_object_storage/models/ir_attachment.py @@ -5,6 +5,7 @@ import inspect import logging import os import time +from distutils.util import strtobool import psycopg2 import odoo @@ -18,6 +19,10 @@ from odoo.tools.safe_eval import const_eval _logger = logging.getLogger(__name__) +def is_true(strval): + return bool(strtobool(strval or '0')) + + def clean_fs(files): _logger.info('cleaning old files from filestore') for full_path in files: @@ -40,6 +45,18 @@ def clean_fs(files): class IrAttachment(models.Model): _inherit = 'ir.attachment' + @staticmethod + def is_storage_inactive(storage=None, log=True): + msg = _("Storages are inactive (see environment configuration).") + if storage: + msg = _( + "Storage '%s' is inactive (see environment configuration)." + ) % (storage,) + is_inactive = is_true(os.environ.get("ATTACHMENT_STORAGE_INACTIVE")) + if is_inactive and log: + _logger.warning(msg) + return is_inactive + def _register_hook(self): super()._register_hook() location = self.env.context.get('storage_location') or self._storage() @@ -151,6 +168,8 @@ class IrAttachment(models.Model): ``_store_in_db_instead_of_object_storage_domain``. """ + if self.is_storage_inactive(): + return True storage_config = self._get_storage_force_db_config() for mimetype_key, limit in storage_config.items(): if mimetype.startswith(mimetype_key): @@ -190,8 +209,9 @@ class IrAttachment(models.Model): ) def _store_file_write(self, key, bin_data): + storage = self.storage() raise NotImplementedError( - 'No implementation for %s' % (self.storage(),) + 'No implementation for %s' % (storage,) ) def _store_file_delete(self, fname): @@ -229,6 +249,8 @@ class IrAttachment(models.Model): @api.model def _is_file_from_a_store(self, fname): for store_name in self._get_stores(): + if self.is_storage_inactive(store_name): + continue uri = '{}://'.format(store_name) if fname.startswith(uri): return True @@ -263,6 +285,9 @@ class IrAttachment(models.Model): self.ensure_one() _logger.info('inspecting attachment %s (%d)', self.name, self.id) fname = self.store_fname + storage = fname.partition('://')[0] + if self.is_storage_inactive(storage): + fname = False if fname: # migrating from filesystem filestore # or from the old 'store_fname' without the bucket name @@ -305,6 +330,8 @@ class IrAttachment(models.Model): It is not called anywhere, but can be called by RPC or scripts. """ storage = self._storage() + if self.is_storage_inactive(storage): + return if storage not in self._get_stores(): return @@ -358,6 +385,8 @@ class IrAttachment(models.Model): def _force_storage_to_object_storage(self, new_cr=False): _logger.info('migrating files to the object storage') storage = self.env.context.get('storage_location') or self._storage() + if self.is_storage_inactive(storage): + return # The weird "res_field = False OR res_field != False" domain # is required! It's because of an override of _search in ir.attachment # which adds ('res_field', '=', False) when the domain does not From e7c502bba814af9da8463013821f16ac7f91cadb Mon Sep 17 00:00:00 2001 From: Stephane Mangin Date: Mon, 9 May 2022 12:34:04 +0200 Subject: [PATCH 2/4] logging_json: fix W0622(redefined-builtin) --- logging_json/json_log.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/logging_json/json_log.py b/logging_json/json_log.py index 6d8e528..8215df4 100644 --- a/logging_json/json_log.py +++ b/logging_json/json_log.py @@ -35,9 +35,10 @@ class OdooJsonFormatter(jsonlogger.JsonFormatter): if is_true(os.environ.get('ODOO_LOGGING_JSON')): - format = ('%(asctime)s %(pid)s %(levelname)s' - '%(dbname)s %(name)s: %(message)s') - formatter = OdooJsonFormatter(format) + formatted_message = ( + '%(asctime)s %(pid)s %(levelname)s %(dbname)s %(name)s: %(message)s' + ) + formatter = OdooJsonFormatter(formatted_message) logging.getLogger().handlers[0].formatter = formatter From f124cf1380fccec380fc196d3d76d561dd524269 Mon Sep 17 00:00:00 2001 From: Stephane Mangin Date: Mon, 9 May 2022 12:50:49 +0200 Subject: [PATCH 3/4] Object storage inactivation: changes INACTIVE concept for DISABLE --- README.md | 4 ++-- base_attachment_object_storage/README.rst | 6 ++--- .../models/ir_attachment.py | 22 +++++++++---------- 3 files changed, 16 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index e69c69b..a3e6490 100644 --- a/README.md +++ b/README.md @@ -163,7 +163,7 @@ environment. It will refuse to start if anything is badly configured. The checks can be bypassed with the environment variable `ODOO_CLOUD_PLATFORM_UNSAFE` set to `1`. -### Attachment storage inactivation +### Attachment storage disability To prevent object storage to be accessed while failing for any kind of reason -set this environment variable `ATTACHMENT_STORAGE_INACTIVE` set to `1`. +set this environment variable `DISABLE_ATTACHMENT_STORAGE` set to `1`. diff --git a/base_attachment_object_storage/README.rst b/base_attachment_object_storage/README.rst index 48dfdd3..0ff25c9 100644 --- a/base_attachment_object_storage/README.rst +++ b/base_attachment_object_storage/README.rst @@ -39,8 +39,8 @@ Default configuration means: * application/javascript are stored in database whatever their size * text/css are stored in database whatever their size -Inactivate attachment storage I/O ---------------------------------- +Disable attachment storage I/O +------------------------------ -Define a environment variable `ATTACHMENT_STORAGE_INACTIVE` set to `1` +Define a environment variable `DISABLE_ATTACHMENT_STORAGE` set to `1` This will prevent any kind of exceptions and read/write on storage attachments. diff --git a/base_attachment_object_storage/models/ir_attachment.py b/base_attachment_object_storage/models/ir_attachment.py index 8813ea4..ed43c69 100644 --- a/base_attachment_object_storage/models/ir_attachment.py +++ b/base_attachment_object_storage/models/ir_attachment.py @@ -46,16 +46,16 @@ class IrAttachment(models.Model): _inherit = 'ir.attachment' @staticmethod - def is_storage_inactive(storage=None, log=True): - msg = _("Storages are inactive (see environment configuration).") + def is_storage_disabled(storage=None, log=True): + msg = _("Storages are disabled (see environment configuration).") if storage: msg = _( - "Storage '%s' is inactive (see environment configuration)." + "Storage '%s' is disabled (see environment configuration)." ) % (storage,) - is_inactive = is_true(os.environ.get("ATTACHMENT_STORAGE_INACTIVE")) - if is_inactive and log: + is_disabled = is_true(os.environ.get("DISABLE_ATTACHMENT_STORAGE")) + if is_disabled and log: _logger.warning(msg) - return is_inactive + return is_disabled def _register_hook(self): super()._register_hook() @@ -168,7 +168,7 @@ class IrAttachment(models.Model): ``_store_in_db_instead_of_object_storage_domain``. """ - if self.is_storage_inactive(): + if self.is_storage_disabled(): return True storage_config = self._get_storage_force_db_config() for mimetype_key, limit in storage_config.items(): @@ -249,7 +249,7 @@ class IrAttachment(models.Model): @api.model def _is_file_from_a_store(self, fname): for store_name in self._get_stores(): - if self.is_storage_inactive(store_name): + if self.is_storage_disabled(store_name): continue uri = '{}://'.format(store_name) if fname.startswith(uri): @@ -286,7 +286,7 @@ class IrAttachment(models.Model): _logger.info('inspecting attachment %s (%d)', self.name, self.id) fname = self.store_fname storage = fname.partition('://')[0] - if self.is_storage_inactive(storage): + if self.is_storage_disabled(storage): fname = False if fname: # migrating from filesystem filestore @@ -330,7 +330,7 @@ class IrAttachment(models.Model): It is not called anywhere, but can be called by RPC or scripts. """ storage = self._storage() - if self.is_storage_inactive(storage): + if self.is_storage_disabled(storage): return if storage not in self._get_stores(): return @@ -385,7 +385,7 @@ class IrAttachment(models.Model): def _force_storage_to_object_storage(self, new_cr=False): _logger.info('migrating files to the object storage') storage = self.env.context.get('storage_location') or self._storage() - if self.is_storage_inactive(storage): + if self.is_storage_disabled(storage): return # The weird "res_field = False OR res_field != False" domain # is required! It's because of an override of _search in ir.attachment From 4ad04f65bf9b463a189aa4be7f1ccff7d12900b9 Mon Sep 17 00:00:00 2001 From: Stephane Mangin Date: Mon, 9 May 2022 12:53:30 +0200 Subject: [PATCH 4/4] base_fileurl_field: fix W8106(method-required-super) --- base_fileurl_field/fields.py | 1 + 1 file changed, 1 insertion(+) diff --git a/base_fileurl_field/fields.py b/base_fileurl_field/fields.py index 0d2b4b6..133082a 100644 --- a/base_fileurl_field/fields.py +++ b/base_fileurl_field/fields.py @@ -35,6 +35,7 @@ class FileURL(fields.Binary): 'filename': '', # Field to use to store the filename on ir.attachment } + # pylint: disable=method-required-super def create(self, record_values): assert self.attachment if not record_values: