From 2f8fcedd88ec0ff58a1e4f1a0c4d23e63c1a7e93 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Wed, 1 May 2019 16:02:41 +0200 Subject: [PATCH 1/3] Add method to force storage of special attachments to DB Some attachments (e.g. image_small, image_medium) are stored in DB instead of the object storage for faster access. In some situations, we may have pushed all these files on the Object Storage (migration from a filesystem to object storage) and want to bring back these attachments from the object storage to the database. This method is not called anywhere but can be called by RPC or scripts. --- .../models/ir_attachment.py | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/base_attachment_object_storage/models/ir_attachment.py b/base_attachment_object_storage/models/ir_attachment.py index 85296c8..30620c6 100644 --- a/base_attachment_object_storage/models/ir_attachment.py +++ b/base_attachment_object_storage/models/ir_attachment.py @@ -5,11 +5,14 @@ import base64 import inspect import logging import os +import time + import psycopg2 import odoo from contextlib import closing, contextmanager from odoo import api, exceptions, models, _ +from odoo.osv.expression import AND, normalize_domain _logger = logging.getLogger(__name__) @@ -67,6 +70,26 @@ class IrAttachment(models.Model): if update_module: self.env['ir.attachment'].sudo()._force_storage_to_object_storage() + @api.model + def _save_in_db_domain(self): + """Return a domain for attachments that must be forced to DB + + Read the docstring of ``_save_in_db_anyway`` for more details. + + The domain must be inline with the conditions in + ``_save_in_db_anyway``. + """ + return [ + '|', + # assets are stored in 'ir.ui.view' + ('res_model', '=', 'ir.ui.view'), + # Binary fields are stored with the name of the field in + # 'res_field' + # 'image' fields can be rather large and should usually + # not be requests in bulk in lists + ('res_field', 'in', self._local_fields) + ] + @api.multi def _save_in_db_anyway(self): """ Return whether an attachment must be stored in db @@ -83,8 +106,13 @@ class IrAttachment(models.Model): when assets are invalidated, they are deleted so we don't have an old database with attachments pointing to deleted assets. + The conditions must be inline with the domain in + ``_save_in_db_domain``. + """ self.ensure_one() + # Note: we cannot use _save_in_db_domain because we can be working + # with new records here. The conditions must stay inline though. # assets if self.res_model == 'ir.ui.view': # assets are stored in 'ir.ui.view' @@ -242,6 +270,65 @@ class IrAttachment(models.Model): return super(IrAttachment, self).force_storage() self._force_storage_to_object_storage() + @api.model + def force_storage_to_db_for_special_fields(self, new_cr=False): + """Migrate special attachments from Object Storage back to database + + The access to a file stored on the objects storage is slower + than a local disk or database access. For attachments like + image_small that are accessed in batch for kanban views, this + is too slow. We store this type of attachment in the database. + + This method can be used when migrating a filestore where all the files, + including the special files (assets, image_small, ...) have been pushed + to the Object Storage and we want to write them back in the database. + + It is not called anywhere, but can be called by RPC or scripts. + """ + storage = self._storage() + if storage not in self._get_stores(): + return + + domain = AND(( + normalize_domain( + [('store_fname', '=like', '{}://%'.format(storage))] + ), + normalize_domain(self._save_in_db_domain()) + )) + + with self.do_in_new_env(new_cr=new_cr) as new_env: + model_env = new_env['ir.attachment'].with_context( + prefetch_fields=False + ) + attachment_ids = model_env.search(domain).ids + if not attachment_ids: + return + total = len(attachment_ids) + start_time = time.time() + _logger.info('Moving %d attachments from %s to' + ' DB for fast access', total, storage) + current = 0 + for attachment_id in attachment_ids: + current += 1 + # if we browse attachments outside of the loop, the first + # access to 'datas' will compute all the 'datas' fields at + # once, which means reading hundreds or thousands of files at + # once, exhausting memory + attachment = model_env.browse(attachment_id) + # this write will read the datas from the Object Storage and + # write them back in the DB (the logic for location to write is + # in the 'datas' inverse computed field) + attachment.write({'datas': attachment.datas}) + # as the file will potentially be dropped on the bucket, + # we should commit the changes here + new_env.cr.commit() + if current % 100 == 0 or total - current == 0: + _logger.info( + 'attachment %s/%s after %.2fs', + current, total, + time.time() - start_time + ) + @api.model def _force_storage_to_object_storage(self, new_cr=False): _logger.info('migrating files to the object storage') From 19126ee4f0ab8d264cab58348861f41633569a92 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Tue, 4 Jun 2019 13:19:17 +0200 Subject: [PATCH 2/3] Update base_attachment_object_storage/models/ir_attachment.py --- base_attachment_object_storage/models/ir_attachment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_attachment_object_storage/models/ir_attachment.py b/base_attachment_object_storage/models/ir_attachment.py index 30620c6..c1988ae 100644 --- a/base_attachment_object_storage/models/ir_attachment.py +++ b/base_attachment_object_storage/models/ir_attachment.py @@ -319,7 +319,7 @@ class IrAttachment(models.Model): # write them back in the DB (the logic for location to write is # in the 'datas' inverse computed field) attachment.write({'datas': attachment.datas}) - # as the file will potentially be dropped on the bucket, + # as the file will potentially be deleted from the bucket, # we should commit the changes here new_env.cr.commit() if current % 100 == 0 or total - current == 0: From 493e49ba009ceb78e4991ac5bf359e598376d1d0 Mon Sep 17 00:00:00 2001 From: sebalix Date: Fri, 9 Aug 2019 16:05:06 +0200 Subject: [PATCH 3/3] attachment_s3: remove old migration script --- .../migrations/10.0.1.1.0/post-migration.py | 61 ------------------- 1 file changed, 61 deletions(-) delete mode 100644 attachment_s3/migrations/10.0.1.1.0/post-migration.py diff --git a/attachment_s3/migrations/10.0.1.1.0/post-migration.py b/attachment_s3/migrations/10.0.1.1.0/post-migration.py deleted file mode 100644 index a8ffd31..0000000 --- a/attachment_s3/migrations/10.0.1.1.0/post-migration.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2016-2018 Camptocamp SA -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - -import logging -import os - -from contextlib import closing - -import odoo - -_logger = logging.getLogger(__name__) - - -def migrate(cr, version): - if not version: - return - cr.execute(""" - SELECT value FROM ir_config_parameter - WHERE key = 'ir_attachment.location' - """) - row = cr.fetchone() - bucket = os.environ.get('AWS_BUCKETNAME') - - if row[0] == 's3' and bucket: - uid = odoo.SUPERUSER_ID - registry = odoo.modules.registry.RegistryManager.get(cr.dbname) - new_cr = registry.cursor() - with closing(new_cr): - with odoo.api.Environment.manage(): - env = odoo.api.Environment(new_cr, uid, {}) - store_local = env['ir.attachment'].search( - [('store_fname', '=like', 's3://%'), - '|', ('res_model', '=', 'ir.ui.view'), - ('res_field', 'in', ['image_small', - 'image_medium', - 'web_icon_data']) - ], - ) - - _logger.info( - 'Moving %d attachments from S3 to DB for fast access', - len(store_local) - ) - for attachment_id in store_local.ids: - # force re-storing the document, will move - # it from the object storage to the database - - # This is a trick to avoid having the 'datas' function - # fields computed for every attachment on each - # iteration of the loop. The former issue being that - # it reads the content of the file of ALL the - # attachments on each loop. - try: - env.clear() - attachment = env['ir.attachment'].browse(attachment_id) - _logger.info('Moving attachment %s (id: %s)', - attachment.name, attachment.id) - attachment.write({'datas': attachment.datas}) - new_cr.commit() - except: - new_cr.rollback()