diff --git a/base_attachment_object_storage/models/ir_attachment.py b/base_attachment_object_storage/models/ir_attachment.py index 64f800a..dbf6979 100644 --- a/base_attachment_object_storage/models/ir_attachment.py +++ b/base_attachment_object_storage/models/ir_attachment.py @@ -2,8 +2,8 @@ # Copyright 2017 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - import base64 +import inspect import logging import os import psycopg2 @@ -16,11 +16,57 @@ from odoo import api, exceptions, models, _ _logger = logging.getLogger(__name__) +def clean_fs(files): + _logger.info('cleaning old files from filestore') + for full_path in files: + if os.path.exists(full_path): + try: + os.unlink(full_path) + except OSError: + _logger.info( + "_file_delete could not unlink %s", + full_path, exc_info=True + ) + except IOError: + # Harmless and needed for race conditions + _logger.info( + "_file_delete could not unlink %s", + full_path, exc_info=True + ) + + class IrAttachment(models.Model): _inherit = 'ir.attachment' _local_fields = ('image_small', 'image_medium', 'web_icon_data') + @api.cr + def _register_hook(self): + super(IrAttachment, self)._register_hook() + # ignore if we are not using an object storage + if self._storage() not in self._get_stores(): + return + curframe = inspect.currentframe() + calframe = inspect.getouterframes(curframe, 2) + # the caller of _register_hook is 'load_modules' in + # odoo/modules/loading.py + load_modules_frame = calframe[1][0] + # 'update_module' is an argument that 'load_modules' receives with a + # True-ish value meaning that an install or upgrade of addon has been + # done during the initialization. We need to move the attachments that + # could have been created or updated in other addons before this addon + # was loaded + update_module = load_modules_frame.f_locals.get('update_module') + + # We need to call the migration on the loading of the model because + # when we are upgrading addons, some of them might add attachments. + # To be sure they are migrated to the storage we need to call the + # migration here. + # Typical example is images of ir.ui.menu which are updated in + # ir.attachment at every upgrade of the addons + if update_module: + self.env['ir.attachment'].sudo()._force_storage_to_object_storage() + @api.multi def _save_in_db_anyway(self): """ Return whether an attachment must be stored in db @@ -178,22 +224,7 @@ class IrAttachment(models.Model): # on assets 'mimetype': self.mimetype}) _logger.info('moved %s on the object storage', fname) - full_path = self._full_path(fname) - _logger.info('cleaning fs self') - if os.path.exists(full_path): - try: - os.unlink(full_path) - except OSError: - _logger.info( - "_file_delete could not unlink %s", - full_path, exc_info=True - ) - except IOError: - # Harmless and needed for race conditions - _logger.info( - "_file_delete could not unlink %s", - full_path, exc_info=True - ) + return self._full_path(fname) elif self.db_datas: _logger.info('moving on the object storage from database') self.write({'datas': self.datas}) @@ -212,6 +243,11 @@ class IrAttachment(models.Model): def _force_storage_to_object_storage(self, new_cr=False): _logger.info('migrating files to the object storage') storage = self._storage() + # The weird "res_field = False OR res_field != False" domain + # is required! It's because of an override of _search in ir.attachment + # which adds ('res_field', '=', False) when the domain does not + # contain 'res_field'. + # https://github.com/odoo/odoo/blob/9032617120138848c63b3cfa5d1913c5e5ad76db/odoo/addons/base/ir/ir_attachment.py#L344-L347 domain = ['!', ('store_fname', '=like', '{}://%'.format(storage)), '|', ('res_field', '=', False), @@ -223,11 +259,12 @@ class IrAttachment(models.Model): with self.do_in_new_env(new_cr=new_cr) as new_env: model_env = new_env['ir.attachment'] ids = model_env.search(domain).ids + files_to_clean = [] for attachment_id in ids: try: with new_env.cr.savepoint(): # check that no other transaction has - # locked the row, don't send a file to S3 + # locked the row, don't send a file to storage # in that case self.env.cr.execute("SELECT id " "FROM ir_attachment " @@ -243,11 +280,21 @@ class IrAttachment(models.Model): # ALL the attachments on each loop. new_env.clear() attachment = model_env.browse(attachment_id) - attachment._move_attachment_to_store() + path = attachment._move_attachment_to_store() + if path: + files_to_clean.append(path) except psycopg2.OperationalError: _logger.error('Could not migrate attachment %s to S3', attachment_id) + def clean(): + clean_fs(files_to_clean) + + # delete the files from the filesystem once we know the changes + # have been committed in ir.attachment + if files_to_clean: + new_env.cr.after('commit', clean) + def _get_stores(self): """ To get the list of stores activated in the system """ return []