Add method to force storage of special attachments to DB

Some attachments (e.g. image_small, image_medium) are stored in DB
instead of the object storage for faster access.

In some situations, we may have pushed all these files on the Object
Storage (migration from a filesystem to object storage) and want to
bring back these attachments from the object storage to the database.

This method is not called anywhere but can be called by RPC or scripts.

base_attachment_object_storage/models/ir_attachment.py

@@ -5,12 +5,15 @@ import base64
 import inspect
 import logging
 import os
+import time
+
 import psycopg2
 import odoo
 
 from contextlib import closing, contextmanager
 from odoo import api, exceptions, models, _
 from odoo.tools.mimetypes import guess_mimetype
+from odoo.osv.expression import AND, normalize_domain
 
 
 _logger = logging.getLogger(__name__)
@@ -65,6 +68,36 @@ class IrAttachment(models.Model):
         if update_module:
             self.env['ir.attachment'].sudo()._force_storage_to_object_storage()
 
+    @api.model
+    def _save_in_db_domain(self):
+        """Return a domain for attachments that must be forced to DB
+
+        Read the docstring of ``_save_in_db_anyway`` for more details.
+
+        The domain must be inline with the conditions in
+        ``_save_in_db_anyway``.
+        """
+        excluded_model_settings = self.env['ir.config_parameter'].sudo().\
+            get_param('excluded.models.storedb', default='')
+        excluded_model_for_db_store = excluded_model_settings.split(',')
+        mimetypes_settings = self.env['ir.config_parameter'].sudo().get_param(
+            'mimetypes.list.storedb', default='')
+        mimetypes_for_db_store = mimetypes_settings.split(',')
+        filesize = self.env['ir.config_parameter'].sudo().get_param(
+            'file.maxsize.storedb', default='0')
+        domain = [
+            '|',
+            # assets are stored in 'ir.ui.view'
+            ('res_model', '=', 'ir.ui.view'),
+            '&', '&',
+            ('file_size', '<', int(filesize)),
+            ('res_model', 'not in', excluded_model_for_db_store),
+        ]
+        domain += ['|'] * (len(mimetypes_for_db_store) - 1)
+        domain += [('mimetype', '=like', mimetype) for mimetype in
+                   mimetypes_for_db_store]
+        return domain
+
     def _save_in_db_anyway(self):
         """ Return whether an attachment must be stored in db
 
@@ -80,8 +113,13 @@ class IrAttachment(models.Model):
         when assets are invalidated, they are deleted so we don't have
         an old database with attachments pointing to deleted assets.
 
+        The conditions must be inline with the domain in
+        ``_save_in_db_domain``.
+
         """
         self.ensure_one()
+        # Note: we cannot use _save_in_db_domain because we can be working
+        # with new records here. The conditions must stay inline though.
         # assets
         if self.res_model == 'ir.ui.view':
             # assets are stored in 'ir.ui.view'
@@ -201,7 +239,7 @@ class IrAttachment(models.Model):
         """
         with api.Environment.manage():
             if new_cr:
-                registry = odoo.modules.registry.RegistryManager.get(
+                registry = odoo.modules.registry.Registry.new(
                     self.env.cr.dbname
                 )
                 with closing(registry.cursor()) as cr:
@@ -248,6 +286,65 @@ class IrAttachment(models.Model):
             return super().force_storage()
         self._force_storage_to_object_storage()
 
+    @api.model
+    def force_storage_to_db_for_special_fields(self, new_cr=False):
+        """Migrate special attachments from Object Storage back to database
+
+        The access to a file stored on the objects storage is slower
+        than a local disk or database access. For attachments like
+        image_small that are accessed in batch for kanban views, this
+        is too slow. We store this type of attachment in the database.
+
+        This method can be used when migrating a filestore where all the files,
+        including the special files (assets, image_small, ...) have been pushed
+        to the Object Storage and we want to write them back in the database.
+
+        It is not called anywhere, but can be called by RPC or scripts.
+        """
+        storage = self._storage()
+        if storage not in self._get_stores():
+            return
+
+        domain = AND((
+            normalize_domain(
+                [('store_fname', '=like', '{}://%'.format(storage))]
+            ),
+            normalize_domain(self._save_in_db_domain())
+        ))
+
+        with self.do_in_new_env(new_cr=new_cr) as new_env:
+            model_env = new_env['ir.attachment'].with_context(
+                prefetch_fields=False
+            )
+            attachment_ids = model_env.search(domain).ids
+            if not attachment_ids:
+                return
+            total = len(attachment_ids)
+            start_time = time.time()
+            _logger.info('Moving %d attachments from %s to'
+                         ' DB for fast access', total, storage)
+            current = 0
+            for attachment_id in attachment_ids:
+                current += 1
+                # if we browse attachments outside of the loop, the first
+                # access to 'datas' will compute all the 'datas' fields at
+                # once, which means reading hundreds or thousands of files at
+                # once, exhausting memory
+                attachment = model_env.browse(attachment_id)
+                # this write will read the datas from the Object Storage and
+                # write them back in the DB (the logic for location to write is
+                # in the 'datas' inverse computed field)
+                attachment.write({'datas': attachment.datas})
+                # as the file will potentially be dropped on the bucket,
+                # we should commit the changes here
+                new_env.cr.commit()
+                if current % 100 == 0 or total - current == 0:
+                    _logger.info(
+                        'attachment %s/%s after %.2fs',
+                        current, total,
+                        time.time() - start_time
+                    )
+
     @api.model
     def _force_storage_to_object_storage(self, new_cr=False):
         _logger.info('migrating files to the object storage')