Burtiraven2/session_redis_public
1
0
Fork 0
mirror of https://github.com/camptocamp/odoo-cloud-platform.git synced 2026-06-24 08:47:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Be more defensive against concurrent workers

Browse Source 
When migrating attachments to S3, several workers may try to migrate
the same attachments. Do not fail when both update the same attachment.
This commit is contained in:
Guewen Baconnier
2016-12-13 12:01:50 +01:00
parent 5f61ed8985
commit d138a1fd3d
1 changed files with 63 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files
attachment_s3/models/ir_attachment.py
+63 -41
View File
@@ -10,6 +10,8 @@ import xml.dom.minidom
from contextlib import closing, contextmanager from contextlib import closing, contextmanager
from functools import partial from functools import partial
import psycopg2
import openerp import openerp
from openerp import _, api, exceptions, models, SUPERUSER_ID from openerp import _, api, exceptions, models, SUPERUSER_ID
from ..s3uri import S3Uri from ..s3uri import S3Uri
@@ -170,6 +172,43 @@ class IrAttachment(models.Model):
else: else:
super(IrAttachment, self)._file_delete(fname) super(IrAttachment, self)._file_delete(fname)
@api.multi
def _move_attachment_to_s3(self):
self.ensure_one()
_logger.info('inspecting attachment %s (%d)',
self.name, self.id)
fname = self.store_fname
if fname:
# migrating from filesystem filestore
# or from the old 'store_fname' without the bucket name
_logger.info('moving %s on the object storage', fname)
self.write({'datas': self.datas,
# this is required otherwise the
# mimetype gets overriden with
# 'application/octet-stream'
# on assets
'mimetype': self.mimetype})
_logger.info('moved %s on the object storage', fname)
full_path = self._full_path(fname)
_logger.info('cleaning fs self')
if os.path.exists(full_path):
try:
os.unlink(full_path)
except OSError:
_logger.info(
"_file_delete could not unlink %s",
full_path, exc_info=True
)
except IOError:
# Harmless and needed for race conditions
_logger.info(
"_file_delete could not unlink %s",
full_path, exc_info=True
)
elif self.db_datas:
_logger.info('moving on the object storage from database')
self.write({'datas': self.datas})
@api.model @api.model
def _force_storage_s3(self, new_cr=False): def _force_storage_s3(self, new_cr=False):
if not self.env['res.users'].browse(self.env.uid)._is_admin(): if not self.env['res.users'].browse(self.env.uid)._is_admin():
@@ -190,48 +229,31 @@ class IrAttachment(models.Model):
# it causes serialization issues due to concurrent updates on # it causes serialization issues due to concurrent updates on
# attachments during the installation # attachments during the installation
with self.do_in_new_env(new_cr=new_cr) as new_env: with self.do_in_new_env(new_cr=new_cr) as new_env:
attachment_model = new_env['ir.attachment'] attachment_model_env = new_env['ir.attachment']
ids = attachment_model.search(domain).ids ids = attachment_model_env.search(domain).ids
for attachment_id in ids: for attachment_id in ids:
# This is a trick to avoid having the 'datas' function fields try:
# computed for every attachment on each iteration of the loop. with new_env.cr.savepoint():
# The former issue being that it reads the content of the file # check that no other transaction has
# of ALL the attachments on each loop. # locked the row, don't send a file to S3
new_env.clear() # in that case
attachment = attachment_model.browse(attachment_id) self.env.cr.execute("SELECT id "
_logger.info('inspecting attachment %s (%d)', "FROM ir_attachment "
attachment.name, attachment.id) "WHERE id = %s "
fname = attachment.store_fname "FOR UPDATE NOWAIT",
if fname: (attachment_id,),
# migrating from filesystem filestore log_exceptions=False)
# or from the old 'store_fname' without the bucket name
_logger.info('moving %s on the object storage', fname) # This is a trick to avoid having the 'datas' function
attachment.write({'datas': attachment.datas, # fields computed for every attachment on each
# this is required otherwise the # iteration of the loop. The former issue being that
# mimetype gets overriden with # it reads the content of the file of ALL the
# 'application/octet-stream' # attachments on each loop.
# on assets new_env.clear()
'mimetype': attachment.mimetype}) attachment = attachment_model_env.browse(attachment_id)
_logger.info('moved %s on the object storage', fname) attachment._move_attachment_to_s3()
full_path = attachment_model._full_path(fname) except psycopg2.OperationalError:
_logger.info('cleaning fs attachment') _logger.error('Could not migrate attachment %s to S3', id)
if os.path.exists(full_path):
try:
os.unlink(full_path)
except OSError:
_logger.info(
"_file_delete could not unlink %s",
full_path, exc_info=True
)
except IOError:
# Harmless and needed for race conditions
_logger.info(
"_file_delete could not unlink %s",
full_path, exc_info=True
)
elif attachment.db_datas:
_logger.info('moving on the object storage from database')
attachment.write({'datas': attachment.datas})
@contextmanager @contextmanager
def do_in_new_env(self, new_cr=False): def do_in_new_env(self, new_cr=False):