Burtiraven2/session_redis_public
1
0
Fork 0
mirror of https://github.com/camptocamp/odoo-cloud-platform.git synced 2026-06-23 18:04:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: remove after method (#393)

Browse Source 
* fix: azure reading in stream monkey patch documents
This commit is contained in:
Vincent Renaville
2022-11-11 15:17:03 +01:00
committed by GitHub
co-authored by GitHub
parent 988d4906bf
commit fc452c6a2a
5 changed files with 194 additions and 122 deletions
Download Patch File Download Diff File Expand all files Collapse all files
attachment_azure/models/__init__.py
+1
View File
@@ -2,3 +2,4 @@
# Copyright 2021 Open Source Integrators
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
from . import ir_attachment
from . import ir_binary
attachment_azure/models/ir_attachment.py
+9 -12
View File
@@ -117,11 +117,8 @@ class IrAttachment(models.Model):
https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names
"""
running_env = os.environ.get("RUNNING_ENV", "dev")
storage_name = os.environ.get('AZURE_STORAGE_NAME', r'{env}-{db}')
storage_name = storage_name.format(
env=running_env,
db=self.env.cr.dbname
)
storage_name = os.environ.get("AZURE_STORAGE_NAME", r"{env}-{db}")
storage_name = storage_name.format(env=running_env, db=self.env.cr.dbname)
# replace invalid characters by _
storage_name = re.sub(r"[\W_]+", "-", storage_name)
# lowercase, max 63 chars
@@ -136,7 +133,7 @@ class IrAttachment(models.Model):
except exceptions.UserError:
_logger.exception(
"error accessing to storage '%s' please check credentials ",
container_name
container_name,
)
return False
container_client = blob_service_client.get_container_client(container_name)
@@ -153,14 +150,14 @@ class IrAttachment(models.Model):
def _store_file_read(self, fname, bin_size=False):
if fname.startswith("azure://"):
key = fname.replace("azure://", "", 1).lower()
if '/' in key:
container_name, key = key.split('/', 1)
if "/" in key:
container_name, key = key.split("/", 1)
else:
container_name = None
container_client = self._get_azure_container(container_name)
# if container cannot be retrived, abort reading from azure storage
if not container_client:
return ''
return ""
try:
blob_client = container_client.get_blob_client(key)
read = blob_client.download_blob().readall()
@@ -200,13 +197,13 @@ class IrAttachment(models.Model):
def _store_file_delete(self, fname):
if fname.startswith("azure://"):
key = fname.replace("azure://", "", 1).lower()
if '/' in key:
container_name, key = key.split('/', 1)
if "/" in key:
container_name, key = key.split("/", 1)
else:
container_name = None
container_client = self._get_azure_container(container_name)
if not container_client:
return ''
return ""
# delete the file only if it is on the current configured container
# otherwise, we might delete files used on a different environment
try:
attachment_azure/models/ir_binary.py
+65
View File
@@ -0,0 +1,65 @@
# Copyright 2016-2019 Camptocamp SA
# Copyright 2021 Open Source Integrators
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
from odoo import models
from odoo.http import Stream
class IrBinary(models.AbstractModel):
_inherit = "ir.binary"
_description = "File streaming helper model for controllers"
def _azure_stream(self, attachment):
# we will create or own tream and return it
stream_data = self.env["ir.attachment"]._store_file_read(attachment.store_fname)
azurestream = Stream(
type="data",
data=stream_data,
path=None,
url=None,
mimetype=attachment.mimetype or None,
download_name=attachment.name,
size=len(stream_data),
etag=attachment.checksum,
)
return azurestream
def _record_to_stream(self, record, field_name):
"""
Low level method responsible for the actual conversion from a
model record to a stream. This method is an extensible hook for
other modules. It is not meant to be directly called from
outside or the ir.binary model.
:param record: the record where to load the data from.
:param str field_name: the binary field where to load the data
from.
:rtype: odoo.http.Stream
"""
if (
record._name == "ir.attachment"
and record.store_fname
and record.store_fname.startswith("azure://")
):
# we will create or own tream and return it
return self._azure_stream(record)
elif (
record._name == "documents.document"
and record.attachment_id
and record.attachment_id.store_fname
and record.attachment_id.store_fname.startswith("azure://")
):
return self._azure_stream(record.attachment_id)
else:
return super()._record_to_stream(record, field_name)
# This part is used if the customer install tne enterprise module documents
try:
from odoo.addons import documents
documents.models.ir_binary.IrBinary._record_to_stream = IrBinary._record_to_stream
except ImportError:
# document enterprise module if not installed, we just ignore
pass
base_attachment_object_storage/models/ir_attachment.py
+91 -83
View File
@@ -20,38 +20,36 @@ _logger = logging.getLogger(__name__)
def is_true(strval):
return bool(strtobool(strval or '0'))
return bool(strtobool(strval or "0"))
def clean_fs(files):
_logger.info('cleaning old files from filestore')
_logger.info("cleaning old files from filestore")
for full_path in files:
if os.path.exists(full_path):
try:
os.unlink(full_path)
except OSError:
_logger.info(
"_file_delete could not unlink %s",
full_path, exc_info=True
"_file_delete could not unlink %s", full_path, exc_info=True
)
except IOError:
# Harmless and needed for race conditions
_logger.info(
"_file_delete could not unlink %s",
full_path, exc_info=True
"_file_delete could not unlink %s", full_path, exc_info=True
)
class IrAttachment(models.Model):
_inherit = 'ir.attachment'
_inherit = "ir.attachment"
@staticmethod
def is_storage_disabled(storage=None, log=True):
msg = _("Storages are disabled (see environment configuration).")
if storage:
msg = _(
"Storage '%s' is disabled (see environment configuration)."
) % (storage,)
msg = _("Storage '%s' is disabled (see environment configuration).") % (
storage,
)
is_disabled = is_true(os.environ.get("DISABLE_ATTACHMENT_STORAGE"))
if is_disabled and log:
_logger.warning(msg)
@@ -59,7 +57,7 @@ class IrAttachment(models.Model):
def _register_hook(self):
super()._register_hook()
location = self.env.context.get('storage_location') or self._storage()
location = self.env.context.get("storage_location") or self._storage()
# ignore if we are not using an object storage
if location not in self._get_stores():
return
@@ -73,7 +71,7 @@ class IrAttachment(models.Model):
# done during the initialization. We need to move the attachments that
# could have been created or updated in other addons before this addon
# was loaded
update_module = load_modules_frame.f_locals.get('update_module')
update_module = load_modules_frame.f_locals.get("update_module")
# We need to call the migration on the loading of the model because
# when we are upgrading addons, some of them might add attachments.
@@ -82,15 +80,19 @@ class IrAttachment(models.Model):
# Typical example is images of ir.ui.menu which are updated in
# ir.attachment at every upgrade of the addons
if update_module:
self.env['ir.attachment'].sudo()._force_storage_to_object_storage()
self.env["ir.attachment"].sudo()._force_storage_to_object_storage()
@property
def _object_storage_default_force_db_config(self):
return {"image/": 51200, "application/javascript": 0, "text/css": 0}
def _get_storage_force_db_config(self):
param = self.env['ir.config_parameter'].sudo().get_param(
'ir_attachment.storage.force.database',
param = (
self.env["ir.config_parameter"]
.sudo()
.get_param(
"ir_attachment.storage.force.database",
)
)
storage_config = None
if param:
@@ -100,7 +102,8 @@ class IrAttachment(models.Model):
_logger.exception(
"Could not parse system parameter"
" 'ir_attachment.storage.force.database', reverting to the"
" default configuration.")
" default configuration."
)
if not storage_config:
storage_config = self._object_storage_default_force_db_config
@@ -180,17 +183,17 @@ class IrAttachment(models.Model):
return False
def _get_datas_related_values(self, data, mimetype):
storage = self.env.context.get('storage_location') or self._storage()
storage = self.env.context.get("storage_location") or self._storage()
if data and storage in self._get_stores():
if self._store_in_db_instead_of_object_storage(data, mimetype):
# compute the fields that depend on datas
bin_data = data
values = {
'file_size': len(bin_data),
'checksum': self._compute_checksum(bin_data),
'index_content': self._index(bin_data, mimetype),
'store_fname': False,
'db_datas': data,
"file_size": len(bin_data),
"checksum": self._compute_checksum(bin_data),
"index_content": self._index(bin_data, mimetype),
"store_fname": False,
"db_datas": data,
}
return values
return super()._get_datas_related_values(data, mimetype)
@@ -203,28 +206,22 @@ class IrAttachment(models.Model):
return super()._file_read(fname)
def _store_file_read(self, fname):
storage = fname.partition('://')[0]
raise NotImplementedError(
'No implementation for %s' % (storage,)
)
storage = fname.partition("://")[0]
raise NotImplementedError("No implementation for %s" % (storage,))
def _store_file_write(self, key, bin_data):
storage = self.storage()
raise NotImplementedError(
'No implementation for %s' % (storage,)
)
raise NotImplementedError("No implementation for %s" % (storage,))
def _store_file_delete(self, fname):
storage = fname.partition('://')[0]
raise NotImplementedError(
'No implementation for %s' % (storage,)
)
storage = fname.partition("://")[0]
raise NotImplementedError("No implementation for %s" % (storage,))
@api.model
def _file_write(self, bin_data, checksum):
location = self.env.context.get('storage_location') or self._storage()
location = self.env.context.get("storage_location") or self._storage()
if location in self._get_stores():
key = self.env.context.get('force_storage_key')
key = self.env.context.get("force_storage_key")
if not key:
key = self._compute_checksum(bin_data)
filename = self._store_file_write(key, bin_data)
@@ -238,8 +235,9 @@ class IrAttachment(models.Model):
cr = self.env.cr
# using SQL to include files hidden through unlink or due to record
# rules
cr.execute("SELECT COUNT(*) FROM ir_attachment "
"WHERE store_fname = %s", (fname,))
cr.execute(
"SELECT COUNT(*) FROM ir_attachment " "WHERE store_fname = %s", (fname,)
)
count = cr.fetchone()[0]
if not count:
self._store_file_delete(fname)
@@ -251,7 +249,7 @@ class IrAttachment(models.Model):
for store_name in self._get_stores():
if self.is_storage_disabled(store_name):
continue
uri = '{}://'.format(store_name)
uri = "{}://".format(store_name)
if fname.startswith(uri):
return True
return False
@@ -264,9 +262,7 @@ class IrAttachment(models.Model):
"""
with api.Environment.manage():
if new_cr:
registry = odoo.modules.registry.Registry.new(
self.env.cr.dbname
)
registry = odoo.modules.registry.Registry.new(self.env.cr.dbname)
with closing(registry.cursor()) as cr:
try:
yield self.env(cr=cr)
@@ -283,33 +279,38 @@ class IrAttachment(models.Model):
def _move_attachment_to_store(self):
self.ensure_one()
_logger.info('inspecting attachment %s (%d)', self.name, self.id)
_logger.info("inspecting attachment %s (%d)", self.name, self.id)
fname = self.store_fname
storage = fname.partition('://')[0]
storage = fname.partition("://")[0]
if self.is_storage_disabled(storage):
fname = False
if fname:
# migrating from filesystem filestore
# or from the old 'store_fname' without the bucket name
_logger.info('moving %s on the object storage', fname)
self.write({'datas': self.datas,
_logger.info("moving %s on the object storage", fname)
self.write(
{
"datas": self.datas,
# this is required otherwise the
# mimetype gets overriden with
# 'application/octet-stream'
# on assets
'mimetype': self.mimetype})
_logger.info('moved %s on the object storage', fname)
"mimetype": self.mimetype,
}
)
_logger.info("moved %s on the object storage", fname)
return self._full_path(fname)
elif self.db_datas:
_logger.info('moving on the object storage from database')
self.write({'datas': self.datas})
_logger.info("moving on the object storage from database")
self.write({"datas": self.datas})
@api.model
def force_storage(self):
if not self.env['res.users'].browse(self.env.uid)._is_admin():
if not self.env["res.users"].browse(self.env.uid)._is_admin():
raise exceptions.AccessError(
_('Only administrators can execute this action.'))
location = self.env.context.get('storage_location') or self._storage()
_("Only administrators can execute this action.")
)
location = self.env.context.get("storage_location") or self._storage()
if location not in self._get_stores():
return super().force_storage()
self._force_storage_to_object_storage()
@@ -335,30 +336,32 @@ class IrAttachment(models.Model):
if storage not in self._get_stores():
return
domain = AND((
domain = AND(
(
normalize_domain(
[('store_fname', '=like', '{}://%'.format(storage)),
[
("store_fname", "=like", "{}://%".format(storage)),
# for res_field, see comment in
# _force_storage_to_object_storage
'|',
('res_field', '=', False),
('res_field', '!=', False),
"|",
("res_field", "=", False),
("res_field", "!=", False),
]
),
normalize_domain(self._store_in_db_instead_of_object_storage_domain())
))
normalize_domain(self._store_in_db_instead_of_object_storage_domain()),
)
)
with self.do_in_new_env(new_cr=new_cr) as new_env:
model_env = new_env['ir.attachment'].with_context(
prefetch_fields=False
)
model_env = new_env["ir.attachment"].with_context(prefetch_fields=False)
attachment_ids = model_env.search(domain).ids
if not attachment_ids:
return
total = len(attachment_ids)
start_time = time.time()
_logger.info('Moving %d attachments from %s to'
' DB for fast access', total, storage)
_logger.info(
"Moving %d attachments from %s to" " DB for fast access", total, storage
)
current = 0
for attachment_id in attachment_ids:
current += 1
@@ -370,21 +373,22 @@ class IrAttachment(models.Model):
# this write will read the datas from the Object Storage and
# write them back in the DB (the logic for location to write is
# in the 'datas' inverse computed field)
attachment.write({'datas': attachment.datas})
attachment.write({"datas": attachment.datas})
# as the file will potentially be dropped on the bucket,
# we should commit the changes here
new_env.cr.commit()
if current % 100 == 0 or total - current == 0:
_logger.info(
'attachment %s/%s after %.2fs',
current, total,
time.time() - start_time
"attachment %s/%s after %.2fs",
current,
total,
time.time() - start_time,
)
@api.model
def _force_storage_to_object_storage(self, new_cr=False):
_logger.info('migrating files to the object storage')
storage = self.env.context.get('storage_location') or self._storage()
_logger.info("migrating files to the object storage")
storage = self.env.context.get("storage_location") or self._storage()
if self.is_storage_disabled(storage):
return
# The weird "res_field = False OR res_field != False" domain
@@ -392,16 +396,19 @@ class IrAttachment(models.Model):
# which adds ('res_field', '=', False) when the domain does not
# contain 'res_field'.
# https://github.com/odoo/odoo/blob/9032617120138848c63b3cfa5d1913c5e5ad76db/odoo/addons/base/ir/ir_attachment.py#L344-L347
domain = ['!', ('store_fname', '=like', '{}://%'.format(storage)),
'|',
('res_field', '=', False),
('res_field', '!=', False)]
domain = [
"!",
("store_fname", "=like", "{}://%".format(storage)),
"|",
("res_field", "=", False),
("res_field", "!=", False),
]
# We do a copy of the environment so we can workaround the cache issue
# below. We do not create a new cursor by default because it causes
# serialization issues due to concurrent updates on attachments during
# the installation
with self.do_in_new_env(new_cr=new_cr) as new_env:
model_env = new_env['ir.attachment']
model_env = new_env["ir.attachment"]
ids = model_env.search(domain).ids
files_to_clean = []
for attachment_id in ids:
@@ -410,12 +417,14 @@ class IrAttachment(models.Model):
# check that no other transaction has
# locked the row, don't send a file to storage
# in that case
self.env.cr.execute("SELECT id "
self.env.cr.execute(
"SELECT id "
"FROM ir_attachment "
"WHERE id = %s "
"FOR UPDATE NOWAIT",
(attachment_id,),
log_exceptions=False)
log_exceptions=False,
)
# This is a trick to avoid having the 'datas'
# function fields computed for every attachment on
@@ -428,16 +437,15 @@ class IrAttachment(models.Model):
if path:
files_to_clean.append(path)
except psycopg2.OperationalError:
_logger.error('Could not migrate attachment %s to S3',
attachment_id)
def clean():
clean_fs(files_to_clean)
_logger.error(
"Could not migrate attachment %s to S3", attachment_id
)
# delete the files from the filesystem once we know the changes
# have been committed in ir.attachment
if files_to_clean:
new_env.cr.after('commit', clean)
new_env.cr.commit()
clean_fs(files_to_clean)
def _get_stores(self):
"""To get the list of stores activated in the system"""
requirements.txt
+12 -11
View File
@@ -1,12 +1,13 @@
azure-storage-blob==12.8.1
azure-identity==1.6.0
boto3==1.9.102
redis==2.10.5
python-json-logger==0.1.5
statsd==3.2.1
python-swiftclient==3.9.0
python-keystoneclient==3.22.0
keystoneauth1==3.14.0
azure-storage-blob==12.14.1
azure-identity==1.12.0
boto3==1.26.7
redis==4.3.4
python-json-logger==2.0.4
statsd==4.0.1
python-swiftclient==4.1.0
python-keystoneclient==5.0.1
keystoneauth1==5.0.0
# error with 5.x (ConstructorError: could not determine a constructor for the tag '!record')
PyYAML==4.2b4
prometheus_client==0.11.0
PyYAML==6.0
prometheus_client==0.15.0
pyopenssl==22.1.0