Burtiraven2/session_redis_public
1
0
Fork 0
mirror of https://github.com/camptocamp/odoo-cloud-platform.git synced 2026-06-24 02:08:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Change CI to GitHub actions

Browse Source 
Use copier template from oca/oca-addons-repo-template

Target Python3.8

Apply linting

Fix a missing call to super
Ensure all modules have a 13.0.x.x.x version
This commit is contained in:
Yannick Payot
2023-05-11 11:17:54 +02:00
parent 0000eab313
commit 71da584087
80 changed files with 1596 additions and 879 deletions
Download Patch File Download Diff File Expand all files Collapse all files
base_attachment_object_storage/__manifest__.py
+13 -12
View File
@@ -2,15 +2,16 @@
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
{'name': 'Base Attachment Object Store',
'summary': 'Base module for the implementation of external object store.',
'version': '13.0.1.1.0',
'author': 'Camptocamp,Odoo Community Association (OCA)',
'license': 'AGPL-3',
'category': 'Knowledge Management',
'depends': ['base'],
'website': 'http://www.camptocamp.com',
'data': ['data/res_config_settings_data.xml'],
'installable': True,
'auto_install': True,
}
{
"name": "Base Attachment Object Store",
"summary": "Base module for the implementation of external object store.",
"version": "13.0.1.1.0",
"author": "Camptocamp,Odoo Community Association (OCA)",
"license": "AGPL-3",
"category": "Knowledge Management",
"depends": ["base"],
"website": "https://github.com/camptocamp/odoo-cloud-platform",
"data": ["data/res_config_settings_data.xml"],
"installable": True,
"auto_install": True,
}
base_attachment_object_storage/data/res_config_settings_data.xml
+4 -2
View File
@@ -1,9 +1,11 @@
<?xml version='1.0' encoding='utf-8'?>
<?xml version='1.0' encoding='utf-8' ?>
<odoo noupdate="1">
<record id="ir_attachment_storage_force_database" model="ir.config_parameter">
<field name="key">ir_attachment.storage.force.database</field>
<field name="value">{"image/": 51200, "application/javascript": 0, "text/css": 0}</field>
<field
name="value"
>{"image/": 51200, "application/javascript": 0, "text/css": 0}</field>
</record>
</odoo>
base_attachment_object_storage/models/ir_attachment.py
+102 -95
View File
@@ -6,44 +6,41 @@ import inspect
import logging
import os
import time
from contextlib import closing, contextmanager
import psycopg2
import odoo
from contextlib import closing, contextmanager
from odoo import api, exceptions, models, _
import odoo
from odoo import _, api, exceptions, models
from odoo.osv.expression import AND, OR, normalize_domain
from odoo.tools.safe_eval import const_eval
_logger = logging.getLogger(__name__)
def clean_fs(files):
_logger.info('cleaning old files from filestore')
_logger.info("cleaning old files from filestore")
for full_path in files:
if os.path.exists(full_path):
try:
os.unlink(full_path)
except OSError:
_logger.info(
"_file_delete could not unlink %s",
full_path, exc_info=True
"_file_delete could not unlink %s", full_path, exc_info=True
)
except IOError:
# Harmless and needed for race conditions
_logger.info(
"_file_delete could not unlink %s",
full_path, exc_info=True
"_file_delete could not unlink %s", full_path, exc_info=True
)
class IrAttachment(models.Model):
_inherit = 'ir.attachment'
_inherit = "ir.attachment"
def _register_hook(self):
super()._register_hook()
location = self.env.context.get('storage_location') or self._storage()
location = self.env.context.get("storage_location") or self._storage()
# ignore if we are not using an object storage
if location not in self._get_stores():
return
@@ -57,7 +54,7 @@ class IrAttachment(models.Model):
# done during the initialization. We need to move the attachments that
# could have been created or updated in other addons before this addon
# was loaded
update_module = load_modules_frame.f_locals.get('update_module')
update_module = load_modules_frame.f_locals.get("update_module")
# We need to call the migration on the loading of the model because
# when we are upgrading addons, some of them might add attachments.
@@ -66,15 +63,17 @@ class IrAttachment(models.Model):
# Typical example is images of ir.ui.menu which are updated in
# ir.attachment at every upgrade of the addons
if update_module:
self.env['ir.attachment'].sudo()._force_storage_to_object_storage()
self.env["ir.attachment"].sudo()._force_storage_to_object_storage()
@property
def _object_storage_default_force_db_config(self):
return {"image/": 51200, "application/javascript": 0, "text/css": 0}
def _get_storage_force_db_config(self):
param = self.env['ir.config_parameter'].sudo().get_param(
'ir_attachment.storage.force.database',
param = (
self.env["ir.config_parameter"]
.sudo()
.get_param("ir_attachment.storage.force.database",)
)
storage_config = None
if param:
@@ -84,7 +83,8 @@ class IrAttachment(models.Model):
_logger.exception(
"Could not parse system parameter"
" 'ir_attachment.storage.force.database', reverting to the"
" default configuration.")
" default configuration."
)
if not storage_config:
storage_config = self._object_storage_default_force_db_config
@@ -157,22 +157,22 @@ class IrAttachment(models.Model):
if mimetype.startswith(mimetype_key):
if not limit:
return True
bin_data = base64.b64decode(data) if data else b''
bin_data = base64.b64decode(data) if data else b""
return len(bin_data) <= limit
return False
def _get_datas_related_values(self, data, mimetype):
storage = self.env.context.get('storage_location') or self._storage()
storage = self.env.context.get("storage_location") or self._storage()
if data and storage in self._get_stores():
if self._store_in_db_instead_of_object_storage(data, mimetype):
# compute the fields that depend on datas
bin_data = base64.b64decode(data) if data else b''
bin_data = base64.b64decode(data) if data else b""
values = {
'file_size': len(bin_data),
'checksum': self._compute_checksum(bin_data),
'index_content': self._index(bin_data, mimetype),
'store_fname': False,
'db_datas': data,
"file_size": len(bin_data),
"checksum": self._compute_checksum(bin_data),
"index_content": self._index(bin_data, mimetype),
"store_fname": False,
"db_datas": data,
}
return values
return super()._get_datas_related_values(data, mimetype)
@@ -185,28 +185,22 @@ class IrAttachment(models.Model):
return super()._file_read(fname, bin_size=bin_size)
def _store_file_read(self, fname, bin_size=False):
storage = fname.partition('://')[0]
raise NotImplementedError(
'No implementation for %s' % (storage,)
)
storage = fname.partition("://")[0]
raise NotImplementedError("No implementation for %s" % (storage,))
def _store_file_write(self, key, bin_data):
raise NotImplementedError(
'No implementation for %s' % (self.storage(),)
)
raise NotImplementedError("No implementation for %s" % (self.storage(),))
def _store_file_delete(self, fname):
storage = fname.partition('://')[0]
raise NotImplementedError(
'No implementation for %s' % (storage,)
)
storage = fname.partition("://")[0]
raise NotImplementedError("No implementation for %s" % (storage,))
@api.model
def _file_write(self, value, checksum):
location = self.env.context.get('storage_location') or self._storage()
location = self.env.context.get("storage_location") or self._storage()
if location in self._get_stores():
bin_data = base64.b64decode(value)
key = self.env.context.get('force_storage_key')
key = self.env.context.get("force_storage_key")
if not key:
key = self._compute_checksum(bin_data)
filename = self._store_file_write(key, bin_data)
@@ -220,8 +214,9 @@ class IrAttachment(models.Model):
cr = self.env.cr
# using SQL to include files hidden through unlink or due to record
# rules
cr.execute("SELECT COUNT(*) FROM ir_attachment "
"WHERE store_fname = %s", (fname,))
cr.execute(
"SELECT COUNT(*) FROM ir_attachment " "WHERE store_fname = %s", (fname,)
)
count = cr.fetchone()[0]
if not count:
self._store_file_delete(fname)
@@ -231,7 +226,7 @@ class IrAttachment(models.Model):
@api.model
def _is_file_from_a_store(self, fname):
for store_name in self._get_stores():
uri = '{}://'.format(store_name)
uri = "{}://".format(store_name)
if fname.startswith(uri):
return True
return False
@@ -244,9 +239,7 @@ class IrAttachment(models.Model):
"""
with api.Environment.manage():
if new_cr:
registry = odoo.modules.registry.Registry.new(
self.env.cr.dbname
)
registry = odoo.modules.registry.Registry.new(self.env.cr.dbname)
with closing(registry.cursor()) as cr:
try:
yield self.env(cr=cr)
@@ -263,30 +256,35 @@ class IrAttachment(models.Model):
def _move_attachment_to_store(self):
self.ensure_one()
_logger.info('inspecting attachment %s (%d)', self.name, self.id)
_logger.info("inspecting attachment %s (%d)", self.name, self.id)
fname = self.store_fname
if fname:
# migrating from filesystem filestore
# or from the old 'store_fname' without the bucket name
_logger.info('moving %s on the object storage', fname)
self.write({'datas': self.datas,
# this is required otherwise the
# mimetype gets overriden with
# 'application/octet-stream'
# on assets
'mimetype': self.mimetype})
_logger.info('moved %s on the object storage', fname)
_logger.info("moving %s on the object storage", fname)
self.write(
{
"datas": self.datas,
# this is required otherwise the
# mimetype gets overriden with
# 'application/octet-stream'
# on assets
"mimetype": self.mimetype,
}
)
_logger.info("moved %s on the object storage", fname)
return self._full_path(fname)
elif self.db_datas:
_logger.info('moving on the object storage from database')
self.write({'datas': self.datas})
_logger.info("moving on the object storage from database")
self.write({"datas": self.datas})
@api.model
def force_storage(self):
if not self.env['res.users'].browse(self.env.uid)._is_admin():
if not self.env["res.users"].browse(self.env.uid)._is_admin():
raise exceptions.AccessError(
_('Only administrators can execute this action.'))
location = self.env.context.get('storage_location') or self._storage()
_("Only administrators can execute this action.")
)
location = self.env.context.get("storage_location") or self._storage()
if location not in self._get_stores():
return super().force_storage()
self._force_storage_to_object_storage()
@@ -310,30 +308,32 @@ class IrAttachment(models.Model):
if storage not in self._get_stores():
return
domain = AND((
normalize_domain(
[('store_fname', '=like', '{}://%'.format(storage)),
# for res_field, see comment in
# _force_storage_to_object_storage
'|',
('res_field', '=', False),
('res_field', '!=', False),
]
),
normalize_domain(self._store_in_db_instead_of_object_storage_domain())
))
domain = AND(
(
normalize_domain(
[
("store_fname", "=like", "{}://%".format(storage)),
# for res_field, see comment in
# _force_storage_to_object_storage
"|",
("res_field", "=", False),
("res_field", "!=", False),
]
),
normalize_domain(self._store_in_db_instead_of_object_storage_domain()),
)
)
with self.do_in_new_env(new_cr=new_cr) as new_env:
model_env = new_env['ir.attachment'].with_context(
prefetch_fields=False
)
model_env = new_env["ir.attachment"].with_context(prefetch_fields=False)
attachment_ids = model_env.search(domain).ids
if not attachment_ids:
return
total = len(attachment_ids)
start_time = time.time()
_logger.info('Moving %d attachments from %s to'
' DB for fast access', total, storage)
_logger.info(
"Moving %d attachments from %s to" " DB for fast access", total, storage
)
current = 0
for attachment_id in attachment_ids:
current += 1
@@ -345,36 +345,40 @@ class IrAttachment(models.Model):
# this write will read the datas from the Object Storage and
# write them back in the DB (the logic for location to write is
# in the 'datas' inverse computed field)
attachment.write({'datas': attachment.datas})
attachment.write({"datas": attachment.datas})
# as the file will potentially be dropped on the bucket,
# we should commit the changes here
new_env.cr.commit()
if current % 100 == 0 or total - current == 0:
_logger.info(
'attachment %s/%s after %.2fs',
current, total,
time.time() - start_time
"attachment %s/%s after %.2fs",
current,
total,
time.time() - start_time,
)
@api.model
def _force_storage_to_object_storage(self, new_cr=False):
_logger.info('migrating files to the object storage')
storage = self.env.context.get('storage_location') or self._storage()
_logger.info("migrating files to the object storage")
storage = self.env.context.get("storage_location") or self._storage()
# The weird "res_field = False OR res_field != False" domain
# is required! It's because of an override of _search in ir.attachment
# which adds ('res_field', '=', False) when the domain does not
# contain 'res_field'.
# https://github.com/odoo/odoo/blob/9032617120138848c63b3cfa5d1913c5e5ad76db/odoo/addons/base/ir/ir_attachment.py#L344-L347
domain = ['!', ('store_fname', '=like', '{}://%'.format(storage)),
'|',
('res_field', '=', False),
('res_field', '!=', False)]
# https://github.com/odoo/odoo/blob/9032617120138848c63b3cfa5d1913c5e5ad76db/odoo/addons/base/ir/ir_attachment.py#L344-L347 # noqa: B950
domain = [
"!",
("store_fname", "=like", "{}://%".format(storage)),
"|",
("res_field", "=", False),
("res_field", "!=", False),
]
# We do a copy of the environment so we can workaround the cache issue
# below. We do not create a new cursor by default because it causes
# serialization issues due to concurrent updates on attachments during
# the installation
with self.do_in_new_env(new_cr=new_cr) as new_env:
model_env = new_env['ir.attachment']
model_env = new_env["ir.attachment"]
ids = model_env.search(domain).ids
files_to_clean = []
for attachment_id in ids:
@@ -383,12 +387,14 @@ class IrAttachment(models.Model):
# check that no other transaction has
# locked the row, don't send a file to storage
# in that case
self.env.cr.execute("SELECT id "
"FROM ir_attachment "
"WHERE id = %s "
"FOR UPDATE NOWAIT",
(attachment_id,),
log_exceptions=False)
self.env.cr.execute(
"SELECT id "
"FROM ir_attachment "
"WHERE id = %s "
"FOR UPDATE NOWAIT",
(attachment_id,),
log_exceptions=False,
)
# This is a trick to avoid having the 'datas'
# function fields computed for every attachment on
@@ -401,8 +407,9 @@ class IrAttachment(models.Model):
if path:
files_to_clean.append(path)
except psycopg2.OperationalError:
_logger.error('Could not migrate attachment %s to S3',
attachment_id)
_logger.error(
"Could not migrate attachment %s to S3", attachment_id
)
def clean():
clean_fs(files_to_clean)
@@ -410,7 +417,7 @@ class IrAttachment(models.Model):
# delete the files from the filesystem once we know the changes
# have been committed in ir.attachment
if files_to_clean:
new_env.cr.after('commit', clean)
new_env.cr.after("commit", clean)
def _get_stores(self):
""" To get the list of stores activated in the system """