Burtiraven2/session_redis_public
1
0
Fork 0
mirror of https://github.com/camptocamp/odoo-cloud-platform.git synced 2026-06-23 18:04:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat: remove unused modules

Browse Source
This commit is contained in:
vrenaville
2025-05-15 11:04:22 +02:00
committed by Florent Xicluna
co-authored by Florent Xicluna
parent e529e00fd7
commit a04aa15191
22 changed files with 4 additions and 1091 deletions
Download Patch File Download Diff File Expand all files Collapse all files
attachment_azure/README.rst
-46
View File
@@ -1,46 +0,0 @@
===========================================
Attachments on Microsoft Azure Blob Storage
===========================================
This addon allows to store the attachments (documents and assets) on `Microsoft Azure
Blob Storage <https://docs.microsoft.com/azure/storage/blobs/>`_.
Configuration
-------------
Activate Azure Blob storage:
* Create or set the system parameter with the key ``ir_attachment.location``
and the value in the form ``azure``.
Configure accesses with environment variables:
* ``AZURE_STORAGE_CONNECTION_STRING`` or
* ``AZURE_STORAGE_ACCOUNT_NAME``
* ``AZURE_STORAGE_ACCOUNT_URL``
* ``AZURE_STORAGE_ACCOUNT_KEY``
One container will be created per database using the `RUNNING_ENV` environment variable
and the name of the database. By default, `RUNNING_ENV` is set to `dev`.
The container name can be overridden with environment variable ``AZURE_STORAGE_NAME``.
The strings ``{db}`` and ``{env}`` can be used inside that variable and the values
will be replaced respectively by the database name and environment name.
The container name will also be stored in the database for each attachment,
and will be used to access the right container in the storage.
This addon must be added in the server wide addons with (``--load`` option):
``--load=web,attachment_azure``
The System Parameter ``ir_attachment.storage.force.database`` can be customized to
force storage of files in the database. See the documentation of the module
``base_attachment_object_storage``.
Limitations
-----------
* You need to call ``env['ir.attachment'].force_storage()`` after
having changed the ``ir_attachment.location`` configuration in order to
migrate the existing attachments to Azure Blob Storage.
attachment_azure/__init__.py
-4
View File
@@ -1,4 +0,0 @@
# Copyright 2016-2019 Camptocamp SA
# Copyright 2021 Open Source Integrators
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
from . import models
attachment_azure/__manifest__.py
-22
View File
@@ -1,22 +0,0 @@
# Copyright 2016-2019 Camptocamp SA
# Copyright 2021 Open Source Integrators
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
{
"name": "Attachments on Azure storage",
"summary": "Store assets and attachments on a Azure compatible object storage",
"version": "17.0.1.0.0",
"author": "Camptocamp, "
"Open Source Integrators, "
"Serpent Consulting Services, "
"Odoo Community Association (OCA)",
"license": "AGPL-3",
"category": "Knowledge Management",
"depends": ["base_attachment_object_storage"],
"external_dependencies": {
"python": ["azure-storage-blob", "azure-identity"],
},
"website": "https://github.com/camptocamp/odoo-cloud-platform",
"installable": False,
"development_status": "Beta",
"maintainers": ["max3903"],
}
attachment_azure/models/__init__.py
-4
View File
@@ -1,4 +0,0 @@
# Copyright 2016-2019 Camptocamp SA
# Copyright 2021 Open Source Integrators
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
from . import ir_attachment
attachment_azure/models/ir_attachment.py
-218
View File
@@ -1,218 +0,0 @@
# Copyright 2016-2019 Camptocamp SA
# Copyright 2021 Open Source Integrators
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
import io
import logging
import os
import re
from datetime import datetime, timedelta
from odoo import _, api, exceptions, models
_logger = logging.getLogger(__name__)
try:
from azure.core.exceptions import HttpResponseError, ResourceExistsError
from azure.storage.blob import (
AccountSasPermissions,
BlobServiceClient,
ResourceTypes,
generate_account_sas,
)
except ImportError:
_logger.debug("Cannot 'import azure-storage-blob'.")
try:
from azure.identity import DefaultAzureCredential
except ImportError:
_logger.debug("Cannot 'import azure-identity'.")
class IrAttachment(models.Model):
_inherit = "ir.attachment"
def _get_stores(self):
return ["azure"] + super()._get_stores()
@api.model
def _get_blob_service_client(self):
"""Connect to Azure and return the blob service client
The following environment variables must be set:
* ``AZURE_STORAGE_CONNECTION_STRING``
or
* ``AZURE_STORAGE_ACCOUNT_NAME``
* ``AZURE_STORAGE_ACCOUNT_URL``
* ``AZURE_STORAGE_ACCOUNT_KEY``
or if you want to use AAD (pod identity), set it to 1 or 0
* ``AZURE_STORAGE_USE_AAD``
"""
connect_str = os.environ.get("AZURE_STORAGE_CONNECTION_STRING")
account_name = os.environ.get("AZURE_STORAGE_ACCOUNT_NAME")
account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL")
account_key = os.environ.get("AZURE_STORAGE_ACCOUNT_KEY")
account_use_aad = os.environ.get("AZURE_STORAGE_USE_AAD")
if not (
connect_str
or (account_name and account_url and account_key)
or account_use_aad
):
msg = _(
"If you want to read from the Azure container, you must provide the "
"following environment variables:\n"
"* AZURE_STORAGE_CONNECTION_STRING\n"
"or\n"
"* AZURE_STORAGE_ACCOUNT_NAME\n"
"* AZURE_STORAGE_ACCOUNT_URL\n"
"* AZURE_STORAGE_ACCOUNT_KEY\n"
"or\n"
"* AZURE_STORAGE_USE_AAD\n"
)
raise exceptions.UserError(msg)
blob_service_client = None
if account_use_aad:
token_credential = DefaultAzureCredential()
blob_service_client = BlobServiceClient(
account_url=account_url, credential=token_credential
)
elif connect_str:
try:
blob_service_client = BlobServiceClient.from_connection_string(
connect_str
)
except HttpResponseError as error:
_logger.exception(
"Error during the connection to Azure container using the "
"connection string."
)
raise exceptions.UserError(str(error)) from None
else:
try:
sas_token = generate_account_sas(
account_name=account_name,
account_key=account_key,
resource_types=ResourceTypes(container=True, object=True),
permission=AccountSasPermissions(read=True, write=True),
expiry=datetime.utcnow() + timedelta(hours=1),
)
blob_service_client = BlobServiceClient(
account_url=account_url,
credential=sas_token,
)
except HttpResponseError as error:
_logger.exception(
"Error during the connection to Azure container using the Shared "
"Access Signature (SAS)"
)
raise exceptions.UserError(str(error)) from None
return blob_service_client
@api.model
def _get_container_name(self):
# Container naming rules:
# https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names # noqa: E501
running_env = os.environ.get("RUNNING_ENV", "dev")
storage_name = os.environ.get("AZURE_STORAGE_NAME", r"{env}-{db}")
storage_name = storage_name.format(env=running_env, db=self.env.cr.dbname)
# replace invalid characters by _
storage_name = re.sub(r"[\W_]+", "-", storage_name)
# lowercase, max 63 chars
return str.lower(storage_name)[:63]
@api.model
def _get_azure_container(self, container_name=None):
if not container_name:
container_name = self._get_container_name()
try:
blob_service_client = self._get_blob_service_client()
except exceptions.UserError:
_logger.exception(
"error accessing to storage '%s' please check credentials ",
container_name,
)
return False
container_client = blob_service_client.get_container_client(container_name)
if not container_client.exists():
try:
# Create the container
container_client.create_container()
except HttpResponseError as error:
_logger.exception("Error during the creation of the Azure container")
raise exceptions.UserError(str(error)) from None
return container_client
@api.model
def _store_file_read(self, fname, bin_size=False):
if fname.startswith("azure://"):
key = fname.replace("azure://", "", 1).lower()
if "/" in key:
container_name, key = key.split("/", 1)
else:
container_name = None
container_client = self._get_azure_container(container_name)
# if container cannot be retrived, abort reading from azure storage
if not container_client:
return ""
try:
blob_client = container_client.get_blob_client(key)
read = blob_client.download_blob().readall()
except HttpResponseError:
read = ""
_logger.info("Attachment '%s' missing on object storage", fname)
return read
else:
return super()._store_file_read(fname, bin_size)
@api.model
def _store_file_write(self, key, bin_data):
location = self.env.context.get("storage_location") or self._storage()
if location == "azure":
container_client = self._get_azure_container()
filename = f"azure://{container_client.container_name}/{key}"
with io.BytesIO() as file:
blob_client = container_client.get_blob_client(key.lower())
file.write(bin_data)
file.seek(0)
try:
blob_client.upload_blob(file, blob_type="BlockBlob")
except ResourceExistsError:
_logger.exception(
"Trying to re create an existing resource %s" % filename
)
except HttpResponseError as error:
# log verbose error from azure, return short message for user
_logger.exception(
"HTTP Error during storage of the file %s" % filename
)
raise exceptions.UserError(
_("The file could not be stored: %s") % str(error)
) from None
else:
_super = super()
filename = _super._store_file_write(key, bin_data)
return filename
@api.model
def _store_file_delete(self, fname):
if fname.startswith("azure://"):
key = fname.replace("azure://", "", 1).lower()
if "/" in key:
container_name, key = key.split("/", 1)
else:
container_name = None
container_client = self._get_azure_container(container_name)
if not container_client:
return ""
# delete the file only if it is on the current configured container
# otherwise, we might delete files used on a different environment
try:
blob_client = container_client.get_blob_client(key)
blob_client.delete_blob()
_logger.info("File %s deleted on the object storage" % (fname))
except HttpResponseError:
# log verbose error from azure, return short message for
# user
_logger.exception("Error during deletion of the file %s" % fname)
else:
super()._store_file_delete(fname)
attachment_azure/pyproject.toml
-3
View File
@@ -1,3 +0,0 @@
[build-system]
requires = ["whool"]
build-backend = "whool.buildapi"
base_attachment_object_storage/README.rst
-46
View File
@@ -1,46 +0,0 @@
Base class for attachments on external object store
===================================================
This is a base addon that regroup common code used by addons targeting specific object store
Configuration
-------------
Object storage may be slow, and for this reason, we want to store
some files in the database whatever.
Small images (128, 256) are used in Odoo in list / kanban views. We
want them to be fast to read.
They are generally < 50KB (default configuration) so they don't take
that much space in database, but they'll be read much faster than from
the object storage.
The assets (application/javascript, text/css) are stored in database
as well whatever their size is:
* a database doesn't have thousands of them
* of course better for performance
* better portability of a database: when replicating a production
instance for dev, the assets are included
This storage configuration can be modified in the system parameter
``ir_attachment.storage.force.database``, as a JSON value, for instance::
{"image/": 51200, "application/javascript": 0, "text/css": 0}
Where the key is the beginning of the mimetype to configure and the
value is the limit in size below which attachments are kept in DB.
0 means no limit.
Default configuration means:
* images mimetypes (image/png, image/jpeg, ...) below 50KB are
stored in database
* application/javascript are stored in database whatever their size
* text/css are stored in database whatever their size
Disable attachment storage I/O
------------------------------
Define a environment variable `DISABLE_ATTACHMENT_STORAGE` set to `1`
This will prevent any kind of exceptions and read/write on storage attachments.
base_attachment_object_storage/__init__.py
-26
View File
@@ -1,26 +0,0 @@
from . import models
from odoo.http import Stream
old_from_attachment = Stream.from_attachment
@classmethod
def from_attachment(cls, attachment):
if attachment.store_fname and attachment._is_file_from_a_store(
attachment.store_fname
):
self = cls(
mimetype=attachment.mimetype,
download_name=attachment.name,
conditional=True,
etag=attachment.checksum,
)
self.type = "data"
self.data = attachment.raw
self.size = len(self.data)
return self
return old_from_attachment(attachment)
Stream.from_attachment = from_attachment
base_attachment_object_storage/__manifest__.py
-17
View File
@@ -1,17 +0,0 @@
# Copyright 2017-2021 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
{
"name": "Base Attachment Object Store",
"summary": "Base module for the implementation of external object store.",
"version": "17.0.1.0.0",
"author": "Camptocamp,Odoo Community Association (OCA)",
"license": "AGPL-3",
"category": "Knowledge Management",
"depends": ["base"],
"website": "https://github.com/camptocamp/odoo-cloud-platform",
"data": ["data/res_config_settings_data.xml"],
"installable": False,
"auto_install": True,
}
base_attachment_object_storage/data/res_config_settings_data.xml
-11
View File
@@ -1,11 +0,0 @@
<?xml version='1.0' encoding='utf-8' ?>
<odoo noupdate="1">
<record id="ir_attachment_storage_force_database" model="ir.config_parameter">
<field name="key">ir_attachment.storage.force.database</field>
<field
name="value"
>{"image/": 51200, "application/javascript": 0, "text/css": 0}</field>
</record>
</odoo>
base_attachment_object_storage/models/__init__.py
-1
View File
@@ -1 +0,0 @@
from . import ir_attachment
base_attachment_object_storage/models/ir_attachment.py
-448
View File
@@ -1,448 +0,0 @@
# Copyright 2017-2019 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
import inspect
import logging
import os
import time
from contextlib import closing, contextmanager
import psycopg2
import odoo
from odoo import _, api, exceptions, models
from odoo.osv.expression import AND, OR, normalize_domain
from odoo.tools.safe_eval import const_eval
from .strtobool import strtobool
_logger = logging.getLogger(__name__)
def is_true(strval):
return bool(strtobool(strval or "0"))
def clean_fs(files):
_logger.info("cleaning old files from filestore")
for full_path in files:
if os.path.exists(full_path):
try:
os.unlink(full_path)
except OSError:
# Harmless and needed for race conditions
_logger.info(
"_file_delete could not unlink %s", full_path, exc_info=True
)
class IrAttachment(models.Model):
_inherit = "ir.attachment"
@staticmethod
def is_storage_disabled(storage=None, log=True):
msg = _("Storages are disabled (see environment configuration).")
if storage:
msg = _("Storage '%s' is disabled (see environment configuration).") % (
storage,
)
is_disabled = is_true(os.environ.get("DISABLE_ATTACHMENT_STORAGE"))
if is_disabled and log:
_logger.warning(msg)
return is_disabled
def _register_hook(self):
super()._register_hook()
location = self.env.context.get("storage_location") or self._storage()
# ignore if we are not using an object storage
if location not in self._get_stores():
return
curframe = inspect.currentframe()
calframe = inspect.getouterframes(curframe, 2)
# the caller of _register_hook is 'load_modules' in
# odoo/modules/loading.py
load_modules_frame = calframe[1][0]
# 'update_module' is an argument that 'load_modules' receives with a
# True-ish value meaning that an install or upgrade of addon has been
# done during the initialization. We need to move the attachments that
# could have been created or updated in other addons before this addon
# was loaded
update_module = load_modules_frame.f_locals.get("update_module")
# We need to call the migration on the loading of the model because
# when we are upgrading addons, some of them might add attachments.
# To be sure they are migrated to the storage we need to call the
# migration here.
# Typical example is images of ir.ui.menu which are updated in
# ir.attachment at every upgrade of the addons
if update_module:
self.env["ir.attachment"].sudo()._force_storage_to_object_storage()
@property
def _object_storage_default_force_db_config(self):
return {"image/": 51200, "application/javascript": 0, "text/css": 0}
def _get_storage_force_db_config(self):
param = (
self.env["ir.config_parameter"]
.sudo()
.get_param(
"ir_attachment.storage.force.database",
)
)
storage_config = None
if param:
try:
storage_config = const_eval(param)
except (SyntaxError, TypeError, ValueError):
_logger.exception(
"Could not parse system parameter"
" 'ir_attachment.storage.force.database', reverting to the"
" default configuration."
)
if not storage_config:
storage_config = self._object_storage_default_force_db_config
return storage_config
def _store_in_db_instead_of_object_storage_domain(self):
"""Return a domain for attachments that must be forced to DB
Read the docstring of ``_store_in_db_instead_of_object_storage`` for
more details.
Used in ``force_storage_to_db_for_special_fields`` to find records
to move from the object storage to the database.
The domain must be inline with the conditions in
``_store_in_db_instead_of_object_storage``.
"""
domain = []
storage_config = self._get_storage_force_db_config()
for mimetype_key, limit in storage_config.items():
part = [("mimetype", "=like", f"{mimetype_key}%")]
if limit:
part = AND([part, [("file_size", "<=", limit)]])
domain = OR([domain, part])
return domain
def _store_in_db_instead_of_object_storage(self, data, mimetype):
"""Return whether an attachment must be stored in db
When we are using an Object Storage. This is sometimes required
because the object storage is slower than the database/filesystem.
Small images (128, 256) are used in Odoo in list / kanban views. We
want them to be fast to read.
They are generally < 50KB (default configuration) so they don't take
that much space in database, but they'll be read much faster than from
the object storage.
The assets (application/javascript, text/css) are stored in database
as well whatever their size is:
* a database doesn't have thousands of them
* of course better for performance
* better portability of a database: when replicating a production
instance for dev, the assets are included
The configuration can be modified in the ir.config_parameter
``ir_attachment.storage.force.database``, as a dictionary, for
instance::
{"image/": 51200, "application/javascript": 0, "text/css": 0}
Where the key is the beginning of the mimetype to configure and the
value is the limit in size below which attachments are kept in DB.
0 means no limit.
Default configuration means:
* images mimetypes (image/png, image/jpeg, ...) below 51200 bytes are
stored in database
* application/javascript are stored in database whatever their size
* text/css are stored in database whatever their size
The conditions must be inline with the domain in
``_store_in_db_instead_of_object_storage_domain``.
"""
if self.is_storage_disabled():
return True
storage_config = self._get_storage_force_db_config()
for mimetype_key, limit in storage_config.items():
if mimetype.startswith(mimetype_key):
if not limit:
return True
bin_data = data
return len(bin_data) <= limit
return False
def _get_datas_related_values(self, data, mimetype):
storage = self.env.context.get("storage_location") or self._storage()
if data and storage in self._get_stores():
if self._store_in_db_instead_of_object_storage(data, mimetype):
# compute the fields that depend on datas
bin_data = data
values = {
"file_size": len(bin_data),
"checksum": self._compute_checksum(bin_data),
"index_content": self._index(bin_data, mimetype),
"store_fname": False,
"db_datas": data,
}
return values
return super()._get_datas_related_values(data, mimetype)
@api.model
def _file_read(self, fname):
if self._is_file_from_a_store(fname):
return self._store_file_read(fname)
else:
return super()._file_read(fname)
def _store_file_read(self, fname):
storage = fname.partition("://")[0]
raise NotImplementedError(f"No implementation for {storage}")
def _store_file_write(self, key, bin_data):
storage = self.storage()
raise NotImplementedError(f"No implementation for {storage}")
def _store_file_delete(self, fname):
storage = fname.partition("://")[0]
raise NotImplementedError(f"No implementation for {storage}")
@api.model
def _file_write(self, bin_data, checksum):
location = self.env.context.get("storage_location") or self._storage()
if location in self._get_stores():
key = self.env.context.get("force_storage_key")
if not key:
key = self._compute_checksum(bin_data)
filename = self._store_file_write(key, bin_data)
else:
filename = super()._file_write(bin_data, checksum)
return filename
@api.model
def _file_delete(self, fname):
if self._is_file_from_a_store(fname):
cr = self.env.cr
# using SQL to include files hidden through unlink or due to record
# rules
cr.execute(
"SELECT COUNT(*) FROM ir_attachment " "WHERE store_fname = %s", (fname,)
)
count = cr.fetchone()[0]
if not count:
self._store_file_delete(fname)
else:
return super()._file_delete(fname)
@api.model
def _is_file_from_a_store(self, fname):
for store_name in self._get_stores():
if self.is_storage_disabled(store_name):
continue
uri = f"{store_name}://"
if fname.startswith(uri):
return True
return False
@contextmanager
def do_in_new_env(self, new_cr=False):
"""Context manager that yields a new environment
Using a new Odoo Environment thus a new PG transaction.
"""
if new_cr:
registry = odoo.modules.registry.Registry.new(self.env.cr.dbname)
with closing(registry.cursor()) as cr:
try:
yield self.env(cr=cr)
except Exception:
cr.rollback()
raise
else:
# disable pylint error because this is a valid commit,
# we are in a new env
cr.commit() # pylint: disable=invalid-commit
else:
# make a copy
yield self.env()
def _move_attachment_to_store(self):
self.ensure_one()
_logger.info("inspecting attachment %s (%d)", self.name, self.id)
fname = self.store_fname
storage = fname.partition("://")[0]
if self.is_storage_disabled(storage):
fname = False
if fname:
# migrating from filesystem filestore
# or from the old 'store_fname' without the bucket name
_logger.info("moving %s on the object storage", fname)
self.write(
{
"datas": self.datas,
# this is required otherwise the
# mimetype gets overriden with
# 'application/octet-stream'
# on assets
"mimetype": self.mimetype,
}
)
_logger.info("moved %s on the object storage", fname)
return self._full_path(fname)
elif self.db_datas:
_logger.info("moving on the object storage from database")
self.write({"datas": self.datas})
@api.model
def force_storage(self):
if not self.env["res.users"].browse(self.env.uid)._is_admin():
raise exceptions.AccessError(
_("Only administrators can execute this action.")
)
location = self.env.context.get("storage_location") or self._storage()
if location not in self._get_stores():
return super().force_storage()
self._force_storage_to_object_storage()
@api.model
def force_storage_to_db_for_special_fields(self, new_cr=False):
"""Migrate special attachments from Object Storage back to database
The access to a file stored on the objects storage is slower
than a local disk or database access. For attachments like
image_small that are accessed in batch for kanban views, this
is too slow. We store this type of attachment in the database.
This method can be used when migrating a filestore where all the files,
including the special files (assets, image_small, ...) have been pushed
to the Object Storage and we want to write them back in the database.
It is not called anywhere, but can be called by RPC or scripts.
"""
storage = self._storage()
if self.is_storage_disabled(storage):
return
if storage not in self._get_stores():
return
domain = AND(
(
normalize_domain(
[
("store_fname", "=like", f"{storage}://%"),
# for res_field, see comment in
# _force_storage_to_object_storage
"|",
("res_field", "=", False),
("res_field", "!=", False),
]
),
normalize_domain(self._store_in_db_instead_of_object_storage_domain()),
)
)
with self.do_in_new_env(new_cr=new_cr) as new_env:
model_env = new_env["ir.attachment"].with_context(prefetch_fields=False)
attachment_ids = model_env.search(domain).ids
if not attachment_ids:
return
total = len(attachment_ids)
start_time = time.time()
_logger.info(
"Moving %d attachments from %s to" " DB for fast access", total, storage
)
current = 0
for attachment_id in attachment_ids:
current += 1
# if we browse attachments outside of the loop, the first
# access to 'datas' will compute all the 'datas' fields at
# once, which means reading hundreds or thousands of files at
# once, exhausting memory
attachment = model_env.browse(attachment_id)
# this write will read the datas from the Object Storage and
# write them back in the DB (the logic for location to write is
# in the 'datas' inverse computed field)
attachment.write({"datas": attachment.datas})
# as the file will potentially be dropped on the bucket,
# we should commit the changes here
new_env.cr.commit()
if current % 100 == 0 or total - current == 0:
_logger.info(
"attachment %s/%s after %.2fs",
current,
total,
time.time() - start_time,
)
@api.model
def _force_storage_to_object_storage(self, new_cr=False):
_logger.info("migrating files to the object storage")
storage = self.env.context.get("storage_location") or self._storage()
if self.is_storage_disabled(storage):
return
# The weird "res_field = False OR res_field != False" domain
# is required! It's because of an override of _search in ir.attachment
# which adds ('res_field', '=', False) when the domain does not
# contain 'res_field'.
# https://github.com/odoo/odoo/blob/17.0/odoo/addons/base/models/ir_attachment.py#L523
domain = [
"!",
("store_fname", "=like", f"{storage}://%"),
"|",
("res_field", "=", False),
("res_field", "!=", False),
]
# We do a copy of the environment so we can workaround the cache issue
# below. We do not create a new cursor by default because it causes
# serialization issues due to concurrent updates on attachments during
# the installation
with self.do_in_new_env(new_cr=new_cr) as new_env:
model_env = new_env["ir.attachment"]
ids = model_env.search(domain).ids
files_to_clean = []
for attachment_id in ids:
try:
with new_env.cr.savepoint():
# check that no other transaction has
# locked the row, don't send a file to storage
# in that case
self.env.cr.execute(
"SELECT id "
"FROM ir_attachment "
"WHERE id = %s "
"FOR UPDATE NOWAIT",
(attachment_id,),
log_exceptions=False,
)
# This is a trick to avoid having the 'datas'
# function fields computed for every attachment on
# each iteration of the loop. The former issue
# being that it reads the content of the file of
# ALL the attachments on each loop.
new_env.clear()
attachment = model_env.browse(attachment_id)
path = attachment._move_attachment_to_store()
if path:
files_to_clean.append(path)
except psycopg2.OperationalError:
_logger.error(
"Could not migrate attachment %s to S3", attachment_id
)
# delete the files from the filesystem once we know the changes
# have been committed in ir.attachment
if files_to_clean:
new_env.cr.commit()
clean_fs(files_to_clean)
def _get_stores(self):
"""To get the list of stores activated in the system"""
return []
base_attachment_object_storage/models/strtobool.py
-21
View File
@@ -1,21 +0,0 @@
_MAP = {
"y": True,
"yes": True,
"t": True,
"true": True,
"on": True,
"1": True,
"n": False,
"no": False,
"f": False,
"false": False,
"off": False,
"0": False,
}
def strtobool(value):
try:
return _MAP[str(value).lower()]
except KeyError as error:
raise ValueError(f'"{value}" is not a valid bool value') from error
base_attachment_object_storage/pyproject.toml
-3
View File
@@ -1,3 +0,0 @@
[build-system]
requires = ["whool"]
build-backend = "whool.buildapi"
cloud_platform/__manifest__.py
+1 -1
View File
@@ -17,5 +17,5 @@
], ],
"website": "https://github.com/camptocamp/odoo-cloud-platform", "website": "https://github.com/camptocamp/odoo-cloud-platform",
"data": [], "data": [],
"installable": False, "installable": True,
} }
cloud_platform/models/cloud_platform.py
+3 -60
View File
@@ -1,4 +1,4 @@
# Copyright 2016-2019 Camptocamp SA # Copyright 2016-2025 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
import logging import logging
@@ -18,39 +18,10 @@ def is_true(strval):
return bool(strtobool(strval or "0")) return bool(strtobool(strval or "0"))
PlatformConfig = namedtuple("PlatformConfig", "filestore")
FilestoreKind = namedtuple("FilestoreKind", ["name", "location"])
class CloudPlatform(models.AbstractModel): class CloudPlatform(models.AbstractModel):
_name = "cloud.platform" _name = "cloud.platform"
_description = "cloud.platform" _description = "cloud.platform"
@api.model
def _default_config(self):
return PlatformConfig(self._filestore_kinds()["db"])
@api.model
def _filestore_kinds(self):
return {
"db": FilestoreKind("db", "local"),
"file": FilestoreKind("file", "local"),
}
@api.model
def _platform_kinds(self):
return []
@api.model
def _config_by_server_env(self, platform_kind, environment):
configs_getter = getattr(
self, "_config_by_server_env_for_%s" % platform_kind, None
)
configs = configs_getter() if configs_getter else {}
return configs.get(environment) or self._default_config()
def _get_running_env(self): def _get_running_env(self):
environment_name = config["running_env"] environment_name = config["running_env"]
if environment_name.startswith("labs"): if environment_name.startswith("labs"):
@@ -60,25 +31,9 @@ class CloudPlatform(models.AbstractModel):
return environment_name return environment_name
@api.model @api.model
def _install(self, platform_kind): def _install(self, environment):
assert platform_kind in self._platform_kinds()
params = self.env["ir.config_parameter"].sudo()
params.set_param("cloud.platform.kind", platform_kind)
environment_name = self._get_running_env()
configs = self._config_by_server_env(platform_kind, environment_name)
params.set_param("ir_attachment.location", configs.filestore.name)
self.check() self.check()
if configs.filestore.location == "remote": _logger.info(f"cloud platform configured for {environment}")
self.env["ir.attachment"].sudo().force_storage()
_logger.info(f"cloud platform configured for {platform_kind}")
@api.model
def install(self):
raise NotImplementedError
@api.model
def _check_filestore(self, environment_name):
raise NotImplementedError
@api.model @api.model
def _check_redis(self, environment_name): def _check_redis(self, environment_name):
@@ -110,19 +65,7 @@ class CloudPlatform(models.AbstractModel):
@api.model @api.model
def check(self): def check(self):
if is_true(os.environ.get("ODOO_CLOUD_PLATFORM_UNSAFE")):
_logger.warning("cloud platform checks disabled, this is not safe")
return
params = self.env["ir.config_parameter"].sudo()
kind = params.get_param("cloud.platform.kind")
if not kind:
_logger.warning(
"cloud platform not configured, you should "
"probably run 'env['cloud.platform'].install()'"
)
return
environment_name = self._get_running_env() environment_name = self._get_running_env()
self._check_filestore(environment_name)
self._check_redis(environment_name) self._check_redis(environment_name)
def _register_hook(self): def _register_hook(self):
cloud_platform_azure/README.md
-5
View File
@@ -1,5 +0,0 @@
# Cloud Platform Azure
Install addons specific to the Azure setup.
* The object storage is Azure blob storage
cloud_platform_azure/__init__.py
-1
View File
@@ -1 +0,0 @@
from . import models
cloud_platform_azure/__manifest__.py
-24
View File
@@ -1,24 +0,0 @@
# Copyright 2017-2021 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
{
"name": "Cloud Platform Azure",
"summary": "Addons required for the Camptocamp Cloud Platform on Azure",
"version": "17.0.1.0.0",
"author": "Camptocamp,Odoo Community Association (OCA)",
"license": "AGPL-3",
"category": "Extra Tools",
"depends": [
"cloud_platform",
"attachment_azure",
"monitoring_prometheus",
],
"excludes": [
"cloud_platform_ovh",
"cloud_platform_exoscale",
],
"website": "https://github.com/camptocamp/odoo-cloud-platform",
"data": [],
"installable": False,
}
cloud_platform_azure/models/__init__.py
-1
View File
@@ -1 +0,0 @@
from . import cloud_platform
cloud_platform_azure/models/cloud_platform.py
-126
View File
@@ -1,126 +0,0 @@
# Copyright 2016-2021 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
import os
import re
from odoo import api, models
from odoo.addons.cloud_platform.models.cloud_platform import (
FilestoreKind,
PlatformConfig,
)
AZURE_STORE_KIND = FilestoreKind("azure", "remote")
class CloudPlatform(models.AbstractModel):
_inherit = "cloud.platform"
@api.model
def _filestore_kinds(self):
kinds = super()._filestore_kinds()
kinds["azure"] = AZURE_STORE_KIND
return kinds
@api.model
def _platform_kinds(self):
kinds = super()._platform_kinds()
kinds.append("azure")
return kinds
@api.model
def _config_by_server_env_for_azure(self):
fs_kinds = self._filestore_kinds()
configs = {
"prod": PlatformConfig(filestore=fs_kinds["azure"]),
"integration": PlatformConfig(filestore=fs_kinds["azure"]),
"labs": PlatformConfig(filestore=fs_kinds["azure"]),
"test": PlatformConfig(filestore=fs_kinds["db"]),
"dev": PlatformConfig(filestore=fs_kinds["db"]),
}
return configs
@api.model
def _check_filestore(self, environment_name):
params = self.env["ir.config_parameter"].sudo()
use_azure = params.get_param("ir_attachment.location") == AZURE_STORE_KIND.name
if environment_name in ("prod", "integration"):
# Labs instances use azure by default, but we don't want
# to enforce it in case we want to test something with a different
# storage. At your own risks!
assert use_azure, (
"azure must be used on production and integration instances. "
"It is activated by setting 'ir_attachment.location.' to 'azure'."
" The 'install()' function sets this option "
"automatically."
)
if use_azure:
key_sets = [
["AZURE_STORAGE_USE_AAD", "AZURE_STORAGE_ACCOUNT_URL"],
["AZURE_STORAGE_CONNECTION_STRING"],
[
"AZURE_STORAGE_ACCOUNT_NAME",
"AZURE_STORAGE_ACCOUNT_URL",
"AZURE_STORAGE_ACCOUNT_KEY",
],
]
is_valid = False
for key_set in key_sets:
if all([os.environ.get(key) for key in key_set]):
is_valid = True
break
assert is_valid, (
"When ir_attachment.location is set to 'azure', "
"at least one of the following enviromnent variable set "
"is required : {}".format(
" or ".join(
[" + ".join([key for key in key_set]) for key_set in key_sets]
)
)
)
storage_name = os.environ.get("AZURE_STORAGE_NAME", "")
if environment_name in ("prod", "integration", "labs"):
assert storage_name, (
"AZURE_STORAGE_NAME environment variable is required when "
"ir_attachment.location is 'azure'.\n"
"Normally, 'azure' is activated on labs, integration "
"and production, but should not be used in dev environment"
" (or using a dedicated dev bucket, never using the "
"integration/prod bucket).\n"
"If you don't actually need a bucket, change the"
" 'ir_attachment.location' parameter."
)
# A bucket name is defined under the following format
# ^[a-z]+\-[a-z]+\-\d+$
# Anything other than prod bucket must be suffixed with env name
#
# Use AZURE_STORAGE_NAME_UNSTRUCTURED to by-pass check
# on bucket name structure
if os.environ.get("AZURE_STORAGE_NAME_UNSTRUCTURED"):
return
prod_bucket = bool(re.match(r"^[a-z]+\-[a-z]+\-\d+$", storage_name))
if environment_name == "prod":
assert prod_bucket, (
"AZURE_STORAGE_NAME should match '^[a-z]+\\-[a-z]+\\-\\d+$', "
f"we got: '{storage_name}'"
)
else:
# if we are using the prod bucket on another instance
# such as an integration, we must be sure to be in read only!
assert not prod_bucket, (
"AZURE_STORAGE_NAME should not match '^[a-z]+\\-[a-z]+\\-\\d+$', "
f"we got: '{storage_name}'"
)
elif environment_name == "test":
# store in DB so we don't have files local to the host
assert params.get_param("ir_attachment.location") == "db", (
"In test instances, files must be stored in the database with "
"'ir_attachment.location' set to 'db'. This is "
"automatically set by the function 'install()'."
)
@api.model
def install(self):
self._install("azure")
cloud_platform_azure/pyproject.toml
-3
View File
@@ -1,3 +0,0 @@
[build-system]
requires = ["whool"]
build-backend = "whool.buildapi"