mirror of
https://github.com/camptocamp/odoo-cloud-platform.git
synced 2026-06-23 18:04:34 +00:00
Merge pull request #496 from camptocamp/18.0-upd_version
fix: this is Odoo 18.0 branch
This commit is contained in:
+2
-3
@@ -13,12 +13,11 @@ github_enable_stale_action: true
|
|||||||
github_enforce_dev_status_compatibility: false
|
github_enforce_dev_status_compatibility: false
|
||||||
include_wkhtmltopdf: false
|
include_wkhtmltopdf: false
|
||||||
odoo_test_flavor: Both
|
odoo_test_flavor: Both
|
||||||
odoo_version: 17.0
|
odoo_version: 18.0
|
||||||
org_name: Camptocamp
|
org_name: Camptocamp
|
||||||
org_slug: camptocamp
|
org_slug: camptocamp
|
||||||
rebel_module_groups:
|
rebel_module_groups:
|
||||||
- attachment_azure,cloud_platform_azure
|
repo_description: 'Tools to run Odoo on a cloud platform.'
|
||||||
repo_description: 'Tools to run Odoo on a cloud platform. Mainly Azure at the moment. '
|
|
||||||
repo_name: Odoo Cloud Addons
|
repo_name: Odoo Cloud Addons
|
||||||
repo_slug: odoo-cloud-platform
|
repo_slug: odoo-cloud-platform
|
||||||
repo_website: https://github.com/camptocamp/odoo-cloud-platform
|
repo_website: https://github.com/camptocamp/odoo-cloud-platform
|
||||||
|
|||||||
@@ -3,11 +3,11 @@ name: pre-commit
|
|||||||
on:
|
on:
|
||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- "17.0*"
|
- "18.0*"
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- "17.0"
|
- "18.0"
|
||||||
- "17.0-ocabot-*"
|
- "18.0-ocabot-*"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
pre-commit:
|
pre-commit:
|
||||||
|
|||||||
@@ -3,11 +3,11 @@ name: tests
|
|||||||
on:
|
on:
|
||||||
pull_request:
|
pull_request:
|
||||||
branches:
|
branches:
|
||||||
- "17.0*"
|
- "18.0*"
|
||||||
push:
|
push:
|
||||||
branches:
|
branches:
|
||||||
- "17.0"
|
- "18.0"
|
||||||
- "17.0-ocabot-*"
|
- "18.0-ocabot-*"
|
||||||
|
|
||||||
jobs:
|
jobs:
|
||||||
unreleased-deps:
|
unreleased-deps:
|
||||||
@@ -35,18 +35,9 @@ jobs:
|
|||||||
fail-fast: false
|
fail-fast: false
|
||||||
matrix:
|
matrix:
|
||||||
include:
|
include:
|
||||||
- container: ghcr.io/oca/oca-ci/py3.10-odoo17.0:latest
|
- container: ghcr.io/oca/oca-ci/py3.10-odoo18.0:latest
|
||||||
include: "attachment_azure,cloud_platform_azure"
|
|
||||||
name: test with Odoo
|
name: test with Odoo
|
||||||
- container: ghcr.io/oca/oca-ci/py3.10-ocb17.0:latest
|
- container: ghcr.io/oca/oca-ci/py3.10-ocb18.0:latest
|
||||||
include: "attachment_azure,cloud_platform_azure"
|
|
||||||
name: test with OCB
|
|
||||||
makepot: "false"
|
|
||||||
- container: ghcr.io/oca/oca-ci/py3.10-odoo17.0:latest
|
|
||||||
exclude: "attachment_azure,cloud_platform_azure"
|
|
||||||
name: test with Odoo
|
|
||||||
- container: ghcr.io/oca/oca-ci/py3.10-ocb17.0:latest
|
|
||||||
exclude: "attachment_azure,cloud_platform_azure"
|
|
||||||
name: test with OCB
|
name: test with OCB
|
||||||
makepot: "false"
|
makepot: "false"
|
||||||
services:
|
services:
|
||||||
@@ -61,6 +52,8 @@ jobs:
|
|||||||
env:
|
env:
|
||||||
INCLUDE: "${{ matrix.include }}"
|
INCLUDE: "${{ matrix.include }}"
|
||||||
EXCLUDE: "${{ matrix.exclude }}"
|
EXCLUDE: "${{ matrix.exclude }}"
|
||||||
|
# Disable Redis check
|
||||||
|
ODOO_CLOUD_PLATFORM_UNSAFE: 1
|
||||||
steps:
|
steps:
|
||||||
- uses: actions/checkout@v3
|
- uses: actions/checkout@v3
|
||||||
with:
|
with:
|
||||||
|
|||||||
+6
-3
@@ -1,9 +1,6 @@
|
|||||||
# Byte-compiled / optimized / DLL files
|
# Byte-compiled / optimized / DLL files
|
||||||
__pycache__/
|
__pycache__/
|
||||||
*.py[cod]
|
*.py[cod]
|
||||||
/.venv
|
|
||||||
/.pytest_cache
|
|
||||||
/.ruff_cache
|
|
||||||
|
|
||||||
# C extensions
|
# C extensions
|
||||||
*.so
|
*.so
|
||||||
@@ -30,6 +27,9 @@ pip-log.txt
|
|||||||
pip-delete-this-directory.txt
|
pip-delete-this-directory.txt
|
||||||
|
|
||||||
# Unit test / coverage reports
|
# Unit test / coverage reports
|
||||||
|
/.pytest_cache
|
||||||
|
/.ruff_cache
|
||||||
|
/.venv
|
||||||
htmlcov/
|
htmlcov/
|
||||||
.tox/
|
.tox/
|
||||||
.coverage
|
.coverage
|
||||||
@@ -64,6 +64,9 @@ coverage.xml
|
|||||||
# Rope
|
# Rope
|
||||||
.ropeproject
|
.ropeproject
|
||||||
|
|
||||||
|
# Pyenv
|
||||||
|
.python-version
|
||||||
|
|
||||||
# Sphinx documentation
|
# Sphinx documentation
|
||||||
docs/_build/
|
docs/_build/
|
||||||
|
|
||||||
|
|||||||
@@ -56,7 +56,7 @@ repos:
|
|||||||
- id: oca-gen-addon-readme
|
- id: oca-gen-addon-readme
|
||||||
args:
|
args:
|
||||||
- --addons-dir=.
|
- --addons-dir=.
|
||||||
- --branch=17.0
|
- --branch=18.0
|
||||||
- --org-name=camptocamp
|
- --org-name=camptocamp
|
||||||
- --repo-name=odoo-cloud-platform
|
- --repo-name=odoo-cloud-platform
|
||||||
- --if-source-changed
|
- --if-source-changed
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ manifest-required-authors=Camptocamp
|
|||||||
manifest-required-keys=license
|
manifest-required-keys=license
|
||||||
manifest-deprecated-keys=description,active
|
manifest-deprecated-keys=description,active
|
||||||
license-allowed=AGPL-3,GPL-2,GPL-2 or any later version,GPL-3,GPL-3 or any later version,LGPL-3
|
license-allowed=AGPL-3,GPL-2,GPL-2 or any later version,GPL-3,GPL-3 or any later version,LGPL-3
|
||||||
valid-odoo-versions=17.0
|
valid-odoo-versions=18.0
|
||||||
|
|
||||||
[MESSAGES CONTROL]
|
[MESSAGES CONTROL]
|
||||||
disable=all
|
disable=all
|
||||||
|
|||||||
+1
-1
@@ -9,7 +9,7 @@ manifest-required-authors=Camptocamp
|
|||||||
manifest-required-keys=license
|
manifest-required-keys=license
|
||||||
manifest-deprecated-keys=description,active
|
manifest-deprecated-keys=description,active
|
||||||
license-allowed=AGPL-3,GPL-2,GPL-2 or any later version,GPL-3,GPL-3 or any later version,LGPL-3
|
license-allowed=AGPL-3,GPL-2,GPL-2 or any later version,GPL-3,GPL-3 or any later version,LGPL-3
|
||||||
valid-odoo-versions=17.0
|
valid-odoo-versions=18.0
|
||||||
|
|
||||||
[MESSAGES CONTROL]
|
[MESSAGES CONTROL]
|
||||||
disable=all
|
disable=all
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
|
|
||||||
<!-- /!\ Non OCA Context : Set here the badge of your runbot / runboat instance. -->
|
<!-- /!\ Non OCA Context : Set here the badge of your runbot / runboat instance. -->
|
||||||
[](https://github.com/camptocamp/odoo-cloud-platform/actions/workflows/pre-commit.yml?query=branch%3A17.0)
|
[](https://github.com/camptocamp/odoo-cloud-platform/actions/workflows/pre-commit.yml?query=branch%3A18.0)
|
||||||
[](https://github.com/camptocamp/odoo-cloud-platform/actions/workflows/test.yml?query=branch%3A17.0)
|
[](https://github.com/camptocamp/odoo-cloud-platform/actions/workflows/test.yml?query=branch%3A18.0)
|
||||||
[](https://codecov.io/gh/camptocamp/odoo-cloud-platform)
|
[](https://codecov.io/gh/camptocamp/odoo-cloud-platform)
|
||||||
<!-- /!\ Non OCA Context : Set here the badge of your translation instance. -->
|
<!-- /!\ Non OCA Context : Set here the badge of your translation instance. -->
|
||||||
|
|
||||||
<!-- /!\ do not modify above this line -->
|
<!-- /!\ do not modify above this line -->
|
||||||
|
|||||||
@@ -1,46 +0,0 @@
|
|||||||
===========================================
|
|
||||||
Attachments on Microsoft Azure Blob Storage
|
|
||||||
===========================================
|
|
||||||
|
|
||||||
This addon allows to store the attachments (documents and assets) on `Microsoft Azure
|
|
||||||
Blob Storage <https://docs.microsoft.com/azure/storage/blobs/>`_.
|
|
||||||
|
|
||||||
Configuration
|
|
||||||
-------------
|
|
||||||
|
|
||||||
Activate Azure Blob storage:
|
|
||||||
|
|
||||||
* Create or set the system parameter with the key ``ir_attachment.location``
|
|
||||||
and the value in the form ``azure``.
|
|
||||||
|
|
||||||
Configure accesses with environment variables:
|
|
||||||
|
|
||||||
* ``AZURE_STORAGE_CONNECTION_STRING`` or
|
|
||||||
* ``AZURE_STORAGE_ACCOUNT_NAME``
|
|
||||||
* ``AZURE_STORAGE_ACCOUNT_URL``
|
|
||||||
* ``AZURE_STORAGE_ACCOUNT_KEY``
|
|
||||||
|
|
||||||
One container will be created per database using the `RUNNING_ENV` environment variable
|
|
||||||
and the name of the database. By default, `RUNNING_ENV` is set to `dev`.
|
|
||||||
|
|
||||||
The container name can be overridden with environment variable ``AZURE_STORAGE_NAME``.
|
|
||||||
The strings ``{db}`` and ``{env}`` can be used inside that variable and the values
|
|
||||||
will be replaced respectively by the database name and environment name.
|
|
||||||
|
|
||||||
The container name will also be stored in the database for each attachment,
|
|
||||||
and will be used to access the right container in the storage.
|
|
||||||
|
|
||||||
This addon must be added in the server wide addons with (``--load`` option):
|
|
||||||
|
|
||||||
``--load=web,attachment_azure``
|
|
||||||
|
|
||||||
The System Parameter ``ir_attachment.storage.force.database`` can be customized to
|
|
||||||
force storage of files in the database. See the documentation of the module
|
|
||||||
``base_attachment_object_storage``.
|
|
||||||
|
|
||||||
Limitations
|
|
||||||
-----------
|
|
||||||
|
|
||||||
* You need to call ``env['ir.attachment'].force_storage()`` after
|
|
||||||
having changed the ``ir_attachment.location`` configuration in order to
|
|
||||||
migrate the existing attachments to Azure Blob Storage.
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
# Copyright 2016-2019 Camptocamp SA
|
|
||||||
# Copyright 2021 Open Source Integrators
|
|
||||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
|
||||||
from . import models
|
|
||||||
@@ -1,22 +0,0 @@
|
|||||||
# Copyright 2016-2019 Camptocamp SA
|
|
||||||
# Copyright 2021 Open Source Integrators
|
|
||||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
|
||||||
{
|
|
||||||
"name": "Attachments on Azure storage",
|
|
||||||
"summary": "Store assets and attachments on a Azure compatible object storage",
|
|
||||||
"version": "17.0.1.0.0",
|
|
||||||
"author": "Camptocamp, "
|
|
||||||
"Open Source Integrators, "
|
|
||||||
"Serpent Consulting Services, "
|
|
||||||
"Odoo Community Association (OCA)",
|
|
||||||
"license": "AGPL-3",
|
|
||||||
"category": "Knowledge Management",
|
|
||||||
"depends": ["base_attachment_object_storage"],
|
|
||||||
"external_dependencies": {
|
|
||||||
"python": ["azure-storage-blob", "azure-identity"],
|
|
||||||
},
|
|
||||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
|
||||||
"installable": False,
|
|
||||||
"development_status": "Beta",
|
|
||||||
"maintainers": ["max3903"],
|
|
||||||
}
|
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
# Copyright 2016-2019 Camptocamp SA
|
|
||||||
# Copyright 2021 Open Source Integrators
|
|
||||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
|
||||||
from . import ir_attachment
|
|
||||||
@@ -1,218 +0,0 @@
|
|||||||
# Copyright 2016-2019 Camptocamp SA
|
|
||||||
# Copyright 2021 Open Source Integrators
|
|
||||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
|
||||||
import io
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
from datetime import datetime, timedelta
|
|
||||||
|
|
||||||
from odoo import _, api, exceptions, models
|
|
||||||
|
|
||||||
_logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
try:
|
|
||||||
from azure.core.exceptions import HttpResponseError, ResourceExistsError
|
|
||||||
from azure.storage.blob import (
|
|
||||||
AccountSasPermissions,
|
|
||||||
BlobServiceClient,
|
|
||||||
ResourceTypes,
|
|
||||||
generate_account_sas,
|
|
||||||
)
|
|
||||||
except ImportError:
|
|
||||||
_logger.debug("Cannot 'import azure-storage-blob'.")
|
|
||||||
|
|
||||||
try:
|
|
||||||
from azure.identity import DefaultAzureCredential
|
|
||||||
except ImportError:
|
|
||||||
_logger.debug("Cannot 'import azure-identity'.")
|
|
||||||
|
|
||||||
|
|
||||||
class IrAttachment(models.Model):
|
|
||||||
_inherit = "ir.attachment"
|
|
||||||
|
|
||||||
def _get_stores(self):
|
|
||||||
return ["azure"] + super()._get_stores()
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _get_blob_service_client(self):
|
|
||||||
"""Connect to Azure and return the blob service client
|
|
||||||
|
|
||||||
The following environment variables must be set:
|
|
||||||
* ``AZURE_STORAGE_CONNECTION_STRING``
|
|
||||||
or
|
|
||||||
* ``AZURE_STORAGE_ACCOUNT_NAME``
|
|
||||||
* ``AZURE_STORAGE_ACCOUNT_URL``
|
|
||||||
* ``AZURE_STORAGE_ACCOUNT_KEY``
|
|
||||||
or if you want to use AAD (pod identity), set it to 1 or 0
|
|
||||||
* ``AZURE_STORAGE_USE_AAD``
|
|
||||||
|
|
||||||
"""
|
|
||||||
connect_str = os.environ.get("AZURE_STORAGE_CONNECTION_STRING")
|
|
||||||
account_name = os.environ.get("AZURE_STORAGE_ACCOUNT_NAME")
|
|
||||||
account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL")
|
|
||||||
account_key = os.environ.get("AZURE_STORAGE_ACCOUNT_KEY")
|
|
||||||
account_use_aad = os.environ.get("AZURE_STORAGE_USE_AAD")
|
|
||||||
if not (
|
|
||||||
connect_str
|
|
||||||
or (account_name and account_url and account_key)
|
|
||||||
or account_use_aad
|
|
||||||
):
|
|
||||||
msg = _(
|
|
||||||
"If you want to read from the Azure container, you must provide the "
|
|
||||||
"following environment variables:\n"
|
|
||||||
"* AZURE_STORAGE_CONNECTION_STRING\n"
|
|
||||||
"or\n"
|
|
||||||
"* AZURE_STORAGE_ACCOUNT_NAME\n"
|
|
||||||
"* AZURE_STORAGE_ACCOUNT_URL\n"
|
|
||||||
"* AZURE_STORAGE_ACCOUNT_KEY\n"
|
|
||||||
"or\n"
|
|
||||||
"* AZURE_STORAGE_USE_AAD\n"
|
|
||||||
)
|
|
||||||
raise exceptions.UserError(msg)
|
|
||||||
blob_service_client = None
|
|
||||||
if account_use_aad:
|
|
||||||
token_credential = DefaultAzureCredential()
|
|
||||||
blob_service_client = BlobServiceClient(
|
|
||||||
account_url=account_url, credential=token_credential
|
|
||||||
)
|
|
||||||
elif connect_str:
|
|
||||||
try:
|
|
||||||
blob_service_client = BlobServiceClient.from_connection_string(
|
|
||||||
connect_str
|
|
||||||
)
|
|
||||||
except HttpResponseError as error:
|
|
||||||
_logger.exception(
|
|
||||||
"Error during the connection to Azure container using the "
|
|
||||||
"connection string."
|
|
||||||
)
|
|
||||||
raise exceptions.UserError(str(error)) from None
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
sas_token = generate_account_sas(
|
|
||||||
account_name=account_name,
|
|
||||||
account_key=account_key,
|
|
||||||
resource_types=ResourceTypes(container=True, object=True),
|
|
||||||
permission=AccountSasPermissions(read=True, write=True),
|
|
||||||
expiry=datetime.utcnow() + timedelta(hours=1),
|
|
||||||
)
|
|
||||||
blob_service_client = BlobServiceClient(
|
|
||||||
account_url=account_url,
|
|
||||||
credential=sas_token,
|
|
||||||
)
|
|
||||||
except HttpResponseError as error:
|
|
||||||
_logger.exception(
|
|
||||||
"Error during the connection to Azure container using the Shared "
|
|
||||||
"Access Signature (SAS)"
|
|
||||||
)
|
|
||||||
raise exceptions.UserError(str(error)) from None
|
|
||||||
return blob_service_client
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _get_container_name(self):
|
|
||||||
# Container naming rules:
|
|
||||||
# https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names # noqa: E501
|
|
||||||
running_env = os.environ.get("RUNNING_ENV", "dev")
|
|
||||||
storage_name = os.environ.get("AZURE_STORAGE_NAME", r"{env}-{db}")
|
|
||||||
storage_name = storage_name.format(env=running_env, db=self.env.cr.dbname)
|
|
||||||
# replace invalid characters by _
|
|
||||||
storage_name = re.sub(r"[\W_]+", "-", storage_name)
|
|
||||||
# lowercase, max 63 chars
|
|
||||||
return str.lower(storage_name)[:63]
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _get_azure_container(self, container_name=None):
|
|
||||||
if not container_name:
|
|
||||||
container_name = self._get_container_name()
|
|
||||||
try:
|
|
||||||
blob_service_client = self._get_blob_service_client()
|
|
||||||
except exceptions.UserError:
|
|
||||||
_logger.exception(
|
|
||||||
"error accessing to storage '%s' please check credentials ",
|
|
||||||
container_name,
|
|
||||||
)
|
|
||||||
return False
|
|
||||||
container_client = blob_service_client.get_container_client(container_name)
|
|
||||||
if not container_client.exists():
|
|
||||||
try:
|
|
||||||
# Create the container
|
|
||||||
container_client.create_container()
|
|
||||||
except HttpResponseError as error:
|
|
||||||
_logger.exception("Error during the creation of the Azure container")
|
|
||||||
raise exceptions.UserError(str(error)) from None
|
|
||||||
return container_client
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _store_file_read(self, fname, bin_size=False):
|
|
||||||
if fname.startswith("azure://"):
|
|
||||||
key = fname.replace("azure://", "", 1).lower()
|
|
||||||
if "/" in key:
|
|
||||||
container_name, key = key.split("/", 1)
|
|
||||||
else:
|
|
||||||
container_name = None
|
|
||||||
container_client = self._get_azure_container(container_name)
|
|
||||||
# if container cannot be retrived, abort reading from azure storage
|
|
||||||
if not container_client:
|
|
||||||
return ""
|
|
||||||
try:
|
|
||||||
blob_client = container_client.get_blob_client(key)
|
|
||||||
read = blob_client.download_blob().readall()
|
|
||||||
except HttpResponseError:
|
|
||||||
read = ""
|
|
||||||
_logger.info("Attachment '%s' missing on object storage", fname)
|
|
||||||
return read
|
|
||||||
else:
|
|
||||||
return super()._store_file_read(fname, bin_size)
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _store_file_write(self, key, bin_data):
|
|
||||||
location = self.env.context.get("storage_location") or self._storage()
|
|
||||||
if location == "azure":
|
|
||||||
container_client = self._get_azure_container()
|
|
||||||
filename = f"azure://{container_client.container_name}/{key}"
|
|
||||||
with io.BytesIO() as file:
|
|
||||||
blob_client = container_client.get_blob_client(key.lower())
|
|
||||||
file.write(bin_data)
|
|
||||||
file.seek(0)
|
|
||||||
try:
|
|
||||||
blob_client.upload_blob(file, blob_type="BlockBlob")
|
|
||||||
except ResourceExistsError:
|
|
||||||
_logger.exception(
|
|
||||||
"Trying to re create an existing resource %s" % filename
|
|
||||||
)
|
|
||||||
except HttpResponseError as error:
|
|
||||||
# log verbose error from azure, return short message for user
|
|
||||||
_logger.exception(
|
|
||||||
"HTTP Error during storage of the file %s" % filename
|
|
||||||
)
|
|
||||||
raise exceptions.UserError(
|
|
||||||
_("The file could not be stored: %s") % str(error)
|
|
||||||
) from None
|
|
||||||
else:
|
|
||||||
_super = super()
|
|
||||||
filename = _super._store_file_write(key, bin_data)
|
|
||||||
return filename
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _store_file_delete(self, fname):
|
|
||||||
if fname.startswith("azure://"):
|
|
||||||
key = fname.replace("azure://", "", 1).lower()
|
|
||||||
if "/" in key:
|
|
||||||
container_name, key = key.split("/", 1)
|
|
||||||
else:
|
|
||||||
container_name = None
|
|
||||||
container_client = self._get_azure_container(container_name)
|
|
||||||
if not container_client:
|
|
||||||
return ""
|
|
||||||
# delete the file only if it is on the current configured container
|
|
||||||
# otherwise, we might delete files used on a different environment
|
|
||||||
try:
|
|
||||||
blob_client = container_client.get_blob_client(key)
|
|
||||||
blob_client.delete_blob()
|
|
||||||
_logger.info("File %s deleted on the object storage" % (fname))
|
|
||||||
except HttpResponseError:
|
|
||||||
# log verbose error from azure, return short message for
|
|
||||||
# user
|
|
||||||
_logger.exception("Error during deletion of the file %s" % fname)
|
|
||||||
else:
|
|
||||||
super()._store_file_delete(fname)
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
[build-system]
|
|
||||||
requires = ["whool"]
|
|
||||||
build-backend = "whool.buildapi"
|
|
||||||
@@ -1,46 +0,0 @@
|
|||||||
Base class for attachments on external object store
|
|
||||||
===================================================
|
|
||||||
|
|
||||||
This is a base addon that regroup common code used by addons targeting specific object store
|
|
||||||
|
|
||||||
Configuration
|
|
||||||
-------------
|
|
||||||
|
|
||||||
Object storage may be slow, and for this reason, we want to store
|
|
||||||
some files in the database whatever.
|
|
||||||
|
|
||||||
Small images (128, 256) are used in Odoo in list / kanban views. We
|
|
||||||
want them to be fast to read.
|
|
||||||
They are generally < 50KB (default configuration) so they don't take
|
|
||||||
that much space in database, but they'll be read much faster than from
|
|
||||||
the object storage.
|
|
||||||
|
|
||||||
The assets (application/javascript, text/css) are stored in database
|
|
||||||
as well whatever their size is:
|
|
||||||
|
|
||||||
* a database doesn't have thousands of them
|
|
||||||
* of course better for performance
|
|
||||||
* better portability of a database: when replicating a production
|
|
||||||
instance for dev, the assets are included
|
|
||||||
|
|
||||||
This storage configuration can be modified in the system parameter
|
|
||||||
``ir_attachment.storage.force.database``, as a JSON value, for instance::
|
|
||||||
|
|
||||||
{"image/": 51200, "application/javascript": 0, "text/css": 0}
|
|
||||||
|
|
||||||
Where the key is the beginning of the mimetype to configure and the
|
|
||||||
value is the limit in size below which attachments are kept in DB.
|
|
||||||
0 means no limit.
|
|
||||||
|
|
||||||
Default configuration means:
|
|
||||||
|
|
||||||
* images mimetypes (image/png, image/jpeg, ...) below 50KB are
|
|
||||||
stored in database
|
|
||||||
* application/javascript are stored in database whatever their size
|
|
||||||
* text/css are stored in database whatever their size
|
|
||||||
|
|
||||||
Disable attachment storage I/O
|
|
||||||
------------------------------
|
|
||||||
|
|
||||||
Define a environment variable `DISABLE_ATTACHMENT_STORAGE` set to `1`
|
|
||||||
This will prevent any kind of exceptions and read/write on storage attachments.
|
|
||||||
@@ -1,26 +0,0 @@
|
|||||||
from . import models
|
|
||||||
from odoo.http import Stream
|
|
||||||
|
|
||||||
|
|
||||||
old_from_attachment = Stream.from_attachment
|
|
||||||
|
|
||||||
|
|
||||||
@classmethod
|
|
||||||
def from_attachment(cls, attachment):
|
|
||||||
if attachment.store_fname and attachment._is_file_from_a_store(
|
|
||||||
attachment.store_fname
|
|
||||||
):
|
|
||||||
self = cls(
|
|
||||||
mimetype=attachment.mimetype,
|
|
||||||
download_name=attachment.name,
|
|
||||||
conditional=True,
|
|
||||||
etag=attachment.checksum,
|
|
||||||
)
|
|
||||||
self.type = "data"
|
|
||||||
self.data = attachment.raw
|
|
||||||
self.size = len(self.data)
|
|
||||||
return self
|
|
||||||
return old_from_attachment(attachment)
|
|
||||||
|
|
||||||
|
|
||||||
Stream.from_attachment = from_attachment
|
|
||||||
@@ -1,17 +0,0 @@
|
|||||||
# Copyright 2017-2021 Camptocamp SA
|
|
||||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
|
||||||
|
|
||||||
|
|
||||||
{
|
|
||||||
"name": "Base Attachment Object Store",
|
|
||||||
"summary": "Base module for the implementation of external object store.",
|
|
||||||
"version": "17.0.1.0.0",
|
|
||||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
|
||||||
"license": "AGPL-3",
|
|
||||||
"category": "Knowledge Management",
|
|
||||||
"depends": ["base"],
|
|
||||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
|
||||||
"data": ["data/res_config_settings_data.xml"],
|
|
||||||
"installable": False,
|
|
||||||
"auto_install": True,
|
|
||||||
}
|
|
||||||
@@ -1,11 +0,0 @@
|
|||||||
<?xml version='1.0' encoding='utf-8' ?>
|
|
||||||
<odoo noupdate="1">
|
|
||||||
|
|
||||||
<record id="ir_attachment_storage_force_database" model="ir.config_parameter">
|
|
||||||
<field name="key">ir_attachment.storage.force.database</field>
|
|
||||||
<field
|
|
||||||
name="value"
|
|
||||||
>{"image/": 51200, "application/javascript": 0, "text/css": 0}</field>
|
|
||||||
</record>
|
|
||||||
|
|
||||||
</odoo>
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
from . import ir_attachment
|
|
||||||
@@ -1,448 +0,0 @@
|
|||||||
# Copyright 2017-2019 Camptocamp SA
|
|
||||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
|
||||||
|
|
||||||
import inspect
|
|
||||||
import logging
|
|
||||||
import os
|
|
||||||
import time
|
|
||||||
from contextlib import closing, contextmanager
|
|
||||||
|
|
||||||
import psycopg2
|
|
||||||
|
|
||||||
import odoo
|
|
||||||
from odoo import _, api, exceptions, models
|
|
||||||
from odoo.osv.expression import AND, OR, normalize_domain
|
|
||||||
from odoo.tools.safe_eval import const_eval
|
|
||||||
|
|
||||||
from .strtobool import strtobool
|
|
||||||
|
|
||||||
_logger = logging.getLogger(__name__)
|
|
||||||
|
|
||||||
|
|
||||||
def is_true(strval):
|
|
||||||
return bool(strtobool(strval or "0"))
|
|
||||||
|
|
||||||
|
|
||||||
def clean_fs(files):
|
|
||||||
_logger.info("cleaning old files from filestore")
|
|
||||||
for full_path in files:
|
|
||||||
if os.path.exists(full_path):
|
|
||||||
try:
|
|
||||||
os.unlink(full_path)
|
|
||||||
except OSError:
|
|
||||||
# Harmless and needed for race conditions
|
|
||||||
_logger.info(
|
|
||||||
"_file_delete could not unlink %s", full_path, exc_info=True
|
|
||||||
)
|
|
||||||
|
|
||||||
|
|
||||||
class IrAttachment(models.Model):
|
|
||||||
_inherit = "ir.attachment"
|
|
||||||
|
|
||||||
@staticmethod
|
|
||||||
def is_storage_disabled(storage=None, log=True):
|
|
||||||
msg = _("Storages are disabled (see environment configuration).")
|
|
||||||
if storage:
|
|
||||||
msg = _("Storage '%s' is disabled (see environment configuration).") % (
|
|
||||||
storage,
|
|
||||||
)
|
|
||||||
is_disabled = is_true(os.environ.get("DISABLE_ATTACHMENT_STORAGE"))
|
|
||||||
if is_disabled and log:
|
|
||||||
_logger.warning(msg)
|
|
||||||
return is_disabled
|
|
||||||
|
|
||||||
def _register_hook(self):
|
|
||||||
super()._register_hook()
|
|
||||||
location = self.env.context.get("storage_location") or self._storage()
|
|
||||||
# ignore if we are not using an object storage
|
|
||||||
if location not in self._get_stores():
|
|
||||||
return
|
|
||||||
curframe = inspect.currentframe()
|
|
||||||
calframe = inspect.getouterframes(curframe, 2)
|
|
||||||
# the caller of _register_hook is 'load_modules' in
|
|
||||||
# odoo/modules/loading.py
|
|
||||||
load_modules_frame = calframe[1][0]
|
|
||||||
# 'update_module' is an argument that 'load_modules' receives with a
|
|
||||||
# True-ish value meaning that an install or upgrade of addon has been
|
|
||||||
# done during the initialization. We need to move the attachments that
|
|
||||||
# could have been created or updated in other addons before this addon
|
|
||||||
# was loaded
|
|
||||||
update_module = load_modules_frame.f_locals.get("update_module")
|
|
||||||
|
|
||||||
# We need to call the migration on the loading of the model because
|
|
||||||
# when we are upgrading addons, some of them might add attachments.
|
|
||||||
# To be sure they are migrated to the storage we need to call the
|
|
||||||
# migration here.
|
|
||||||
# Typical example is images of ir.ui.menu which are updated in
|
|
||||||
# ir.attachment at every upgrade of the addons
|
|
||||||
if update_module:
|
|
||||||
self.env["ir.attachment"].sudo()._force_storage_to_object_storage()
|
|
||||||
|
|
||||||
@property
|
|
||||||
def _object_storage_default_force_db_config(self):
|
|
||||||
return {"image/": 51200, "application/javascript": 0, "text/css": 0}
|
|
||||||
|
|
||||||
def _get_storage_force_db_config(self):
|
|
||||||
param = (
|
|
||||||
self.env["ir.config_parameter"]
|
|
||||||
.sudo()
|
|
||||||
.get_param(
|
|
||||||
"ir_attachment.storage.force.database",
|
|
||||||
)
|
|
||||||
)
|
|
||||||
storage_config = None
|
|
||||||
if param:
|
|
||||||
try:
|
|
||||||
storage_config = const_eval(param)
|
|
||||||
except (SyntaxError, TypeError, ValueError):
|
|
||||||
_logger.exception(
|
|
||||||
"Could not parse system parameter"
|
|
||||||
" 'ir_attachment.storage.force.database', reverting to the"
|
|
||||||
" default configuration."
|
|
||||||
)
|
|
||||||
|
|
||||||
if not storage_config:
|
|
||||||
storage_config = self._object_storage_default_force_db_config
|
|
||||||
return storage_config
|
|
||||||
|
|
||||||
def _store_in_db_instead_of_object_storage_domain(self):
|
|
||||||
"""Return a domain for attachments that must be forced to DB
|
|
||||||
|
|
||||||
Read the docstring of ``_store_in_db_instead_of_object_storage`` for
|
|
||||||
more details.
|
|
||||||
|
|
||||||
Used in ``force_storage_to_db_for_special_fields`` to find records
|
|
||||||
to move from the object storage to the database.
|
|
||||||
|
|
||||||
The domain must be inline with the conditions in
|
|
||||||
``_store_in_db_instead_of_object_storage``.
|
|
||||||
"""
|
|
||||||
domain = []
|
|
||||||
storage_config = self._get_storage_force_db_config()
|
|
||||||
for mimetype_key, limit in storage_config.items():
|
|
||||||
part = [("mimetype", "=like", f"{mimetype_key}%")]
|
|
||||||
if limit:
|
|
||||||
part = AND([part, [("file_size", "<=", limit)]])
|
|
||||||
domain = OR([domain, part])
|
|
||||||
return domain
|
|
||||||
|
|
||||||
def _store_in_db_instead_of_object_storage(self, data, mimetype):
|
|
||||||
"""Return whether an attachment must be stored in db
|
|
||||||
|
|
||||||
When we are using an Object Storage. This is sometimes required
|
|
||||||
because the object storage is slower than the database/filesystem.
|
|
||||||
|
|
||||||
Small images (128, 256) are used in Odoo in list / kanban views. We
|
|
||||||
want them to be fast to read.
|
|
||||||
They are generally < 50KB (default configuration) so they don't take
|
|
||||||
that much space in database, but they'll be read much faster than from
|
|
||||||
the object storage.
|
|
||||||
|
|
||||||
The assets (application/javascript, text/css) are stored in database
|
|
||||||
as well whatever their size is:
|
|
||||||
|
|
||||||
* a database doesn't have thousands of them
|
|
||||||
* of course better for performance
|
|
||||||
* better portability of a database: when replicating a production
|
|
||||||
instance for dev, the assets are included
|
|
||||||
|
|
||||||
The configuration can be modified in the ir.config_parameter
|
|
||||||
``ir_attachment.storage.force.database``, as a dictionary, for
|
|
||||||
instance::
|
|
||||||
|
|
||||||
{"image/": 51200, "application/javascript": 0, "text/css": 0}
|
|
||||||
|
|
||||||
Where the key is the beginning of the mimetype to configure and the
|
|
||||||
value is the limit in size below which attachments are kept in DB.
|
|
||||||
0 means no limit.
|
|
||||||
|
|
||||||
Default configuration means:
|
|
||||||
|
|
||||||
* images mimetypes (image/png, image/jpeg, ...) below 51200 bytes are
|
|
||||||
stored in database
|
|
||||||
* application/javascript are stored in database whatever their size
|
|
||||||
* text/css are stored in database whatever their size
|
|
||||||
|
|
||||||
The conditions must be inline with the domain in
|
|
||||||
``_store_in_db_instead_of_object_storage_domain``.
|
|
||||||
|
|
||||||
"""
|
|
||||||
if self.is_storage_disabled():
|
|
||||||
return True
|
|
||||||
storage_config = self._get_storage_force_db_config()
|
|
||||||
for mimetype_key, limit in storage_config.items():
|
|
||||||
if mimetype.startswith(mimetype_key):
|
|
||||||
if not limit:
|
|
||||||
return True
|
|
||||||
bin_data = data
|
|
||||||
return len(bin_data) <= limit
|
|
||||||
return False
|
|
||||||
|
|
||||||
def _get_datas_related_values(self, data, mimetype):
|
|
||||||
storage = self.env.context.get("storage_location") or self._storage()
|
|
||||||
if data and storage in self._get_stores():
|
|
||||||
if self._store_in_db_instead_of_object_storage(data, mimetype):
|
|
||||||
# compute the fields that depend on datas
|
|
||||||
bin_data = data
|
|
||||||
values = {
|
|
||||||
"file_size": len(bin_data),
|
|
||||||
"checksum": self._compute_checksum(bin_data),
|
|
||||||
"index_content": self._index(bin_data, mimetype),
|
|
||||||
"store_fname": False,
|
|
||||||
"db_datas": data,
|
|
||||||
}
|
|
||||||
return values
|
|
||||||
return super()._get_datas_related_values(data, mimetype)
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _file_read(self, fname):
|
|
||||||
if self._is_file_from_a_store(fname):
|
|
||||||
return self._store_file_read(fname)
|
|
||||||
else:
|
|
||||||
return super()._file_read(fname)
|
|
||||||
|
|
||||||
def _store_file_read(self, fname):
|
|
||||||
storage = fname.partition("://")[0]
|
|
||||||
raise NotImplementedError(f"No implementation for {storage}")
|
|
||||||
|
|
||||||
def _store_file_write(self, key, bin_data):
|
|
||||||
storage = self.storage()
|
|
||||||
raise NotImplementedError(f"No implementation for {storage}")
|
|
||||||
|
|
||||||
def _store_file_delete(self, fname):
|
|
||||||
storage = fname.partition("://")[0]
|
|
||||||
raise NotImplementedError(f"No implementation for {storage}")
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _file_write(self, bin_data, checksum):
|
|
||||||
location = self.env.context.get("storage_location") or self._storage()
|
|
||||||
if location in self._get_stores():
|
|
||||||
key = self.env.context.get("force_storage_key")
|
|
||||||
if not key:
|
|
||||||
key = self._compute_checksum(bin_data)
|
|
||||||
filename = self._store_file_write(key, bin_data)
|
|
||||||
else:
|
|
||||||
filename = super()._file_write(bin_data, checksum)
|
|
||||||
return filename
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _file_delete(self, fname):
|
|
||||||
if self._is_file_from_a_store(fname):
|
|
||||||
cr = self.env.cr
|
|
||||||
# using SQL to include files hidden through unlink or due to record
|
|
||||||
# rules
|
|
||||||
cr.execute(
|
|
||||||
"SELECT COUNT(*) FROM ir_attachment " "WHERE store_fname = %s", (fname,)
|
|
||||||
)
|
|
||||||
count = cr.fetchone()[0]
|
|
||||||
if not count:
|
|
||||||
self._store_file_delete(fname)
|
|
||||||
else:
|
|
||||||
return super()._file_delete(fname)
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _is_file_from_a_store(self, fname):
|
|
||||||
for store_name in self._get_stores():
|
|
||||||
if self.is_storage_disabled(store_name):
|
|
||||||
continue
|
|
||||||
uri = f"{store_name}://"
|
|
||||||
if fname.startswith(uri):
|
|
||||||
return True
|
|
||||||
return False
|
|
||||||
|
|
||||||
@contextmanager
|
|
||||||
def do_in_new_env(self, new_cr=False):
|
|
||||||
"""Context manager that yields a new environment
|
|
||||||
|
|
||||||
Using a new Odoo Environment thus a new PG transaction.
|
|
||||||
"""
|
|
||||||
if new_cr:
|
|
||||||
registry = odoo.modules.registry.Registry.new(self.env.cr.dbname)
|
|
||||||
with closing(registry.cursor()) as cr:
|
|
||||||
try:
|
|
||||||
yield self.env(cr=cr)
|
|
||||||
except Exception:
|
|
||||||
cr.rollback()
|
|
||||||
raise
|
|
||||||
else:
|
|
||||||
# disable pylint error because this is a valid commit,
|
|
||||||
# we are in a new env
|
|
||||||
cr.commit() # pylint: disable=invalid-commit
|
|
||||||
else:
|
|
||||||
# make a copy
|
|
||||||
yield self.env()
|
|
||||||
|
|
||||||
def _move_attachment_to_store(self):
|
|
||||||
self.ensure_one()
|
|
||||||
_logger.info("inspecting attachment %s (%d)", self.name, self.id)
|
|
||||||
fname = self.store_fname
|
|
||||||
storage = fname.partition("://")[0]
|
|
||||||
if self.is_storage_disabled(storage):
|
|
||||||
fname = False
|
|
||||||
if fname:
|
|
||||||
# migrating from filesystem filestore
|
|
||||||
# or from the old 'store_fname' without the bucket name
|
|
||||||
_logger.info("moving %s on the object storage", fname)
|
|
||||||
self.write(
|
|
||||||
{
|
|
||||||
"datas": self.datas,
|
|
||||||
# this is required otherwise the
|
|
||||||
# mimetype gets overriden with
|
|
||||||
# 'application/octet-stream'
|
|
||||||
# on assets
|
|
||||||
"mimetype": self.mimetype,
|
|
||||||
}
|
|
||||||
)
|
|
||||||
_logger.info("moved %s on the object storage", fname)
|
|
||||||
return self._full_path(fname)
|
|
||||||
elif self.db_datas:
|
|
||||||
_logger.info("moving on the object storage from database")
|
|
||||||
self.write({"datas": self.datas})
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def force_storage(self):
|
|
||||||
if not self.env["res.users"].browse(self.env.uid)._is_admin():
|
|
||||||
raise exceptions.AccessError(
|
|
||||||
_("Only administrators can execute this action.")
|
|
||||||
)
|
|
||||||
location = self.env.context.get("storage_location") or self._storage()
|
|
||||||
if location not in self._get_stores():
|
|
||||||
return super().force_storage()
|
|
||||||
self._force_storage_to_object_storage()
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def force_storage_to_db_for_special_fields(self, new_cr=False):
|
|
||||||
"""Migrate special attachments from Object Storage back to database
|
|
||||||
|
|
||||||
The access to a file stored on the objects storage is slower
|
|
||||||
than a local disk or database access. For attachments like
|
|
||||||
image_small that are accessed in batch for kanban views, this
|
|
||||||
is too slow. We store this type of attachment in the database.
|
|
||||||
|
|
||||||
This method can be used when migrating a filestore where all the files,
|
|
||||||
including the special files (assets, image_small, ...) have been pushed
|
|
||||||
to the Object Storage and we want to write them back in the database.
|
|
||||||
|
|
||||||
It is not called anywhere, but can be called by RPC or scripts.
|
|
||||||
"""
|
|
||||||
storage = self._storage()
|
|
||||||
if self.is_storage_disabled(storage):
|
|
||||||
return
|
|
||||||
if storage not in self._get_stores():
|
|
||||||
return
|
|
||||||
|
|
||||||
domain = AND(
|
|
||||||
(
|
|
||||||
normalize_domain(
|
|
||||||
[
|
|
||||||
("store_fname", "=like", f"{storage}://%"),
|
|
||||||
# for res_field, see comment in
|
|
||||||
# _force_storage_to_object_storage
|
|
||||||
"|",
|
|
||||||
("res_field", "=", False),
|
|
||||||
("res_field", "!=", False),
|
|
||||||
]
|
|
||||||
),
|
|
||||||
normalize_domain(self._store_in_db_instead_of_object_storage_domain()),
|
|
||||||
)
|
|
||||||
)
|
|
||||||
|
|
||||||
with self.do_in_new_env(new_cr=new_cr) as new_env:
|
|
||||||
model_env = new_env["ir.attachment"].with_context(prefetch_fields=False)
|
|
||||||
attachment_ids = model_env.search(domain).ids
|
|
||||||
if not attachment_ids:
|
|
||||||
return
|
|
||||||
total = len(attachment_ids)
|
|
||||||
start_time = time.time()
|
|
||||||
_logger.info(
|
|
||||||
"Moving %d attachments from %s to" " DB for fast access", total, storage
|
|
||||||
)
|
|
||||||
current = 0
|
|
||||||
for attachment_id in attachment_ids:
|
|
||||||
current += 1
|
|
||||||
# if we browse attachments outside of the loop, the first
|
|
||||||
# access to 'datas' will compute all the 'datas' fields at
|
|
||||||
# once, which means reading hundreds or thousands of files at
|
|
||||||
# once, exhausting memory
|
|
||||||
attachment = model_env.browse(attachment_id)
|
|
||||||
# this write will read the datas from the Object Storage and
|
|
||||||
# write them back in the DB (the logic for location to write is
|
|
||||||
# in the 'datas' inverse computed field)
|
|
||||||
attachment.write({"datas": attachment.datas})
|
|
||||||
# as the file will potentially be dropped on the bucket,
|
|
||||||
# we should commit the changes here
|
|
||||||
new_env.cr.commit()
|
|
||||||
if current % 100 == 0 or total - current == 0:
|
|
||||||
_logger.info(
|
|
||||||
"attachment %s/%s after %.2fs",
|
|
||||||
current,
|
|
||||||
total,
|
|
||||||
time.time() - start_time,
|
|
||||||
)
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _force_storage_to_object_storage(self, new_cr=False):
|
|
||||||
_logger.info("migrating files to the object storage")
|
|
||||||
storage = self.env.context.get("storage_location") or self._storage()
|
|
||||||
if self.is_storage_disabled(storage):
|
|
||||||
return
|
|
||||||
# The weird "res_field = False OR res_field != False" domain
|
|
||||||
# is required! It's because of an override of _search in ir.attachment
|
|
||||||
# which adds ('res_field', '=', False) when the domain does not
|
|
||||||
# contain 'res_field'.
|
|
||||||
# https://github.com/odoo/odoo/blob/17.0/odoo/addons/base/models/ir_attachment.py#L523
|
|
||||||
|
|
||||||
domain = [
|
|
||||||
"!",
|
|
||||||
("store_fname", "=like", f"{storage}://%"),
|
|
||||||
"|",
|
|
||||||
("res_field", "=", False),
|
|
||||||
("res_field", "!=", False),
|
|
||||||
]
|
|
||||||
# We do a copy of the environment so we can workaround the cache issue
|
|
||||||
# below. We do not create a new cursor by default because it causes
|
|
||||||
# serialization issues due to concurrent updates on attachments during
|
|
||||||
# the installation
|
|
||||||
with self.do_in_new_env(new_cr=new_cr) as new_env:
|
|
||||||
model_env = new_env["ir.attachment"]
|
|
||||||
ids = model_env.search(domain).ids
|
|
||||||
files_to_clean = []
|
|
||||||
for attachment_id in ids:
|
|
||||||
try:
|
|
||||||
with new_env.cr.savepoint():
|
|
||||||
# check that no other transaction has
|
|
||||||
# locked the row, don't send a file to storage
|
|
||||||
# in that case
|
|
||||||
self.env.cr.execute(
|
|
||||||
"SELECT id "
|
|
||||||
"FROM ir_attachment "
|
|
||||||
"WHERE id = %s "
|
|
||||||
"FOR UPDATE NOWAIT",
|
|
||||||
(attachment_id,),
|
|
||||||
log_exceptions=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
# This is a trick to avoid having the 'datas'
|
|
||||||
# function fields computed for every attachment on
|
|
||||||
# each iteration of the loop. The former issue
|
|
||||||
# being that it reads the content of the file of
|
|
||||||
# ALL the attachments on each loop.
|
|
||||||
new_env.clear()
|
|
||||||
attachment = model_env.browse(attachment_id)
|
|
||||||
path = attachment._move_attachment_to_store()
|
|
||||||
if path:
|
|
||||||
files_to_clean.append(path)
|
|
||||||
except psycopg2.OperationalError:
|
|
||||||
_logger.error(
|
|
||||||
"Could not migrate attachment %s to S3", attachment_id
|
|
||||||
)
|
|
||||||
|
|
||||||
# delete the files from the filesystem once we know the changes
|
|
||||||
# have been committed in ir.attachment
|
|
||||||
if files_to_clean:
|
|
||||||
new_env.cr.commit()
|
|
||||||
clean_fs(files_to_clean)
|
|
||||||
|
|
||||||
def _get_stores(self):
|
|
||||||
"""To get the list of stores activated in the system"""
|
|
||||||
return []
|
|
||||||
@@ -1,21 +0,0 @@
|
|||||||
_MAP = {
|
|
||||||
"y": True,
|
|
||||||
"yes": True,
|
|
||||||
"t": True,
|
|
||||||
"true": True,
|
|
||||||
"on": True,
|
|
||||||
"1": True,
|
|
||||||
"n": False,
|
|
||||||
"no": False,
|
|
||||||
"f": False,
|
|
||||||
"false": False,
|
|
||||||
"off": False,
|
|
||||||
"0": False,
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
def strtobool(value):
|
|
||||||
try:
|
|
||||||
return _MAP[str(value).lower()]
|
|
||||||
except KeyError as error:
|
|
||||||
raise ValueError(f'"{value}" is not a valid bool value') from error
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
[build-system]
|
|
||||||
requires = ["whool"]
|
|
||||||
build-backend = "whool.buildapi"
|
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Base FileURL Field",
|
"name": "Base FileURL Field",
|
||||||
"summary": "Implementation of FileURL type fields",
|
"summary": "Implementation of FileURL type fields",
|
||||||
"version": "17.0.1.0.0",
|
"version": "18.0.1.0.0",
|
||||||
"category": "Technical Settings",
|
"category": "Technical Settings",
|
||||||
"author": "Camptocamp, Odoo Community Association (OCA)",
|
"author": "Camptocamp, Odoo Community Association (OCA)",
|
||||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||||
|
|||||||
@@ -5,7 +5,7 @@
|
|||||||
{
|
{
|
||||||
"name": "Cloud Platform",
|
"name": "Cloud Platform",
|
||||||
"summary": "Addons required for the Camptocamp Cloud Platform",
|
"summary": "Addons required for the Camptocamp Cloud Platform",
|
||||||
"version": "17.0.2.0.0",
|
"version": "18.0.1.0.0",
|
||||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||||
"license": "AGPL-3",
|
"license": "AGPL-3",
|
||||||
"category": "Extra Tools",
|
"category": "Extra Tools",
|
||||||
@@ -17,5 +17,5 @@
|
|||||||
],
|
],
|
||||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||||
"data": [],
|
"data": [],
|
||||||
"installable": False,
|
"installable": True,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,10 +1,9 @@
|
|||||||
# Copyright 2016-2019 Camptocamp SA
|
# Copyright 2016-2025 Camptocamp SA
|
||||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
from collections import namedtuple
|
|
||||||
|
|
||||||
from odoo import api, models
|
from odoo import api, models
|
||||||
from odoo.tools.config import config
|
from odoo.tools.config import config
|
||||||
@@ -18,39 +17,10 @@ def is_true(strval):
|
|||||||
return bool(strtobool(strval or "0"))
|
return bool(strtobool(strval or "0"))
|
||||||
|
|
||||||
|
|
||||||
PlatformConfig = namedtuple("PlatformConfig", "filestore")
|
|
||||||
|
|
||||||
|
|
||||||
FilestoreKind = namedtuple("FilestoreKind", ["name", "location"])
|
|
||||||
|
|
||||||
|
|
||||||
class CloudPlatform(models.AbstractModel):
|
class CloudPlatform(models.AbstractModel):
|
||||||
_name = "cloud.platform"
|
_name = "cloud.platform"
|
||||||
_description = "cloud.platform"
|
_description = "cloud.platform"
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _default_config(self):
|
|
||||||
return PlatformConfig(self._filestore_kinds()["db"])
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _filestore_kinds(self):
|
|
||||||
return {
|
|
||||||
"db": FilestoreKind("db", "local"),
|
|
||||||
"file": FilestoreKind("file", "local"),
|
|
||||||
}
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _platform_kinds(self):
|
|
||||||
return []
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _config_by_server_env(self, platform_kind, environment):
|
|
||||||
configs_getter = getattr(
|
|
||||||
self, "_config_by_server_env_for_%s" % platform_kind, None
|
|
||||||
)
|
|
||||||
configs = configs_getter() if configs_getter else {}
|
|
||||||
return configs.get(environment) or self._default_config()
|
|
||||||
|
|
||||||
def _get_running_env(self):
|
def _get_running_env(self):
|
||||||
environment_name = config["running_env"]
|
environment_name = config["running_env"]
|
||||||
if environment_name.startswith("labs"):
|
if environment_name.startswith("labs"):
|
||||||
@@ -60,25 +30,9 @@ class CloudPlatform(models.AbstractModel):
|
|||||||
return environment_name
|
return environment_name
|
||||||
|
|
||||||
@api.model
|
@api.model
|
||||||
def _install(self, platform_kind):
|
def _install(self, environment):
|
||||||
assert platform_kind in self._platform_kinds()
|
|
||||||
params = self.env["ir.config_parameter"].sudo()
|
|
||||||
params.set_param("cloud.platform.kind", platform_kind)
|
|
||||||
environment_name = self._get_running_env()
|
|
||||||
configs = self._config_by_server_env(platform_kind, environment_name)
|
|
||||||
params.set_param("ir_attachment.location", configs.filestore.name)
|
|
||||||
self.check()
|
self.check()
|
||||||
if configs.filestore.location == "remote":
|
_logger.info(f"cloud platform configured for {environment}")
|
||||||
self.env["ir.attachment"].sudo().force_storage()
|
|
||||||
_logger.info(f"cloud platform configured for {platform_kind}")
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def install(self):
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _check_filestore(self, environment_name):
|
|
||||||
raise NotImplementedError
|
|
||||||
|
|
||||||
@api.model
|
@api.model
|
||||||
def _check_redis(self, environment_name):
|
def _check_redis(self, environment_name):
|
||||||
@@ -113,16 +67,7 @@ class CloudPlatform(models.AbstractModel):
|
|||||||
if is_true(os.environ.get("ODOO_CLOUD_PLATFORM_UNSAFE")):
|
if is_true(os.environ.get("ODOO_CLOUD_PLATFORM_UNSAFE")):
|
||||||
_logger.warning("cloud platform checks disabled, this is not safe")
|
_logger.warning("cloud platform checks disabled, this is not safe")
|
||||||
return
|
return
|
||||||
params = self.env["ir.config_parameter"].sudo()
|
|
||||||
kind = params.get_param("cloud.platform.kind")
|
|
||||||
if not kind:
|
|
||||||
_logger.warning(
|
|
||||||
"cloud platform not configured, you should "
|
|
||||||
"probably run 'env['cloud.platform'].install()'"
|
|
||||||
)
|
|
||||||
return
|
|
||||||
environment_name = self._get_running_env()
|
environment_name = self._get_running_env()
|
||||||
self._check_filestore(environment_name)
|
|
||||||
self._check_redis(environment_name)
|
self._check_redis(environment_name)
|
||||||
|
|
||||||
def _register_hook(self):
|
def _register_hook(self):
|
||||||
|
|||||||
@@ -1,5 +0,0 @@
|
|||||||
# Cloud Platform Azure
|
|
||||||
|
|
||||||
Install addons specific to the Azure setup.
|
|
||||||
|
|
||||||
* The object storage is Azure blob storage
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
from . import models
|
|
||||||
@@ -1,24 +0,0 @@
|
|||||||
# Copyright 2017-2021 Camptocamp SA
|
|
||||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
|
||||||
|
|
||||||
|
|
||||||
{
|
|
||||||
"name": "Cloud Platform Azure",
|
|
||||||
"summary": "Addons required for the Camptocamp Cloud Platform on Azure",
|
|
||||||
"version": "17.0.1.0.0",
|
|
||||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
|
||||||
"license": "AGPL-3",
|
|
||||||
"category": "Extra Tools",
|
|
||||||
"depends": [
|
|
||||||
"cloud_platform",
|
|
||||||
"attachment_azure",
|
|
||||||
"monitoring_prometheus",
|
|
||||||
],
|
|
||||||
"excludes": [
|
|
||||||
"cloud_platform_ovh",
|
|
||||||
"cloud_platform_exoscale",
|
|
||||||
],
|
|
||||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
|
||||||
"data": [],
|
|
||||||
"installable": False,
|
|
||||||
}
|
|
||||||
@@ -1 +0,0 @@
|
|||||||
from . import cloud_platform
|
|
||||||
@@ -1,126 +0,0 @@
|
|||||||
# Copyright 2016-2021 Camptocamp SA
|
|
||||||
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
|
||||||
|
|
||||||
import os
|
|
||||||
import re
|
|
||||||
|
|
||||||
from odoo import api, models
|
|
||||||
|
|
||||||
from odoo.addons.cloud_platform.models.cloud_platform import (
|
|
||||||
FilestoreKind,
|
|
||||||
PlatformConfig,
|
|
||||||
)
|
|
||||||
|
|
||||||
AZURE_STORE_KIND = FilestoreKind("azure", "remote")
|
|
||||||
|
|
||||||
|
|
||||||
class CloudPlatform(models.AbstractModel):
|
|
||||||
_inherit = "cloud.platform"
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _filestore_kinds(self):
|
|
||||||
kinds = super()._filestore_kinds()
|
|
||||||
kinds["azure"] = AZURE_STORE_KIND
|
|
||||||
return kinds
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _platform_kinds(self):
|
|
||||||
kinds = super()._platform_kinds()
|
|
||||||
kinds.append("azure")
|
|
||||||
return kinds
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _config_by_server_env_for_azure(self):
|
|
||||||
fs_kinds = self._filestore_kinds()
|
|
||||||
configs = {
|
|
||||||
"prod": PlatformConfig(filestore=fs_kinds["azure"]),
|
|
||||||
"integration": PlatformConfig(filestore=fs_kinds["azure"]),
|
|
||||||
"labs": PlatformConfig(filestore=fs_kinds["azure"]),
|
|
||||||
"test": PlatformConfig(filestore=fs_kinds["db"]),
|
|
||||||
"dev": PlatformConfig(filestore=fs_kinds["db"]),
|
|
||||||
}
|
|
||||||
return configs
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def _check_filestore(self, environment_name):
|
|
||||||
params = self.env["ir.config_parameter"].sudo()
|
|
||||||
use_azure = params.get_param("ir_attachment.location") == AZURE_STORE_KIND.name
|
|
||||||
if environment_name in ("prod", "integration"):
|
|
||||||
# Labs instances use azure by default, but we don't want
|
|
||||||
# to enforce it in case we want to test something with a different
|
|
||||||
# storage. At your own risks!
|
|
||||||
assert use_azure, (
|
|
||||||
"azure must be used on production and integration instances. "
|
|
||||||
"It is activated by setting 'ir_attachment.location.' to 'azure'."
|
|
||||||
" The 'install()' function sets this option "
|
|
||||||
"automatically."
|
|
||||||
)
|
|
||||||
if use_azure:
|
|
||||||
key_sets = [
|
|
||||||
["AZURE_STORAGE_USE_AAD", "AZURE_STORAGE_ACCOUNT_URL"],
|
|
||||||
["AZURE_STORAGE_CONNECTION_STRING"],
|
|
||||||
[
|
|
||||||
"AZURE_STORAGE_ACCOUNT_NAME",
|
|
||||||
"AZURE_STORAGE_ACCOUNT_URL",
|
|
||||||
"AZURE_STORAGE_ACCOUNT_KEY",
|
|
||||||
],
|
|
||||||
]
|
|
||||||
is_valid = False
|
|
||||||
for key_set in key_sets:
|
|
||||||
if all([os.environ.get(key) for key in key_set]):
|
|
||||||
is_valid = True
|
|
||||||
break
|
|
||||||
assert is_valid, (
|
|
||||||
"When ir_attachment.location is set to 'azure', "
|
|
||||||
"at least one of the following enviromnent variable set "
|
|
||||||
"is required : {}".format(
|
|
||||||
" or ".join(
|
|
||||||
[" + ".join([key for key in key_set]) for key_set in key_sets]
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
storage_name = os.environ.get("AZURE_STORAGE_NAME", "")
|
|
||||||
if environment_name in ("prod", "integration", "labs"):
|
|
||||||
assert storage_name, (
|
|
||||||
"AZURE_STORAGE_NAME environment variable is required when "
|
|
||||||
"ir_attachment.location is 'azure'.\n"
|
|
||||||
"Normally, 'azure' is activated on labs, integration "
|
|
||||||
"and production, but should not be used in dev environment"
|
|
||||||
" (or using a dedicated dev bucket, never using the "
|
|
||||||
"integration/prod bucket).\n"
|
|
||||||
"If you don't actually need a bucket, change the"
|
|
||||||
" 'ir_attachment.location' parameter."
|
|
||||||
)
|
|
||||||
# A bucket name is defined under the following format
|
|
||||||
# ^[a-z]+\-[a-z]+\-\d+$
|
|
||||||
# Anything other than prod bucket must be suffixed with env name
|
|
||||||
#
|
|
||||||
# Use AZURE_STORAGE_NAME_UNSTRUCTURED to by-pass check
|
|
||||||
# on bucket name structure
|
|
||||||
if os.environ.get("AZURE_STORAGE_NAME_UNSTRUCTURED"):
|
|
||||||
return
|
|
||||||
prod_bucket = bool(re.match(r"^[a-z]+\-[a-z]+\-\d+$", storage_name))
|
|
||||||
if environment_name == "prod":
|
|
||||||
assert prod_bucket, (
|
|
||||||
"AZURE_STORAGE_NAME should match '^[a-z]+\\-[a-z]+\\-\\d+$', "
|
|
||||||
f"we got: '{storage_name}'"
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
# if we are using the prod bucket on another instance
|
|
||||||
# such as an integration, we must be sure to be in read only!
|
|
||||||
assert not prod_bucket, (
|
|
||||||
"AZURE_STORAGE_NAME should not match '^[a-z]+\\-[a-z]+\\-\\d+$', "
|
|
||||||
f"we got: '{storage_name}'"
|
|
||||||
)
|
|
||||||
|
|
||||||
elif environment_name == "test":
|
|
||||||
# store in DB so we don't have files local to the host
|
|
||||||
assert params.get_param("ir_attachment.location") == "db", (
|
|
||||||
"In test instances, files must be stored in the database with "
|
|
||||||
"'ir_attachment.location' set to 'db'. This is "
|
|
||||||
"automatically set by the function 'install()'."
|
|
||||||
)
|
|
||||||
|
|
||||||
@api.model
|
|
||||||
def install(self):
|
|
||||||
self._install("azure")
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
[build-system]
|
|
||||||
requires = ["whool"]
|
|
||||||
build-backend = "whool.buildapi"
|
|
||||||
@@ -4,7 +4,7 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
"name": "Monitoring: Requests Logging",
|
"name": "Monitoring: Requests Logging",
|
||||||
"version": "17.0.1.0.0",
|
"version": "18.0.1.0.0",
|
||||||
"author": "Camptocamp,Numigi,Odoo Community Association (OCA)",
|
"author": "Camptocamp,Numigi,Odoo Community Association (OCA)",
|
||||||
"license": "AGPL-3",
|
"license": "AGPL-3",
|
||||||
"category": "category",
|
"category": "category",
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
"name": "Monitoring: Statsd Metrics",
|
"name": "Monitoring: Statsd Metrics",
|
||||||
"version": "17.0.1.0.0",
|
"version": "18.0.1.0.0",
|
||||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||||
"license": "AGPL-3",
|
"license": "AGPL-3",
|
||||||
"category": "category",
|
"category": "category",
|
||||||
|
|||||||
@@ -4,12 +4,12 @@
|
|||||||
|
|
||||||
{
|
{
|
||||||
"name": "Monitoring: Status",
|
"name": "Monitoring: Status",
|
||||||
"version": "17.0.1.0.0",
|
"version": "18.0.1.0.0",
|
||||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||||
"license": "AGPL-3",
|
"license": "AGPL-3",
|
||||||
"category": "category",
|
"category": "category",
|
||||||
"depends": ["base", "web"],
|
"depends": ["base", "web"],
|
||||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||||
"data": [],
|
"data": [],
|
||||||
"installable": False,
|
"installable": True,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ import werkzeug
|
|||||||
|
|
||||||
from odoo import http
|
from odoo import http
|
||||||
|
|
||||||
from odoo.addons.web.controllers.main import ensure_db
|
from odoo.addons.web.controllers.utils import ensure_db
|
||||||
|
|
||||||
|
|
||||||
class HealthCheckFilter(logging.Filter):
|
class HealthCheckFilter(logging.Filter):
|
||||||
|
|||||||
@@ -1,6 +1,4 @@
|
|||||||
# generated from manifests external_dependencies
|
# generated from manifests external_dependencies
|
||||||
azure-identity
|
|
||||||
azure-storage-blob
|
|
||||||
prometheus_client
|
prometheus_client
|
||||||
python-json-logger
|
python-json-logger
|
||||||
redis
|
redis
|
||||||
|
|||||||
@@ -44,6 +44,8 @@ anon_expiration = os.getenv("ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS")
|
|||||||
ssl = os.getenv("ODOO_SESSION_REDIS_SSL", "1")
|
ssl = os.getenv("ODOO_SESSION_REDIS_SSL", "1")
|
||||||
ssl_cert_reqs = os.getenv("ODOO_SESSION_REDIS_SSL_CERT_REQS", "1")
|
ssl_cert_reqs = os.getenv("ODOO_SESSION_REDIS_SSL_CERT_REQS", "1")
|
||||||
redis_cluster = os.getenv("ODOO_SESSION_REDIS_CLUSTER", "0")
|
redis_cluster = os.getenv("ODOO_SESSION_REDIS_CLUSTER", "0")
|
||||||
|
|
||||||
|
|
||||||
@lazy_property
|
@lazy_property
|
||||||
def session_store(self):
|
def session_store(self):
|
||||||
if sentinel_host:
|
if sentinel_host:
|
||||||
@@ -57,12 +59,16 @@ def session_store(self):
|
|||||||
port=port,
|
port=port,
|
||||||
password=password,
|
password=password,
|
||||||
ssl=is_true(ssl),
|
ssl=is_true(ssl),
|
||||||
ssl_cert_reqs=is_true(ssl_cert_reqs))
|
ssl_cert_reqs=is_true(ssl_cert_reqs),
|
||||||
|
)
|
||||||
else:
|
else:
|
||||||
redis_client = redis.Redis(
|
redis_client = redis.Redis(
|
||||||
host=host, port=port, password=password,
|
host=host,
|
||||||
|
port=port,
|
||||||
|
password=password,
|
||||||
ssl=is_true(ssl),
|
ssl=is_true(ssl),
|
||||||
ssl_cert_reqs=is_true(ssl_cert_reqs))
|
ssl_cert_reqs=is_true(ssl_cert_reqs),
|
||||||
|
)
|
||||||
return RedisSessionStore(
|
return RedisSessionStore(
|
||||||
redis=redis_client,
|
redis=redis_client,
|
||||||
prefix=prefix,
|
prefix=prefix,
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
{
|
{
|
||||||
"name": "test base fileurl fields",
|
"name": "test base fileurl fields",
|
||||||
"summary": """A module to verify fileurl field.""",
|
"summary": """A module to verify fileurl field.""",
|
||||||
"version": "17.0.1.0.0",
|
"version": "18.0.1.0.0",
|
||||||
"category": "Tests",
|
"category": "Tests",
|
||||||
"author": "Camptocamp,Odoo Community Association (OCA)",
|
"author": "Camptocamp,Odoo Community Association (OCA)",
|
||||||
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
"website": "https://github.com/camptocamp/odoo-cloud-platform",
|
||||||
|
|||||||
Reference in New Issue
Block a user