From 937e87220a434860d186845fb4e8f67c34ad5a85 Mon Sep 17 00:00:00 2001 From: vrenaville Date: Thu, 12 May 2022 11:04:33 +0200 Subject: [PATCH 01/12] add attachment azure --- attachment_azure/README.rst | 46 +++++ attachment_azure/__init__.py | 4 + attachment_azure/__openerp__.py | 22 +++ attachment_azure/models/__init__.py | 4 + attachment_azure/models/ir_attachment.py | 221 +++++++++++++++++++++++ 5 files changed, 297 insertions(+) create mode 100644 attachment_azure/README.rst create mode 100644 attachment_azure/__init__.py create mode 100644 attachment_azure/__openerp__.py create mode 100644 attachment_azure/models/__init__.py create mode 100644 attachment_azure/models/ir_attachment.py diff --git a/attachment_azure/README.rst b/attachment_azure/README.rst new file mode 100644 index 0000000..b5c26d2 --- /dev/null +++ b/attachment_azure/README.rst @@ -0,0 +1,46 @@ +=========================================== +Attachments on Microsoft Azure Blob Storage +=========================================== + +This addon allows to store the attachments (documents and assets) on `Microsoft Azure +Blob Storage `_. + +Configuration +------------- + +Activate Azure Blob storage: + +* Create or set the system parameter with the key ``ir_attachment.location`` + and the value in the form ``azure``. + +Configure accesses with environment variables: + +* ``AZURE_STORAGE_CONNECTION_STRING`` or +* ``AZURE_STORAGE_ACCOUNT_NAME`` +* ``AZURE_STORAGE_ACCOUNT_URL`` +* ``AZURE_STORAGE_ACCOUNT_KEY`` + +One container will be created per database using the `RUNNING_ENV` environment variable +and the name of the database. By default, `RUNNING_ENV` is set to `dev`. + +The container name can be overridden with environment variable ``AZURE_STORAGE_NAME``. +The strings ``{db}`` and ``{env}`` can be used inside that variable and the values +will be replaced respectively by the database name and environment name. + +The container name will also be stored in the database for each attachment, +and will be used to access the right container in the storage. + +This addon must be added in the server wide addons with (``--load`` option): + +``--load=web,attachment_azure`` + +The System Parameter ``ir_attachment.storage.force.database`` can be customized to +force storage of files in the database. See the documentation of the module +``base_attachment_object_storage``. + +Limitations +----------- + +* You need to call ``env['ir.attachment'].force_storage()`` after + having changed the ``ir_attachment.location`` configuration in order to + migrate the existing attachments to Azure Blob Storage. diff --git a/attachment_azure/__init__.py b/attachment_azure/__init__.py new file mode 100644 index 0000000..49d7105 --- /dev/null +++ b/attachment_azure/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +from . import models diff --git a/attachment_azure/__openerp__.py b/attachment_azure/__openerp__.py new file mode 100644 index 0000000..3570bdc --- /dev/null +++ b/attachment_azure/__openerp__.py @@ -0,0 +1,22 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +{ + "name": "Attachments on Azure storage", + "summary": "Store assets and attachments on a Azure compatible object storage", + "version": "15.0.1.0.0", + "author": "Camptocamp, " + "Open Source Integrators, " + "Serpent Consulting Services, " + "Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Knowledge Management", + "depends": ["base_attachment_object_storage"], + "external_dependencies": { + "python": ["azure-storage-blob", "azure-identity"], + }, + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "installable": True, + "development_status": "Beta", + "maintainers": ["max3903"], +} diff --git a/attachment_azure/models/__init__.py b/attachment_azure/models/__init__.py new file mode 100644 index 0000000..cb9b196 --- /dev/null +++ b/attachment_azure/models/__init__.py @@ -0,0 +1,4 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +from . import ir_attachment diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py new file mode 100644 index 0000000..15fc101 --- /dev/null +++ b/attachment_azure/models/ir_attachment.py @@ -0,0 +1,221 @@ +# Copyright 2016-2019 Camptocamp SA +# Copyright 2021 Open Source Integrators +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +import io +import logging +import os +import re +from datetime import datetime, timedelta + +from odoo import _, api, exceptions, models +from openerp.tools.translate import _ +from openerp.osv import osv +from openerp.osv.orm import except_orm + +_logger = logging.getLogger(__name__) + +try: + from azure.storage.blob import ( + BlobServiceClient, + generate_account_sas, + ResourceTypes, + AccountSasPermissions, + ) + from azure.core.exceptions import ResourceExistsError, HttpResponseError +except ImportError: + _logger.debug("Cannot 'import azure-storage-blob'.") + +try: + from azure.identity import DefaultAzureCredential +except ImportError: + _logger.debug("Cannot 'import azure-identity'.") + + +class IrAttachment(osv.osv): + _inherit = "ir.attachment" + + def _get_stores(self): + l = ["azure"] + l += super(IrAttachment, self)._get_stores() + return l + + @api.model + def _get_blob_service_client(self): + """Connect to Azure and return the blob service client + + The following environment variables must be set: + * ``AZURE_STORAGE_CONNECTION_STRING`` + or + * ``AZURE_STORAGE_ACCOUNT_NAME`` + * ``AZURE_STORAGE_ACCOUNT_URL`` + * ``AZURE_STORAGE_ACCOUNT_KEY`` + or if you want to use AAD (pod identity), set it to 1 or 0 + * ``AZURE_STORAGE_USE_AAD`` + + """ + connect_str = os.environ.get("AZURE_STORAGE_CONNECTION_STRING") + account_name = os.environ.get("AZURE_STORAGE_ACCOUNT_NAME") + account_url = os.environ.get("AZURE_STORAGE_ACCOUNT_URL") + account_key = os.environ.get("AZURE_STORAGE_ACCOUNT_KEY") + account_use_aad = os.environ.get("AZURE_STORAGE_USE_AAD") + if not ( + connect_str + or (account_name and account_url and account_key) + or account_use_aad + ): + msg = _( + "If you want to read from the Azure container, you must provide the " + "following environment variables:\n" + "* AZURE_STORAGE_CONNECTION_STRING\n" + "or\n" + "* AZURE_STORAGE_ACCOUNT_NAME\n" + "* AZURE_STORAGE_ACCOUNT_URL\n" + "* AZURE_STORAGE_ACCOUNT_KEY\n" + "or\n" + "* AZURE_STORAGE_USE_AAD\n" + ) + raise exceptions.UserError(msg) + blob_service_client = None + if account_use_aad: + token_credential = DefaultAzureCredential() + blob_service_client = BlobServiceClient( + account_url=account_url, credential=token_credential + ) + elif connect_str: + try: + blob_service_client = BlobServiceClient.from_connection_string( + connect_str + ) + except HttpResponseError as error: + _logger.exception( + "Error during the connection to Azure container using the " + "connection string." + ) + raise exceptions.UserError(str(error)) + else: + try: + sas_token = generate_account_sas( + account_name=account_name, + account_key=account_key, + resource_types=ResourceTypes(container=True, object=True), + permission=AccountSasPermissions(read=True, write=True), + expiry=datetime.utcnow() + timedelta(hours=1), + ) + blob_service_client = BlobServiceClient( + account_url=account_url, + credential=sas_token, + ) + except HttpResponseError as error: + _logger.exception( + "Error during the connection to Azure container using the Shared " + "Access Signature (SAS)" + ) + raise exceptions.UserError(str(error)) + return blob_service_client + + @api.model + def _get_container_name(self): + """ + Container naming rules: + https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names + """ + running_env = os.environ.get("RUNNING_ENV", "dev") + storage_name = os.environ.get("AZURE_STORAGE_NAME", r"{env}-{db}") + storage_name = storage_name.format(env=running_env, db=self.env.cr.dbname) + # replace invalid characters by _ + storage_name = re.sub(r"[\W_]+", "-", storage_name) + # lowercase, max 63 chars + return str.lower(storage_name)[:63] + + @api.model + def _get_azure_container(self, container_name=None): + if not container_name: + container_name = self._get_container_name() + try: + blob_service_client = self._get_blob_service_client() + except exceptions.UserError: + _logger.exception( + "error accessing to storage '%s' please check credentials ", + container_name, + ) + return False + container_client = blob_service_client.get_container_client(container_name) + if not container_client.exists(): + try: + # Create the container + container_client.create_container() + except HttpResponseError as error: + _logger.exception("Error during the creation of the Azure container") + raise exceptions.UserError(str(error)) + return container_client + + @api.model + def _store_file_read(self, fname, bin_size=False): + if fname.startswith("azure://"): + key = fname.replace("azure://", "", 1).lower() + if "/" in key: + container_name, key = key.split("/", 1) + else: + container_name = None + container_client = self._get_azure_container(container_name) + # if container cannot be retrived, abort reading from azure storage + if not container_client: + return "" + try: + blob_client = container_client.get_blob_client(key) + read = blob_client.download_blob().readall() + except HttpResponseError: + read = "" + _logger.info("Attachment '%s' missing on object storage", fname) + return read + else: + return super(IrAttachment, self)._store_file_read(fname, bin_size) + + @api.model + def _store_file_write(self, key, bin_data): + location = self.env.context.get("storage_location") or self._storage() + if location == "azure": + container_client = self._get_azure_container() + filename = "azure://%s/%s" % (container_client.container_name, key) + with io.BytesIO() as file: + blob_client = container_client.get_blob_client(key.lower()) + file.write(bin_data) + file.seek(0) + try: + blob_client.upload_blob(file, blob_type="BlockBlob") + except ResourceExistsError: + pass + except HttpResponseError as error: + # log verbose error from azure, return short message for user + _logger.exception("Error during storage of the file %s" % filename) + raise exceptions.UserError( + _("The file could not be stored: %s") % str(error) + ) + else: + _super = super(IrAttachment, self) + filename = _super._store_file_write(key, bin_data) + return filename + + @api.model + def _store_file_delete(self, fname): + if fname.startswith("azure://"): + key = fname.replace("azure://", "", 1).lower() + if "/" in key: + container_name, key = key.split("/", 1) + else: + container_name = None + container_client = self._get_azure_container(container_name) + if not container_client: + return "" + # delete the file only if it is on the current configured container + # otherwise, we might delete files used on a different environment + try: + blob_client = container_client.get_blob_client(key) + blob_client.delete_blob() + _logger.info("File %s deleted on the object storage" % (fname)) + except HttpResponseError: + # log verbose error from azure, return short message for + # user + _logger.exception("Error during deletion of the file %s" % fname) + else: + super(IrAttachment, self)._store_file_delete(fname) From 19da00949af95328396ef46dd51a5031623df0de Mon Sep 17 00:00:00 2001 From: vrenaville Date: Thu, 12 May 2022 11:33:25 +0200 Subject: [PATCH 02/12] feat: add prometheus + cloud_plateform azure --- cloud_platform/models/cloud_platform.py | 88 +++++++++++++++++++ cloud_platform_azure/README.md | 6 ++ cloud_platform_azure/__init__.py | 1 + cloud_platform_azure/__openerp__.py | 24 +++++ cloud_platform_azure/models/__init__.py | 1 + cloud_platform_azure/models/cloud_platform.py | 39 ++++++++ monitoring_prometheus/README.rst | 17 ++++ monitoring_prometheus/__init__.py | 2 + monitoring_prometheus/__openerp__.py | 22 +++++ monitoring_prometheus/controllers/__init__.py | 1 + .../controllers/prometheus_metrics.py | 11 +++ monitoring_prometheus/models/__init__.py | 1 + monitoring_prometheus/models/ir_http.py | 40 +++++++++ 13 files changed, 253 insertions(+) create mode 100644 cloud_platform_azure/README.md create mode 100644 cloud_platform_azure/__init__.py create mode 100644 cloud_platform_azure/__openerp__.py create mode 100644 cloud_platform_azure/models/__init__.py create mode 100644 cloud_platform_azure/models/cloud_platform.py create mode 100644 monitoring_prometheus/README.rst create mode 100644 monitoring_prometheus/__init__.py create mode 100644 monitoring_prometheus/__openerp__.py create mode 100644 monitoring_prometheus/controllers/__init__.py create mode 100644 monitoring_prometheus/controllers/prometheus_metrics.py create mode 100644 monitoring_prometheus/models/__init__.py create mode 100644 monitoring_prometheus/models/ir_http.py diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py index 06de6d3..44026f2 100644 --- a/cloud_platform/models/cloud_platform.py +++ b/cloud_platform/models/cloud_platform.py @@ -43,6 +43,12 @@ class CloudPlatform(osv.osv_abstract): # it in cloud_platform_exoscale in V11 return ['exoscale'] + def _filestore_kinds(self): + # XXX for backward compatibility, we need this one here, move + # it in cloud_platform_exoscale in V11 + return ['exoscale'] + + # XXX for backward compatibility, we need this one here, move # it in cloud_platform_exoscale in V11 def _config_by_server_env_for_exoscale(self): @@ -220,6 +226,86 @@ class CloudPlatform(osv.osv_abstract): "automatically set by the function 'install_exoscale()'." ) + def _check_azure(self, cr, uid, environment_name, context=None): + params = self.env["ir.config_parameter"].sudo() + use_azure = params.get_param("ir_attachment.location") == AZURE_STORE_KIND.name + if environment_name in ("prod", "integration"): + # Labs instances use azure by default, but we don't want + # to enforce it in case we want to test something with a different + # storage. At your own risks! + assert use_azure, ( + "azure must be used on production and integration instances. " + "It is activated by setting 'ir_attachment.location.' to 'azure'." + " The 'install()' function sets this option " + "automatically." + ) + if use_azure: + key_sets = [ + ["AZURE_STORAGE_USE_AAD", "AZURE_STORAGE_ACCOUNT_URL"], + ["AZURE_STORAGE_CONNECTION_STRING"], + [ + "AZURE_STORAGE_ACCOUNT_NAME", + "AZURE_STORAGE_ACCOUNT_URL", + "AZURE_STORAGE_ACCOUNT_KEY", + ], + ] + is_valid = False + for key_set in key_sets: + if all([os.environ.get(key) for key in key_set]): + is_valid = True + break + assert is_valid, ( + "When ir_attachment.location is set to 'azure', " + "at least one of the following enviromnent variable set " + "is required : {}".format( + " or ".join( + [" + ".join([key for key in key_set]) for key_set in key_sets] + ) + ) + ) + storage_name = os.environ.get("AZURE_STORAGE_NAME", "") + if environment_name in ("prod", "integration", "labs"): + assert storage_name, ( + "AZURE_STORAGE_NAME environment variable is required when " + "ir_attachment.location is 'azure'.\n" + "Normally, 'azure' is activated on labs, integration " + "and production, but should not be used in dev environment" + " (or using a dedicated dev bucket, never using the " + "integration/prod bucket).\n" + "If you don't actually need a bucket, change the" + " 'ir_attachment.location' parameter." + ) + # A bucket name is defined under the following format + # ^[a-z]+\-[a-z]+\-\d+$ + # Anything other than prod bucket must be suffixed with env name + # + # Use AZURE_STORAGE_NAME_UNSTRUCTURED to by-pass check + # on bucket name structure + if os.environ.get("AZURE_STORAGE_NAME_UNSTRUCTURED"): + return + prod_bucket = bool(re.match(r"^[a-z]+\-[a-z]+\-\d+$", storage_name)) + if environment_name == "prod": + assert prod_bucket, ( + "AZURE_STORAGE_NAME should match '^[a-z]+\\-[a-z]+\\-\\d+$', " + "we got: '%s'" % (storage_name,) + ) + else: + # if we are using the prod bucket on another instance + # such as an integration, we must be sure to be in read only! + assert not prod_bucket, ( + "AZURE_STORAGE_NAME should not match '^[a-z]+\\-[a-z]+\\-\\d+$', " + "we got: '%s'" % (storage_name,) + ) + + elif environment_name == "test": + # store in DB so we don't have files local to the host + assert params.get_param("ir_attachment.location") == "db", ( + "In test instances, files must be stored in the database with " + "'ir_attachment.location' set to 'db'. This is " + "automatically set by the function 'install()'." + ) + + def _check_redis(self, cr, uid, environment_name, context=None): if environment_name in ('prod', 'integration', 'labs', 'test'): assert is_true(os.environ.get('ODOO_SESSION_REDIS')), ( @@ -262,6 +348,8 @@ class CloudPlatform(osv.osv_abstract): self._check_s3(cr, uid, environment_name, context) elif kind == 'ovh': self._check_swift(cr, uid, environment_name, context) + elif kind == 'azure': + self._check_azure(cr, uid, environment_name, context) self._check_redis(cr, uid, environment_name, context) def _register_hook(self, cr): diff --git a/cloud_platform_azure/README.md b/cloud_platform_azure/README.md new file mode 100644 index 0000000..1f7bd5d --- /dev/null +++ b/cloud_platform_azure/README.md @@ -0,0 +1,6 @@ +Cloud Platform Azure +==================== + +Install addons specific to the Azure setup. + + * The object storage is Azure blob storage diff --git a/cloud_platform_azure/__init__.py b/cloud_platform_azure/__init__.py new file mode 100644 index 0000000..0650744 --- /dev/null +++ b/cloud_platform_azure/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/cloud_platform_azure/__openerp__.py b/cloud_platform_azure/__openerp__.py new file mode 100644 index 0000000..5cfa373 --- /dev/null +++ b/cloud_platform_azure/__openerp__.py @@ -0,0 +1,24 @@ +# Copyright 2017-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{ + "name": "Cloud Platform Azure", + "summary": "Addons required for the Camptocamp Cloud Platform on Azure", + "version": "15.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Extra Tools", + "depends": [ + "cloud_platform", + "attachment_azure", + "monitoring_prometheus", + ], + "excludes": [ + "cloud_platform_ovh", + "cloud_platform_exoscale", + ], + "website": "https://www.camptocamp.com", + "data": [], + "installable": True, +} diff --git a/cloud_platform_azure/models/__init__.py b/cloud_platform_azure/models/__init__.py new file mode 100644 index 0000000..5d08f36 --- /dev/null +++ b/cloud_platform_azure/models/__init__.py @@ -0,0 +1 @@ +from . import cloud_platform diff --git a/cloud_platform_azure/models/cloud_platform.py b/cloud_platform_azure/models/cloud_platform.py new file mode 100644 index 0000000..472ebc6 --- /dev/null +++ b/cloud_platform_azure/models/cloud_platform.py @@ -0,0 +1,39 @@ +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import re +import os + +from openerp.osv import osv +from openerp.addons.cloud_platform.models.cloud_platform import FilestoreKind +from openerp.addons.cloud_platform.models.cloud_platform import PlatformConfig + +AZURE_STORE_KIND = FilestoreKind("azure", "remote") + + +class CloudPlatform(osv.osv): + _inherit = "cloud.platform" + + def _filestore_kinds(self): + kinds = super(CloudPlatform, self)._filestore_kinds() + kinds["azure"] = AZURE_STORE_KIND + return kinds + + def _platform_kinds(self): + kinds = super(CloudPlatform, self)._platform_kinds() + kinds.append("azure") + return kinds + + def _config_by_server_env_for_azure(self): + fs_kinds = self._filestore_kinds() + configs = { + "prod": PlatformConfig(filestore=fs_kinds["azure"]), + "integration": PlatformConfig(filestore=fs_kinds["azure"]), + "labs": PlatformConfig(filestore=fs_kinds["azure"]), + "test": PlatformConfig(filestore=fs_kinds["db"]), + "dev": PlatformConfig(filestore=fs_kinds["db"]), + } + return configs + + def install_azure(self, cr, uid, context=None): + self.install(cr, uid, "azure", context) diff --git a/monitoring_prometheus/README.rst b/monitoring_prometheus/README.rst new file mode 100644 index 0000000..aa98ffe --- /dev/null +++ b/monitoring_prometheus/README.rst @@ -0,0 +1,17 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :alt: License + +============================== +Monitoring: Prometheus metrics +============================== + +Add an endpoint */metrics* to allow a Prometheus server to fetch application metrics. +Current available metrics are: + +* Request completion time with 3 differentiators: + * Filestore + * Assets + * Everything else +* Longpolling request count + +No additional configuration is needed, just ensure that the Prometheus server is allowed to communicate with Odoo diff --git a/monitoring_prometheus/__init__.py b/monitoring_prometheus/__init__.py new file mode 100644 index 0000000..91c5580 --- /dev/null +++ b/monitoring_prometheus/__init__.py @@ -0,0 +1,2 @@ +from . import controllers +from . import models diff --git a/monitoring_prometheus/__openerp__.py b/monitoring_prometheus/__openerp__.py new file mode 100644 index 0000000..260570c --- /dev/null +++ b/monitoring_prometheus/__openerp__.py @@ -0,0 +1,22 @@ +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{ + "name": "Monitoring: Prometheus Metrics", + "version": "15.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "category", + "depends": [ + "base", + "web", + "server_environment", + ], + "website": "http://www.camptocamp.com", + "data": [], + "external_dependencies": { + "python": ["prometheus_client"], + }, + "installable": True, +} diff --git a/monitoring_prometheus/controllers/__init__.py b/monitoring_prometheus/controllers/__init__.py new file mode 100644 index 0000000..13ff72f --- /dev/null +++ b/monitoring_prometheus/controllers/__init__.py @@ -0,0 +1 @@ +from . import prometheus_metrics diff --git a/monitoring_prometheus/controllers/prometheus_metrics.py b/monitoring_prometheus/controllers/prometheus_metrics.py new file mode 100644 index 0000000..fae6d36 --- /dev/null +++ b/monitoring_prometheus/controllers/prometheus_metrics.py @@ -0,0 +1,11 @@ +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from openerp.http import Controller, route +from prometheus_client import generate_latest + + +class PrometheusController(Controller): + @route("/metrics", auth="public") + def metrics(self): + return generate_latest() diff --git a/monitoring_prometheus/models/__init__.py b/monitoring_prometheus/models/__init__.py new file mode 100644 index 0000000..9a5eb71 --- /dev/null +++ b/monitoring_prometheus/models/__init__.py @@ -0,0 +1 @@ +from . import ir_http diff --git a/monitoring_prometheus/models/ir_http.py b/monitoring_prometheus/models/ir_http.py new file mode 100644 index 0000000..67d76ad --- /dev/null +++ b/monitoring_prometheus/models/ir_http.py @@ -0,0 +1,40 @@ +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from openerp import osv +from openerp.http import request +from prometheus_client import Summary, Counter + + +REQUEST_TIME = Summary( + "request_latency_sec", "Request response time in sec", ["query_type"] +) +LONGPOLLING_COUNT = Counter("longpolling", "Longpolling request count") + + +class IrHttp(osv.osv_abstract): + _inherit = "ir.http" + + @classmethod + def _dispatch(cls): + path_info = request.httprequest.environ.get("PATH_INFO") + + if path_info.startswith("/longpolling/"): + LONGPOLLING_COUNT.inc() + return super(IrHttp, self)._dispatch() + + if path_info.startswith("/metrics"): + return super(IrHttp, self)._dispatch() + + if path_info.startswith("/web/static"): + label = "assets" + elif path_info.startswith("/web/content"): + label = "filestore" + else: + label = "client" + + res = None + with REQUEST_TIME.labels(label).time(): + res = super(IrHttp, self)._dispatch() + + return res From e7402a80a4f78a6786c3d6e8c1d215fcaa24deea Mon Sep 17 00:00:00 2001 From: vrenaville Date: Thu, 12 May 2022 14:27:13 +0200 Subject: [PATCH 03/12] fix --- attachment_azure/__openerp__.py | 3 --- attachment_azure/models/ir_attachment.py | 1 - 2 files changed, 4 deletions(-) diff --git a/attachment_azure/__openerp__.py b/attachment_azure/__openerp__.py index 3570bdc..7ada22d 100644 --- a/attachment_azure/__openerp__.py +++ b/attachment_azure/__openerp__.py @@ -12,9 +12,6 @@ "license": "AGPL-3", "category": "Knowledge Management", "depends": ["base_attachment_object_storage"], - "external_dependencies": { - "python": ["azure-storage-blob", "azure-identity"], - }, "website": "https://github.com/camptocamp/odoo-cloud-platform", "installable": True, "development_status": "Beta", diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py index 15fc101..53dc08a 100644 --- a/attachment_azure/models/ir_attachment.py +++ b/attachment_azure/models/ir_attachment.py @@ -7,7 +7,6 @@ import os import re from datetime import datetime, timedelta -from odoo import _, api, exceptions, models from openerp.tools.translate import _ from openerp.osv import osv from openerp.osv.orm import except_orm From fde3021cd48755f01a82f6f76b86fe37190517d6 Mon Sep 17 00:00:00 2001 From: vrenaville Date: Thu, 12 May 2022 14:36:14 +0200 Subject: [PATCH 04/12] fea --- attachment_azure/models/ir_attachment.py | 21 ++++++++------------- 1 file changed, 8 insertions(+), 13 deletions(-) diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py index 53dc08a..40c2072 100644 --- a/attachment_azure/models/ir_attachment.py +++ b/attachment_azure/models/ir_attachment.py @@ -38,7 +38,6 @@ class IrAttachment(osv.osv): l += super(IrAttachment, self)._get_stores() return l - @api.model def _get_blob_service_client(self): """Connect to Azure and return the blob service client @@ -73,7 +72,7 @@ class IrAttachment(osv.osv): "or\n" "* AZURE_STORAGE_USE_AAD\n" ) - raise exceptions.UserError(msg) + raise osv.except_osv(_("UserError"), msg) blob_service_client = None if account_use_aad: token_credential = DefaultAzureCredential() @@ -90,7 +89,7 @@ class IrAttachment(osv.osv): "Error during the connection to Azure container using the " "connection string." ) - raise exceptions.UserError(str(error)) + raise osv.except_osv(_("UserError"), str(error)) else: try: sas_token = generate_account_sas( @@ -109,10 +108,9 @@ class IrAttachment(osv.osv): "Error during the connection to Azure container using the Shared " "Access Signature (SAS)" ) - raise exceptions.UserError(str(error)) + raise osv.except_osv(_("UserError"), str(error)) return blob_service_client - @api.model def _get_container_name(self): """ Container naming rules: @@ -126,13 +124,12 @@ class IrAttachment(osv.osv): # lowercase, max 63 chars return str.lower(storage_name)[:63] - @api.model def _get_azure_container(self, container_name=None): if not container_name: container_name = self._get_container_name() try: blob_service_client = self._get_blob_service_client() - except exceptions.UserError: + except Exception: _logger.exception( "error accessing to storage '%s' please check credentials ", container_name, @@ -145,10 +142,9 @@ class IrAttachment(osv.osv): container_client.create_container() except HttpResponseError as error: _logger.exception("Error during the creation of the Azure container") - raise exceptions.UserError(str(error)) + raise osv.except_osv(_("UserError"), str(error)) return container_client - @api.model def _store_file_read(self, fname, bin_size=False): if fname.startswith("azure://"): key = fname.replace("azure://", "", 1).lower() @@ -170,7 +166,6 @@ class IrAttachment(osv.osv): else: return super(IrAttachment, self)._store_file_read(fname, bin_size) - @api.model def _store_file_write(self, key, bin_data): location = self.env.context.get("storage_location") or self._storage() if location == "azure": @@ -187,15 +182,15 @@ class IrAttachment(osv.osv): except HttpResponseError as error: # log verbose error from azure, return short message for user _logger.exception("Error during storage of the file %s" % filename) - raise exceptions.UserError( - _("The file could not be stored: %s") % str(error) + raise osv.except_osv( + _("UserError"), + _("The file could not be stored: %s") % str(error), ) else: _super = super(IrAttachment, self) filename = _super._store_file_write(key, bin_data) return filename - @api.model def _store_file_delete(self, fname): if fname.startswith("azure://"): key = fname.replace("azure://", "", 1).lower() From 0b4d6f2965735ae4d560794a4f3133ae62c8b354 Mon Sep 17 00:00:00 2001 From: vrenaville Date: Thu, 12 May 2022 14:38:02 +0200 Subject: [PATCH 05/12] fix --- cloud_platform_azure/__openerp__.py | 1 - monitoring_prometheus/README.rst | 17 -------- monitoring_prometheus/__init__.py | 2 - monitoring_prometheus/__openerp__.py | 22 ---------- monitoring_prometheus/controllers/__init__.py | 1 - .../controllers/prometheus_metrics.py | 11 ----- monitoring_prometheus/models/__init__.py | 1 - monitoring_prometheus/models/ir_http.py | 40 ------------------- 8 files changed, 95 deletions(-) delete mode 100644 monitoring_prometheus/README.rst delete mode 100644 monitoring_prometheus/__init__.py delete mode 100644 monitoring_prometheus/__openerp__.py delete mode 100644 monitoring_prometheus/controllers/__init__.py delete mode 100644 monitoring_prometheus/controllers/prometheus_metrics.py delete mode 100644 monitoring_prometheus/models/__init__.py delete mode 100644 monitoring_prometheus/models/ir_http.py diff --git a/cloud_platform_azure/__openerp__.py b/cloud_platform_azure/__openerp__.py index 5cfa373..d6c2263 100644 --- a/cloud_platform_azure/__openerp__.py +++ b/cloud_platform_azure/__openerp__.py @@ -12,7 +12,6 @@ "depends": [ "cloud_platform", "attachment_azure", - "monitoring_prometheus", ], "excludes": [ "cloud_platform_ovh", diff --git a/monitoring_prometheus/README.rst b/monitoring_prometheus/README.rst deleted file mode 100644 index aa98ffe..0000000 --- a/monitoring_prometheus/README.rst +++ /dev/null @@ -1,17 +0,0 @@ -.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg - :alt: License - -============================== -Monitoring: Prometheus metrics -============================== - -Add an endpoint */metrics* to allow a Prometheus server to fetch application metrics. -Current available metrics are: - -* Request completion time with 3 differentiators: - * Filestore - * Assets - * Everything else -* Longpolling request count - -No additional configuration is needed, just ensure that the Prometheus server is allowed to communicate with Odoo diff --git a/monitoring_prometheus/__init__.py b/monitoring_prometheus/__init__.py deleted file mode 100644 index 91c5580..0000000 --- a/monitoring_prometheus/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from . import controllers -from . import models diff --git a/monitoring_prometheus/__openerp__.py b/monitoring_prometheus/__openerp__.py deleted file mode 100644 index 260570c..0000000 --- a/monitoring_prometheus/__openerp__.py +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2016-2021 Camptocamp SA -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - - -{ - "name": "Monitoring: Prometheus Metrics", - "version": "15.0.1.0.0", - "author": "Camptocamp,Odoo Community Association (OCA)", - "license": "AGPL-3", - "category": "category", - "depends": [ - "base", - "web", - "server_environment", - ], - "website": "http://www.camptocamp.com", - "data": [], - "external_dependencies": { - "python": ["prometheus_client"], - }, - "installable": True, -} diff --git a/monitoring_prometheus/controllers/__init__.py b/monitoring_prometheus/controllers/__init__.py deleted file mode 100644 index 13ff72f..0000000 --- a/monitoring_prometheus/controllers/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import prometheus_metrics diff --git a/monitoring_prometheus/controllers/prometheus_metrics.py b/monitoring_prometheus/controllers/prometheus_metrics.py deleted file mode 100644 index fae6d36..0000000 --- a/monitoring_prometheus/controllers/prometheus_metrics.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright 2016-2021 Camptocamp SA -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - -from openerp.http import Controller, route -from prometheus_client import generate_latest - - -class PrometheusController(Controller): - @route("/metrics", auth="public") - def metrics(self): - return generate_latest() diff --git a/monitoring_prometheus/models/__init__.py b/monitoring_prometheus/models/__init__.py deleted file mode 100644 index 9a5eb71..0000000 --- a/monitoring_prometheus/models/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from . import ir_http diff --git a/monitoring_prometheus/models/ir_http.py b/monitoring_prometheus/models/ir_http.py deleted file mode 100644 index 67d76ad..0000000 --- a/monitoring_prometheus/models/ir_http.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2016-2021 Camptocamp SA -# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - -from openerp import osv -from openerp.http import request -from prometheus_client import Summary, Counter - - -REQUEST_TIME = Summary( - "request_latency_sec", "Request response time in sec", ["query_type"] -) -LONGPOLLING_COUNT = Counter("longpolling", "Longpolling request count") - - -class IrHttp(osv.osv_abstract): - _inherit = "ir.http" - - @classmethod - def _dispatch(cls): - path_info = request.httprequest.environ.get("PATH_INFO") - - if path_info.startswith("/longpolling/"): - LONGPOLLING_COUNT.inc() - return super(IrHttp, self)._dispatch() - - if path_info.startswith("/metrics"): - return super(IrHttp, self)._dispatch() - - if path_info.startswith("/web/static"): - label = "assets" - elif path_info.startswith("/web/content"): - label = "filestore" - else: - label = "client" - - res = None - with REQUEST_TIME.labels(label).time(): - res = super(IrHttp, self)._dispatch() - - return res From 43ed7178af0ece489e0ccb46d9ca0479a9b54a6c Mon Sep 17 00:00:00 2001 From: vrenaville Date: Thu, 12 May 2022 14:43:54 +0200 Subject: [PATCH 06/12] fix --- attachment_azure/models/ir_attachment.py | 5 ++--- cloud_platform_azure/models/cloud_platform.py | 4 +--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py index 40c2072..b6aaebc 100644 --- a/attachment_azure/models/ir_attachment.py +++ b/attachment_azure/models/ir_attachment.py @@ -166,9 +166,8 @@ class IrAttachment(osv.osv): else: return super(IrAttachment, self)._store_file_read(fname, bin_size) - def _store_file_write(self, key, bin_data): - location = self.env.context.get("storage_location") or self._storage() - if location == "azure": + def _store_file_write(self, storage, key, bin_data): + if storage == "azure": container_client = self._get_azure_container() filename = "azure://%s/%s" % (container_client.container_name, key) with io.BytesIO() as file: diff --git a/cloud_platform_azure/models/cloud_platform.py b/cloud_platform_azure/models/cloud_platform.py index 472ebc6..8fd0233 100644 --- a/cloud_platform_azure/models/cloud_platform.py +++ b/cloud_platform_azure/models/cloud_platform.py @@ -8,15 +8,13 @@ from openerp.osv import osv from openerp.addons.cloud_platform.models.cloud_platform import FilestoreKind from openerp.addons.cloud_platform.models.cloud_platform import PlatformConfig -AZURE_STORE_KIND = FilestoreKind("azure", "remote") - class CloudPlatform(osv.osv): _inherit = "cloud.platform" def _filestore_kinds(self): kinds = super(CloudPlatform, self)._filestore_kinds() - kinds["azure"] = AZURE_STORE_KIND + kinds.append("azure") return kinds def _platform_kinds(self): From 18d07353d9e50095ae223bb1dab3d43e7701a8ff Mon Sep 17 00:00:00 2001 From: vrenaville Date: Tue, 17 May 2022 13:31:17 +0200 Subject: [PATCH 07/12] fix --- .../models/ir_attachment.py | 123 ++++++++---------- 1 file changed, 55 insertions(+), 68 deletions(-) diff --git a/base_attachment_object_storage/models/ir_attachment.py b/base_attachment_object_storage/models/ir_attachment.py index 8ada85f..a857407 100644 --- a/base_attachment_object_storage/models/ir_attachment.py +++ b/base_attachment_object_storage/models/ir_attachment.py @@ -20,21 +20,19 @@ _logger = logging.getLogger(__name__) def clean_fs(files): - _logger.info('cleaning old files from filestore') + _logger.info("cleaning old files from filestore") for full_path in files: if os.path.exists(full_path): try: os.unlink(full_path) except OSError: _logger.info( - "_file_delete could not unlink %s", - full_path, exc_info=True + "_file_delete could not unlink %s", full_path, exc_info=True ) except IOError: # Harmless and needed for race conditions _logger.info( - "_file_delete could not unlink %s", - full_path, exc_info=True + "_file_delete could not unlink %s", full_path, exc_info=True ) @@ -52,34 +50,32 @@ def savepoint(cursor): class IrAttachment(osv.osv): - _inherit = 'ir.attachment' + _inherit = "ir.attachment" @staticmethod def _compute_checksum(bin_data): - """ compute the checksum for the given datas - :param bin_data : datas in its binary form + """compute the checksum for the given datas + :param bin_data : datas in its binary form """ # an empty file has a checksum too (for caching) - return hashlib.sha1(bin_data or '').hexdigest() + return hashlib.sha1(bin_data or "").hexdigest() def _is_user_admin(self, cr, uid): if uid == SUPERUSER_ID: return True else: - return self.pool.get('res.users').has_group( - cr, uid, 'base.group_erp_manager' + return self.pool.get("res.users").has_group( + cr, uid, "base.group_erp_manager" ) def _storage(self, cr, uid, context=None): - return self.pool['ir.config_parameter'].get_param( - cr, SUPERUSER_ID, 'ir_attachment.location', 'file' + return self.pool["ir.config_parameter"].get_param( + cr, SUPERUSER_ID, "ir_attachment.location", "file" ) def _full_path(self, cr, uid, location, path): # Hack to allow filestore migration from local filesystem to any remote - return super(IrAttachment, self)._full_path( - cr, uid, 'file://filestore', path - ) + return super(IrAttachment, self)._full_path(cr, uid, "file://filestore", path) def _register_hook(self, cr): super(IrAttachment, self)._register_hook(cr) @@ -101,7 +97,7 @@ class IrAttachment(osv.osv): # done during the initialization. We need to move the attachments that # could have been created or updated in other addons before this addon # was loaded - update_module = load_modules_frame.f_locals.get('update_module') + update_module = load_modules_frame.f_locals.get("update_module") # We need to call the migration on the loading of the model because # when we are upgrading addons, some of them might add attachments. @@ -110,12 +106,12 @@ class IrAttachment(osv.osv): # Typical example is images of ir.ui.menu which are updated in # ir.attachment at every upgrade of the addons if update_module: - self.pool.get('ir.attachment')._force_storage_to_object_storage( + self.pool.get("ir.attachment")._force_storage_to_object_storage( cr, SUPERUSER_ID ) def _save_in_db_anyway(self, cr, uid, ids, context=None): - """ Return whether an attachment must be stored in db + """Return whether an attachment must be stored in db When we are using an Object Store. This is sometimes required because the object storage is slower than the database/filesystem. @@ -130,12 +126,11 @@ class IrAttachment(osv.osv): an old database with attachments pointing to deleted assets. """ - assert (isinstance(ids, int) or - len(ids) == 1), 'Expecting only one record' + assert isinstance(ids, int) or len(ids) == 1, "Expecting only one record" rec = self.browse(cr, uid, ids, context=context) # assets - if rec.res_model == 'ir.ui.view': + if rec.res_model == "ir.ui.view": # assets are stored in 'ir.ui.view' return True @@ -146,58 +141,50 @@ class IrAttachment(osv.osv): # we keep them in the database instead of the object storage location = self._storage(cr, uid) for attach in self.browse(cr, uid, id, context): - if (location in self._get_stores() and - self._save_in_db_anyway(cr, uid, [id], context)): + if location in self._get_stores() and self._save_in_db_anyway( + cr, uid, [id], context + ): # compute the fields that depend on datas - bin_data = value and value.decode('base64') or '' + bin_data = value and value.decode("base64") or "" vals = { - 'file_size': len(bin_data), - 'checksum': self._compute_checksum(bin_data), - 'db_datas': value, + "file_size": len(bin_data), + "checksum": self._compute_checksum(bin_data), + "db_datas": value, # we seriously don't need index content on those fields - 'index_content': False, - 'store_fname': False, + "index_content": False, + "store_fname": False, } fname = attach.store_fname # write as superuser, as user probably does not # have write access - super(IrAttachment, self).write( - cr, SUPERUSER_ID, id, vals, context - ) + super(IrAttachment, self).write(cr, SUPERUSER_ID, id, vals, context) if fname: self._file_delete(cr, uid, fname) continue - self._data_set(cr, uid, id, 'datas', value, None, context) + self._data_set(cr, uid, id, "datas", value, None, context) def _store_file_read(self, fname, bin_size=False): - storage = fname.partition('://')[0] - raise NotImplementedError( - 'No implementation for %s' % (storage,) - ) + storage = fname.partition("://")[0] + raise NotImplementedError("No implementation for %s" % (storage,)) def _store_file_write(self, storage, key, bin_data): - raise NotImplementedError( - 'No implementation for %s' % (storage,) - ) + raise NotImplementedError("No implementation for %s" % (storage,)) def _store_file_delete(self, fname): - storage = fname.partition('://')[0] - raise NotImplementedError( - 'No implementation for %s' % (storage,) - ) + storage = fname.partition("://")[0] + raise NotImplementedError("No implementation for %s" % (storage,)) def _file_read(self, cr, uid, location, fname, bin_size=False): if self._is_file_from_a_store(fname): return self._store_file_read(fname, bin_size=bin_size) else: _super = super(IrAttachment, self) - return _super._file_read(cr, uid, location, - fname, bin_size=bin_size) + return _super._file_read(cr, uid, location, fname, bin_size=bin_size) def _file_write(self, cr, uid, location, value): storage = self._storage(cr, uid) if storage in self._get_stores(): - bin_data = value.decode('base64') + bin_data = value.decode("base64") key = self._compute_checksum(bin_data) filename = self._store_file_write(storage, key, bin_data) else: @@ -209,8 +196,9 @@ class IrAttachment(osv.osv): if self._is_file_from_a_store(fname): # using SQL to include files hidden through unlink or due to record # rules - cr.execute("SELECT COUNT(*) FROM ir_attachment " - "WHERE store_fname = %s", (fname,)) + cr.execute( + "SELECT COUNT(*) FROM ir_attachment " "WHERE store_fname = %s", (fname,) + ) count = cr.fetchone()[0] if int(count) == 1: self._store_file_delete(fname) @@ -219,33 +207,31 @@ class IrAttachment(osv.osv): def _is_file_from_a_store(self, fname): for store_name in self._get_stores(): - uri = '{}://'.format(store_name) + uri = "{}://".format(store_name) if fname.startswith(uri): return True return False def _move_attachment_to_store(self, cr, uid, ids, context=None): - assert (isinstance(ids, int) or - len(ids) == 1), 'Expecting only one record' + assert isinstance(ids, int) or len(ids) == 1, "Expecting only one record" rec = self.browse(cr, uid, ids, context) - _logger.info('inspecting attachment %s (%d)', rec.name, rec.id) + _logger.info("inspecting attachment %s (%d)", rec.name, rec.id) fname = rec.store_fname if fname: # migrating from filesystem filestore # or from the old 'store_fname' without the bucket name - _logger.info('moving %s on the object storage', fname) - self.write(cr, uid, ids, {'datas': rec.datas}, context) - _logger.info('moved %s on the object storage', fname) + _logger.info("moving %s on the object storage", fname) + self.write(cr, uid, ids, {"datas": rec.datas}, context) + _logger.info("moved %s on the object storage", fname) return self._full_path(cr, uid, None, fname) elif rec.db_datas: - _logger.info('moving on the object storage from database') - self.write(cr, uid, ids, {'datas': rec.datas}, context) + _logger.info("moving on the object storage from database") + self.write(cr, uid, ids, {"datas": rec.datas}, context) def force_storage(self, cr, uid, context=None): if not self._is_user_admin(cr, uid): raise except_orm( - _('Error'), - _('Only administrators can execute this action.') + _("Error"), _("Only administrators can execute this action.") ) storage = self._storage(cr, uid) if storage not in self._get_stores(): @@ -253,10 +239,10 @@ class IrAttachment(osv.osv): self._force_storage_to_object_storage(cr, uid, context) def _force_storage_to_object_storage(self, cr, uid, context=None): - _logger.info('migrating files to the object storage') + _logger.info("migrating files to the object storage") storage = self._storage(cr, uid) - domain = [('store_fname', 'not like', '{}://%'.format(storage))] + domain = [("store_fname", "not like", "{}://%".format(storage))] ids = self.search(cr, uid, domain, context=context) files_to_clean = [] @@ -273,7 +259,7 @@ class IrAttachment(osv.osv): "WHERE id = %s " "FOR UPDATE NOWAIT", (attachment_id,), - log_exceptions=False + log_exceptions=False, ) path = self._move_attachment_to_store( @@ -282,8 +268,9 @@ class IrAttachment(osv.osv): if path: files_to_clean.append(path) except psycopg2.OperationalError: - _logger.error('Could not migrate attachment %s to %s' % - (attachment_id, storage)) + _logger.error( + "Could not migrate attachment %s to %s" % (attachment_id, storage) + ) def clean(): clean_fs(files_to_clean) @@ -291,8 +278,8 @@ class IrAttachment(osv.osv): # delete the files from the filesystem once we know the changes # have been committed in ir.attachment if files_to_clean: - cr.after('commit', clean) + cr.commit() def _get_stores(self): - """ To get the list of stores activated in the system """ + """To get the list of stores activated in the system""" return [] From 126b3de74b12b4c1fa33d871c18a724026a91f8e Mon Sep 17 00:00:00 2001 From: vrenaville Date: Fri, 20 May 2022 14:06:41 +0200 Subject: [PATCH 08/12] fix --- cloud_platform/models/cloud_platform.py | 163 ++++++++++++------------ 1 file changed, 83 insertions(+), 80 deletions(-) diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py index 44026f2..ce92a82 100644 --- a/cloud_platform/models/cloud_platform.py +++ b/cloud_platform/models/cloud_platform.py @@ -19,99 +19,95 @@ _logger = logging.getLogger(__name__) def is_true(strval): - return bool(strtobool(strval or '0'.lower())) + return bool(strtobool(strval or "0".lower())) -PlatformConfig = namedtuple( - 'PlatformConfig', - 'filestore' -) +PlatformConfig = namedtuple("PlatformConfig", "filestore") class FilestoreKind(object): - db = 'db' - s3 = 's3' # or compatible s3 object storage - swift = 'swift' - file = 'file' + db = "db" + s3 = "s3" # or compatible s3 object storage + swift = "swift" + file = "file" + azure = "azure" class CloudPlatform(osv.osv_abstract): - _name = 'cloud.platform' + _name = "cloud.platform" def _platform_kinds(self): # XXX for backward compatibility, we need this one here, move # it in cloud_platform_exoscale in V11 - return ['exoscale'] + return ["exoscale"] def _filestore_kinds(self): # XXX for backward compatibility, we need this one here, move # it in cloud_platform_exoscale in V11 - return ['exoscale'] - + return ["exoscale"] # XXX for backward compatibility, we need this one here, move # it in cloud_platform_exoscale in V11 def _config_by_server_env_for_exoscale(self): configs = { - 'prod': PlatformConfig(filestore=FilestoreKind.s3), - 'integration': PlatformConfig(filestore=FilestoreKind.s3), - 'test': PlatformConfig(filestore=FilestoreKind.db), - 'dev': PlatformConfig(filestore=FilestoreKind.db), + "prod": PlatformConfig(filestore=FilestoreKind.s3), + "integration": PlatformConfig(filestore=FilestoreKind.s3), + "test": PlatformConfig(filestore=FilestoreKind.db), + "dev": PlatformConfig(filestore=FilestoreKind.db), } return configs def _config_by_server_env(self, platform_kind, environment): configs_getter = getattr( - self, - '_config_by_server_env_for_%s' % platform_kind, - None + self, "_config_by_server_env_for_%s" % platform_kind, None ) configs = configs_getter() if configs_getter else {} return configs.get(environment) or FilestoreKind.db def _get_running_env(self): - environment_name = config['running_env'] - if environment_name.startswith('labs'): + environment_name = config["running_env"] + if environment_name.startswith("labs"): # We allow to have environments such as 'labs-logistics' # or 'labs-finance', in order to have the matching ribbon. - environment_name = 'labs' + environment_name = "labs" return environment_name # Due to the addition of the ovh cloud platform # This will be moved to cloud_platform_exoscale on v11 def install_exoscale(self, cr, uid, context=None): - self.install(cr, uid, 'exoscale', context) + self.install(cr, uid, "exoscale", context) def install(self, cr, uid, platform_kind, context=None): assert platform_kind in self._platform_kinds() - params = self.pool.get('ir.config_parameter') + params = self.pool.get("ir.config_parameter") params.set_param( - cr, SUPERUSER_ID, - 'cloud.platform.kind', platform_kind, - context=context + cr, SUPERUSER_ID, "cloud.platform.kind", platform_kind, context=context ) environment_name = self._get_running_env() configs = self._config_by_server_env(platform_kind, environment_name) params.set_param( - cr, SUPERUSER_ID, - 'ir_attachment.location', configs.filestore, - context=context + cr, + SUPERUSER_ID, + "ir_attachment.location", + configs.filestore, + context=context, ) self.check(cr, uid, context) if configs.filestore in [FilestoreKind.swift, FilestoreKind.s3]: - self.pool.get('ir.attachment').force_storage( + self.pool.get("ir.attachment").force_storage( cr, SUPERUSER_ID, context=context ) - _logger.info('cloud platform configured for {}'.format(platform_kind)) + _logger.info("cloud platform configured for {}".format(platform_kind)) def _check_swift(self, cr, uid, environment_name, context=None): - params = self.pool.get('ir.config_parameter') + params = self.pool.get("ir.config_parameter") use_swift = ( params.get_param( - cr, SUPERUSER_ID, 'ir_attachment.location', context=context - ) == FilestoreKind.swift + cr, SUPERUSER_ID, "ir_attachment.location", context=context + ) + == FilestoreKind.swift ) - if environment_name in ('prod', 'integration'): + if environment_name in ("prod", "integration"): # Labs instances use swift or s3 by default, but we don't want # to enforce it in case we want to test something with a different # storage. At your own risks! @@ -122,19 +118,19 @@ class CloudPlatform(osv.osv_abstract): "automatically." ) if use_swift: - assert os.environ.get('SWIFT_AUTH_URL'), ( + assert os.environ.get("SWIFT_AUTH_URL"), ( "SWIFT_AUTH_URL environment variable is required when " "ir_attachment.location is 'swift'." ) - assert os.environ.get('SWIFT_ACCOUNT'), ( + assert os.environ.get("SWIFT_ACCOUNT"), ( "SWIFT_ACCOUNT environment variable is required when " "ir_attachment.location is 'swift'." ) - assert os.environ.get('SWIFT_PASSWORD'), ( + assert os.environ.get("SWIFT_PASSWORD"), ( "SWIFT_PASSWORD environment variable is required when " "ir_attachment.location is 'swift'." ) - container_name = os.environ.get('SWIFT_WRITE_CONTAINER') + container_name = os.environ.get("SWIFT_WRITE_CONTAINER") assert container_name, ( "SWIFT_WRITE_CONTAINER environment variable is required when " "ir_attachment.location is 'swift'.\n" @@ -145,9 +141,8 @@ class CloudPlatform(osv.osv_abstract): "If you don't actually need a bucket, change the" " 'ir_attachment.location' parameter." ) - prod_container = bool(re.match(r'[a-z0-9-]+-odoo-prod', - container_name)) - if environment_name == 'prod': + prod_container = bool(re.match(r"[a-z0-9-]+-odoo-prod", container_name)) + if environment_name == "prod": assert prod_container, ( "SWIFT_WRITE_CONTAINER should match '-odoo-prod', " "we got: '%s'" % (container_name,) @@ -159,21 +154,28 @@ class CloudPlatform(osv.osv_abstract): "SWIFT_WRITE_CONTAINER should not match " "'-odoo-prod', we got: '%s'" % (container_name,) ) - elif environment_name == 'test': + elif environment_name == "test": # store in DB so we don't have files local to the host - assert params.get_param(cr, SUPERUSER_ID, 'ir_attachment.location', - context=context) == 'db', ( + assert ( + params.get_param( + cr, SUPERUSER_ID, "ir_attachment.location", context=context + ) + == "db" + ), ( "In test instances, files must be stored in the database with " "'ir_attachment.location' set to 'db'. This is " "automatically set by the function 'install_ovh()'." ) def _check_s3(self, cr, uid, environment_name, context=None): - params = self.pool.get('ir.config_parameter') - use_s3 = params.get_param( - cr, SUPERUSER_ID, 'ir_attachment.location', context=context - ) == FilestoreKind.s3 - if environment_name in ('prod', 'integration'): + params = self.pool.get("ir.config_parameter") + use_s3 = ( + params.get_param( + cr, SUPERUSER_ID, "ir_attachment.location", context=context + ) + == FilestoreKind.s3 + ) + if environment_name in ("prod", "integration"): # Labs instances use swift or s3 by default, but we don't want # to enforce it in case we want to test something with a different # storage. At your own risks! @@ -184,15 +186,15 @@ class CloudPlatform(osv.osv_abstract): "automatically." ) if use_s3: - assert os.environ.get('AWS_ACCESS_KEY_ID'), ( + assert os.environ.get("AWS_ACCESS_KEY_ID"), ( "AWS_ACCESS_KEY_ID environment variable is required when " "ir_attachment.location is 's3'." ) - assert os.environ.get('AWS_SECRET_ACCESS_KEY'), ( + assert os.environ.get("AWS_SECRET_ACCESS_KEY"), ( "AWS_SECRET_ACCESS_KEY environment variable is required when " "ir_attachment.location is 's3'." ) - bucket_name = os.environ.get('AWS_BUCKETNAME') + bucket_name = os.environ.get("AWS_BUCKETNAME") assert bucket_name, ( "AWS_BUCKETNAME environment variable is required when " "ir_attachment.location is 's3'.\n" @@ -203,8 +205,8 @@ class CloudPlatform(osv.osv_abstract): "If you don't actually need a bucket, change the" " 'ir_attachment.location' parameter." ) - prod_bucket = bool(re.match(r'[a-z-0-9]+-odoo-prod', bucket_name)) - if environment_name == 'prod': + prod_bucket = bool(re.match(r"[a-z-0-9]+-odoo-prod", bucket_name)) + if environment_name == "prod": assert prod_bucket, ( "AWS_BUCKETNAME should match '-odoo-prod', " "we got: '%s'" % (bucket_name,) @@ -217,17 +219,21 @@ class CloudPlatform(osv.osv_abstract): "we got: '%s'" % (bucket_name,) ) - elif environment_name == 'test': + elif environment_name == "test": # store in DB so we don't have files local to the host - assert params.get_param(cr, SUPERUSER_ID, 'ir_attachment.location', - context=context) == 'db', ( + assert ( + params.get_param( + cr, SUPERUSER_ID, "ir_attachment.location", context=context + ) + == "db" + ), ( "In test instances, files must be stored in the database with " "'ir_attachment.location' set to 'db'. This is " "automatically set by the function 'install_exoscale()'." ) def _check_azure(self, cr, uid, environment_name, context=None): - params = self.env["ir.config_parameter"].sudo() + params = self.pool.get("ir.config_parameter") use_azure = params.get_param("ir_attachment.location") == AZURE_STORE_KIND.name if environment_name in ("prod", "integration"): # Labs instances use azure by default, but we don't want @@ -305,38 +311,35 @@ class CloudPlatform(osv.osv_abstract): "automatically set by the function 'install()'." ) - def _check_redis(self, cr, uid, environment_name, context=None): - if environment_name in ('prod', 'integration', 'labs', 'test'): - assert is_true(os.environ.get('ODOO_SESSION_REDIS')), ( + if environment_name in ("prod", "integration", "labs", "test"): + assert is_true(os.environ.get("ODOO_SESSION_REDIS")), ( "Redis must be activated on prod, integration, labs," " test instances. This is done by setting ODOO_SESSION_REDIS=1." ) - assert (os.environ.get('ODOO_SESSION_REDIS_HOST') or - os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST')), ( + assert os.environ.get("ODOO_SESSION_REDIS_HOST") or os.environ.get( + "ODOO_SESSION_REDIS_SENTINEL_HOST" + ), ( "ODOO_SESSION_REDIS_HOST or ODOO_SESSION_REDIS_SENTINEL_HOST " "environment variable is required to connect on Redis" ) - assert os.environ.get('ODOO_SESSION_REDIS_PREFIX'), ( + assert os.environ.get("ODOO_SESSION_REDIS_PREFIX"), ( "ODOO_SESSION_REDIS_PREFIX environment variable is required " "to store sessions on Redis" ) - prefix = os.environ['ODOO_SESSION_REDIS_PREFIX'] - assert re.match(r'^[a-z-0-9]+-odoo-[a-z-0-9]+$', prefix), ( + prefix = os.environ["ODOO_SESSION_REDIS_PREFIX"] + assert re.match(r"^[a-z-0-9]+-odoo-[a-z-0-9]+$", prefix), ( "ODOO_SESSION_REDIS_PREFIX must match '-odoo-'" ", we got: '%s'" % (prefix,) ) def check(self, cr, uid, context=None): - if is_true(os.environ.get('ODOO_CLOUD_PLATFORM_UNSAFE')): - _logger.warning( - "cloud platform checks disabled, this is not safe" - ) + if is_true(os.environ.get("ODOO_CLOUD_PLATFORM_UNSAFE")): + _logger.warning("cloud platform checks disabled, this is not safe") return - params = self.pool.get('ir.config_parameter') - kind = params.get_param(cr, SUPERUSER_ID, - 'cloud.platform.kind', context=None) + params = self.pool.get("ir.config_parameter") + kind = params.get_param(cr, SUPERUSER_ID, "cloud.platform.kind", context=None) if not kind: _logger.warning( "cloud platform not configured, you should " @@ -344,14 +347,14 @@ class CloudPlatform(osv.osv_abstract): ) return environment_name = self._get_running_env() - if kind == 'exoscale': + if kind == "exoscale": self._check_s3(cr, uid, environment_name, context) - elif kind == 'ovh': + elif kind == "ovh": self._check_swift(cr, uid, environment_name, context) - elif kind == 'azure': + elif kind == "azure": self._check_azure(cr, uid, environment_name, context) self._check_redis(cr, uid, environment_name, context) def _register_hook(self, cr): super(CloudPlatform, self)._register_hook(cr) - self.pool.get('cloud.platform').check(cr, SUPERUSER_ID) + self.pool.get("cloud.platform").check(cr, SUPERUSER_ID) From 90005a62f9867ae9f838875218cb4d351504bd95 Mon Sep 17 00:00:00 2001 From: vrenaville Date: Fri, 20 May 2022 14:19:39 +0200 Subject: [PATCH 09/12] fix --- cloud_platform/models/cloud_platform.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py index ce92a82..b0abe35 100644 --- a/cloud_platform/models/cloud_platform.py +++ b/cloud_platform/models/cloud_platform.py @@ -234,7 +234,12 @@ class CloudPlatform(osv.osv_abstract): def _check_azure(self, cr, uid, environment_name, context=None): params = self.pool.get("ir.config_parameter") - use_azure = params.get_param("ir_attachment.location") == AZURE_STORE_KIND.name + use_azure = ( + params.get_param( + cr, SUPERUSER_ID, "ir_attachment.location", context=context + ) + == AZURE_STORE_KIND.name + ) if environment_name in ("prod", "integration"): # Labs instances use azure by default, but we don't want # to enforce it in case we want to test something with a different @@ -305,7 +310,12 @@ class CloudPlatform(osv.osv_abstract): elif environment_name == "test": # store in DB so we don't have files local to the host - assert params.get_param("ir_attachment.location") == "db", ( + assert ( + params.get_param( + cr, SUPERUSER_ID, "ir_attachment.location", context=context + ) + == "db" + ), ( "In test instances, files must be stored in the database with " "'ir_attachment.location' set to 'db'. This is " "automatically set by the function 'install()'." From d7e909ae923e4827da78f7ce9e061fb55982c4cc Mon Sep 17 00:00:00 2001 From: vrenaville Date: Fri, 20 May 2022 14:30:02 +0200 Subject: [PATCH 10/12] fix --- cloud_platform/models/cloud_platform.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py index b0abe35..28041db 100644 --- a/cloud_platform/models/cloud_platform.py +++ b/cloud_platform/models/cloud_platform.py @@ -238,7 +238,7 @@ class CloudPlatform(osv.osv_abstract): params.get_param( cr, SUPERUSER_ID, "ir_attachment.location", context=context ) - == AZURE_STORE_KIND.name + == FilestoreKind.azure ) if environment_name in ("prod", "integration"): # Labs instances use azure by default, but we don't want From 1706ecb1481da9d8b5c0b44aedc8d794447f7196 Mon Sep 17 00:00:00 2001 From: vrenaville Date: Fri, 20 May 2022 14:44:01 +0200 Subject: [PATCH 11/12] fix --- cloud_platform/models/cloud_platform.py | 6 +-- session_redis/http.py | 62 ++++++++++++++----------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py index 28041db..c92200b 100644 --- a/cloud_platform/models/cloud_platform.py +++ b/cloud_platform/models/cloud_platform.py @@ -327,10 +327,10 @@ class CloudPlatform(osv.osv_abstract): "Redis must be activated on prod, integration, labs," " test instances. This is done by setting ODOO_SESSION_REDIS=1." ) - assert os.environ.get("ODOO_SESSION_REDIS_HOST") or os.environ.get( - "ODOO_SESSION_REDIS_SENTINEL_HOST" + assert os.environ.get("ODOO_SESSION_REDIS_URL") or os.environ.get( + "ODOO_SESSION_REDIS_SENTINEL_URL" ), ( - "ODOO_SESSION_REDIS_HOST or ODOO_SESSION_REDIS_SENTINEL_HOST " + "ODOO_SESSION_REDIS_URL or ODOO_SESSION_REDIS_SENTINEL_URL " "environment variable is required to connect on Redis" ) assert os.environ.get("ODOO_SESSION_REDIS_PREFIX"), ( diff --git a/session_redis/http.py b/session_redis/http.py index 7da42ee..6a77af1 100644 --- a/session_redis/http.py +++ b/session_redis/http.py @@ -24,45 +24,45 @@ except ImportError: def is_true(strval): - return bool(strtobool(strval or '0'.lower())) + return bool(strtobool(strval or "0".lower())) -sentinel_host = os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST') -sentinel_master_name = os.environ.get( - 'ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME' -) +sentinel_host = os.environ.get("ODOO_SESSION_REDIS_SENTINEL_HOST") +sentinel_master_name = os.environ.get("ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME") if sentinel_host and not sentinel_master_name: raise Exception( "ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME must be defined " "when using session_redis" ) -sentinel_port = int(os.environ.get('ODOO_SESSION_REDIS_SENTINEL_PORT', 26379)) -host = os.environ.get('ODOO_SESSION_REDIS_HOST', 'localhost') -port = int(os.environ.get('ODOO_SESSION_REDIS_PORT', 6379)) -prefix = os.environ.get('ODOO_SESSION_REDIS_PREFIX') -url = os.environ.get('ODOO_SESSION_REDIS_URL') -password = os.environ.get('ODOO_SESSION_REDIS_PASSWORD') -expiration = os.environ.get('ODOO_SESSION_REDIS_EXPIRATION') -anon_expiration = os.environ.get('ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS') +sentinel_port = int(os.environ.get("ODOO_SESSION_REDIS_SENTINEL_PORT", 26379)) +host = os.environ.get("ODOO_SESSION_REDIS_URL", "localhost") +port = int(os.environ.get("ODOO_SESSION_REDIS_PORT", 6379)) +prefix = os.environ.get("ODOO_SESSION_REDIS_PREFIX") +url = os.environ.get("ODOO_SESSION_REDIS_URL") +password = os.environ.get("ODOO_SESSION_REDIS_PASSWORD") +expiration = os.environ.get("ODOO_SESSION_REDIS_EXPIRATION") +anon_expiration = os.environ.get("ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS") def session_store(): if sentinel_host: - sentinel = Sentinel([(sentinel_host, sentinel_port)], - password=password) + sentinel = Sentinel([(sentinel_host, sentinel_port)], password=password) redis_client = sentinel.master_for(sentinel_master_name) elif url: redis_client = redis.from_url(url) else: redis_client = redis.Redis(host=host, port=port, password=password) - return RedisSessionStore(redis=redis_client, prefix=prefix, - expiration=expiration, - anon_expiration=anon_expiration, - session_class=Session) + return RedisSessionStore( + redis=redis_client, + prefix=prefix, + expiration=expiration, + anon_expiration=anon_expiration, + session_class=Session, + ) def session_gc(session_store): - """ Do not garbage collect the sessions + """Do not garbage collect the sessions Redis keys are automatically cleaned at the end of their expiration. @@ -79,18 +79,26 @@ def purge_fs_sessions(path): pass -if is_true(os.environ.get('ODOO_SESSION_REDIS')): +if is_true(os.environ.get("ODOO_SESSION_REDIS")): if sentinel_host: - _logger.debug("HTTP sessions stored in Redis with prefix '%s'. " - "Using Sentinel on %s:%s", - sentinel_host, sentinel_port, prefix or '') + _logger.debug( + "HTTP sessions stored in Redis with prefix '%s'. " + "Using Sentinel on %s:%s", + sentinel_host, + sentinel_port, + prefix or "", + ) else: - _logger.debug("HTTP sessions stored in Redis with prefix '%s' on " - "%s:%s", host, port, prefix or '') + _logger.debug( + "HTTP sessions stored in Redis with prefix '%s' on " "%s:%s", + host, + port, + prefix or "", + ) store = session_store() for handler in openerp.service.wsgi_server.module_handlers: - if hasattr(handler, 'session_store'): + if hasattr(handler, "session_store"): handler.session_store = store http.session_gc = session_gc # clean the existing sessions on the file system From 3104ae89666313650c0d8c5a032923f423fff160 Mon Sep 17 00:00:00 2001 From: vrenaville Date: Fri, 20 May 2022 15:02:00 +0200 Subject: [PATCH 12/12] fix: dbname env get --- attachment_azure/models/ir_attachment.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py index b6aaebc..81415cd 100644 --- a/attachment_azure/models/ir_attachment.py +++ b/attachment_azure/models/ir_attachment.py @@ -117,8 +117,9 @@ class IrAttachment(osv.osv): https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names """ running_env = os.environ.get("RUNNING_ENV", "dev") + dbname = os.environ.get("DB_NAME", "odoodb") storage_name = os.environ.get("AZURE_STORAGE_NAME", r"{env}-{db}") - storage_name = storage_name.format(env=running_env, db=self.env.cr.dbname) + storage_name = storage_name.format(env=running_env, db=dbname) # replace invalid characters by _ storage_name = re.sub(r"[\W_]+", "-", storage_name) # lowercase, max 63 chars