From 61a7d392ff0ffc2bec8eb430d9a9c6e87ffb79cb Mon Sep 17 00:00:00 2001 From: Vincent Renaville Date: Tue, 15 Mar 2022 10:27:02 +0100 Subject: [PATCH] 9 azure backport (#353) * fix: backport fix + backport modules to azure --- .travis.yml | 7 +- LICENSE | 2 +- attachment_azure/models/ir_attachment.py | 51 +++++-- attachment_swift/tests/test_mock_swift_api.py | 6 +- .../models/ir_attachment.py | 4 +- cloud_platform_azure/README.md | 6 + cloud_platform_azure/__init__.py | 2 + cloud_platform_azure/__openerp__.py | 25 ++++ cloud_platform_azure/models/__init__.py | 2 + cloud_platform_azure/models/cloud_platform.py | 127 ++++++++++++++++++ monitoring_log_requests/models/ir_http.py | 8 +- monitoring_prometheus/README.rst | 17 +++ monitoring_prometheus/__init__.py | 3 + monitoring_prometheus/__openerp__.py | 23 ++++ monitoring_prometheus/controllers/__init__.py | 2 + .../controllers/prometheus_metrics.py | 20 +++ monitoring_prometheus/models/__init__.py | 2 + monitoring_prometheus/models/ir_http.py | 50 +++++++ requirements.txt | 1 + 19 files changed, 332 insertions(+), 26 deletions(-) create mode 100644 cloud_platform_azure/README.md create mode 100644 cloud_platform_azure/__init__.py create mode 100644 cloud_platform_azure/__openerp__.py create mode 100644 cloud_platform_azure/models/__init__.py create mode 100644 cloud_platform_azure/models/cloud_platform.py create mode 100644 monitoring_prometheus/README.rst create mode 100644 monitoring_prometheus/__init__.py create mode 100644 monitoring_prometheus/__openerp__.py create mode 100644 monitoring_prometheus/controllers/__init__.py create mode 100644 monitoring_prometheus/controllers/prometheus_metrics.py create mode 100644 monitoring_prometheus/models/__init__.py create mode 100644 monitoring_prometheus/models/ir_http.py diff --git a/.travis.yml b/.travis.yml index 6ac6217..9bd23ed 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,10 +27,13 @@ virtualenv: env: matrix: - LINT_CHECK="1" + - TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="cloud_platform_azure" + - TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="cloud_platform_azure" - TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="cloud_platform_ovh" - TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="cloud_platform_ovh" - - TESTS="1" ODOO_REPO="odoo/odoo" EXCLUDE="cloud_platform,cloud_platform_ovh" - - TESTS="1" ODOO_REPO="OCA/OCB" EXCLUDE="cloud_platform,cloud_platform_ovh" + - TESTS="1" ODOO_REPO="odoo/odoo" EXCLUDE="cloud_platform,cloud_platform_ovh,cloud_platform_azure" + - TESTS="1" ODOO_REPO="OCA/OCB" EXCLUDE="cloud_platform,cloud_platform_ovh,cloud_platform_azure" + global: - VERSION="9.0" LINT_CHECK="0" TESTS="0" diff --git a/LICENSE b/LICENSE index 3ffc567..58777e3 100644 --- a/LICENSE +++ b/LICENSE @@ -658,4 +658,4 @@ specific requirements. You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see -. \ No newline at end of file +. diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py index dcbcd13..301900b 100644 --- a/attachment_azure/models/ir_attachment.py +++ b/attachment_azure/models/ir_attachment.py @@ -129,25 +129,39 @@ class IrAttachment(models.Model): return str.lower(storage_name)[:63] @api.model - def _get_azure_container(self): - container_name = self._get_container_name() - blob_service_client = self._get_blob_service_client() - container_client = blob_service_client.get_container_client(container_name) + def _get_azure_container(self, container_name=None): + if not container_name: + container_name = self._get_container_name() try: - # Create the container - container_client.create_container() - except ResourceExistsError: - pass - except HttpResponseError as error: - _logger.exception("Error during the creation of the Azure container") - raise exceptions.UserError(str(error)) + blob_service_client = self._get_blob_service_client() + except exceptions.UserError: + _logger.exception( + "error accessing to storage '%s' please check credentials ", + container_name + ) + return False + container_client = blob_service_client.get_container_client(container_name) + if not container_client.exists(): + try: + # Create the container + container_client.create_container() + except HttpResponseError as error: + _logger.exception("Error during the creation of the Azure container") + raise exceptions.UserError(str(error)) return container_client @api.model def _store_file_read(self, fname, bin_size=False): if fname.startswith("azure://"): - container_client = self._get_azure_container() key = fname.replace("azure://", "", 1).lower() + if '/' in key: + container_name, key = key.split('/', 1) + else: + container_name = None + container_client = self._get_azure_container(container_name) + # if container cannot be retrived, abort reading from azure storage + if not container_client: + return '' try: blob_client = container_client.get_blob_client(key) if bin_size: @@ -166,11 +180,13 @@ class IrAttachment(models.Model): location = self.env.context.get("storage_location") or self._storage() if location == "azure": container_client = self._get_azure_container() + if not container_client: + return '' + filename = "azure://%s/%s" % (container_client.container_name, key) with io.BytesIO() as file: blob_client = container_client.get_blob_client(key.lower()) file.write(bin_data) file.seek(0) - filename = "azure://%s" % (key) try: blob_client.upload_blob(file, blob_type="BlockBlob") except ResourceExistsError: @@ -189,8 +205,15 @@ class IrAttachment(models.Model): @api.model def _store_file_delete(self, fname): if fname.startswith("azure://"): - container_client = self._get_azure_container() + # if container cannot be retrived, abort reading from azure storage key = fname.replace("azure://", "", 1).lower() + if '/' in key: + container_name, key = key.split('/', 1) + else: + container_name = None + container_client = self._get_azure_container(container_name) + if not container_client: + return '' # delete the file only if it is on the current configured container # otherwise, we might delete files used on a different environment try: diff --git a/attachment_swift/tests/test_mock_swift_api.py b/attachment_swift/tests/test_mock_swift_api.py index 11a5421..397bf91 100644 --- a/attachment_swift/tests/test_mock_swift_api.py +++ b/attachment_swift/tests/test_mock_swift_api.py @@ -9,9 +9,9 @@ from mock import patch import keystoneauth1 -from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment -from odoo.addons.attachment_swift.models.ir_attachment import SwiftSessionStore -from odoo.addons.attachment_swift.swift_uri import SwiftUri +from openerp.addons.base.tests.test_ir_attachment import TestIrAttachment +from openerp.addons.attachment_swift.models.ir_attachment import SwiftSessionStore +from openerp.addons.attachment_swift.swift_uri import SwiftUri class TestAttachmentSwift(TestIrAttachment): diff --git a/base_attachment_object_storage/models/ir_attachment.py b/base_attachment_object_storage/models/ir_attachment.py index 6fc56ea..225f631 100644 --- a/base_attachment_object_storage/models/ir_attachment.py +++ b/base_attachment_object_storage/models/ir_attachment.py @@ -6,7 +6,7 @@ import inspect import logging import os import psycopg2 -import odoo +import openerp from contextlib import closing, contextmanager from openerp import api, exceptions, fields, models, _ @@ -208,7 +208,7 @@ class IrAttachment(models.Model): """ with api.Environment.manage(): if new_cr: - registry = odoo.modules.registry.RegistryManager.get( + registry = openerp.modules.registry.RegistryManager.get( self.env.cr.dbname ) with closing(registry.cursor()) as cr: diff --git a/cloud_platform_azure/README.md b/cloud_platform_azure/README.md new file mode 100644 index 0000000..1f7bd5d --- /dev/null +++ b/cloud_platform_azure/README.md @@ -0,0 +1,6 @@ +Cloud Platform Azure +==================== + +Install addons specific to the Azure setup. + + * The object storage is Azure blob storage diff --git a/cloud_platform_azure/__init__.py b/cloud_platform_azure/__init__.py new file mode 100644 index 0000000..1466a94 --- /dev/null +++ b/cloud_platform_azure/__init__.py @@ -0,0 +1,2 @@ +from __future__ import absolute_import +from . import models diff --git a/cloud_platform_azure/__openerp__.py b/cloud_platform_azure/__openerp__.py new file mode 100644 index 0000000..d44dc59 --- /dev/null +++ b/cloud_platform_azure/__openerp__.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Copyright 2017-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{ + "name": "Cloud Platform Azure", + "summary": "Addons required for the Camptocamp Cloud Platform on Azure", + "version": "9.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Extra Tools", + "depends": [ + "cloud_platform", + "attachment_azure", + "monitoring_prometheus", + ], + "excludes": [ + "cloud_platform_ovh", + "cloud_platform_exoscale", + ], + "website": "https://www.camptocamp.com", + "data": [], + "installable": True, +} diff --git a/cloud_platform_azure/models/__init__.py b/cloud_platform_azure/models/__init__.py new file mode 100644 index 0000000..b438841 --- /dev/null +++ b/cloud_platform_azure/models/__init__.py @@ -0,0 +1,2 @@ +from __future__ import absolute_import +from . import cloud_platform diff --git a/cloud_platform_azure/models/cloud_platform.py b/cloud_platform_azure/models/cloud_platform.py new file mode 100644 index 0000000..db34b8a --- /dev/null +++ b/cloud_platform_azure/models/cloud_platform.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from __future__ import absolute_import +import re +import os + +from openerp import models, api +from openerp.addons.cloud_platform.models.cloud_platform import FilestoreKind +from openerp.addons.cloud_platform.models.cloud_platform import PlatformConfig + + +AZURE_STORE_KIND = FilestoreKind("azure", "remote") + + +class CloudPlatform(models.AbstractModel): + _inherit = "cloud.platform" + + @api.model + def _filestore_kinds(self): + kinds = super(CloudPlatform, self)._filestore_kinds() + kinds["azure"] = AZURE_STORE_KIND + return kinds + + @api.model + def _platform_kinds(self): + kinds = super(CloudPlatform, self)._platform_kinds() + kinds.append("azure") + return kinds + + @api.model + def _config_by_server_env_for_azure(self): + fs_kinds = self._filestore_kinds() + configs = { + "prod": PlatformConfig(filestore=fs_kinds["azure"]), + "integration": PlatformConfig(filestore=fs_kinds["azure"]), + "labs": PlatformConfig(filestore=fs_kinds["azure"]), + "test": PlatformConfig(filestore=fs_kinds["db"]), + "dev": PlatformConfig(filestore=fs_kinds["db"]), + } + return configs + + @api.model + def _check_filestore(self, environment_name): + params = self.env["ir.config_parameter"].sudo() + use_azure = (params.get_param("ir_attachment.location") == + AZURE_STORE_KIND.name) + if environment_name in ("prod", "integration"): + # Labs instances use azure by default, but we don't want + # to enforce it in case we want to test something with a different + # storage. At your own risks! + assert use_azure, ( + "azure must be used on production and integration instances. " + "It is activated by setting 'ir_attachment.location.' to 'azure'." + " The 'install()' function sets this option " + "automatically." + ) + if use_azure: + key_sets = [ + ["AZURE_STORAGE_USE_AAD", "AZURE_STORAGE_ACCOUNT_URL"], + ["AZURE_STORAGE_CONNECTION_STRING"], + [ + "AZURE_STORAGE_ACCOUNT_NAME", + "AZURE_STORAGE_ACCOUNT_URL", + "AZURE_STORAGE_ACCOUNT_KEY", + ], + ] + is_valid = False + for key_set in key_sets: + if all([os.environ.get(key) for key in key_set]): + is_valid = True + break + assert is_valid, ( + "When ir_attachment.location is set to 'azure', " + "at least one of the following enviromnent variable set " + "is required : {}".format( + " or ".join( + [" + ".join([key for key in key_set]) for key_set in key_sets] + ) + ) + ) + storage_name = os.environ.get("AZURE_STORAGE_NAME", "") + if environment_name in ("prod", "integration", "labs"): + assert storage_name, ( + "AZURE_STORAGE_NAME environment variable is required when " + "ir_attachment.location is 'azure'.\n" + "Normally, 'azure' is activated on labs, integration " + "and production, but should not be used in dev environment" + " (or using a dedicated dev bucket, never using the " + "integration/prod bucket).\n" + "If you don't actually need a bucket, change the" + " 'ir_attachment.location' parameter." + ) + # A bucket name is defined under the following format + # ^[a-z]+\-[a-z]+\-\d+$ + # Anything other than prod bucket must be suffixed with env name + # + # Use AZURE_STORAGE_NAME_UNSTRUCTURED to by-pass check + # on bucket name structure + if os.environ.get("AZURE_STORAGE_NAME_UNSTRUCTURED"): + return + prod_bucket = bool(re.match(ur"^[a-z]+\-[a-z]+\-\d+$", storage_name)) + if environment_name == "prod": + assert prod_bucket, ( + "AZURE_STORAGE_NAME should match '^[a-z]+\\-[a-z]+\\-\\d+$', " + "we got: '%s'" % (storage_name,) + ) + else: + # if we are using the prod bucket on another instance + # such as an integration, we must be sure to be in read only! + assert not prod_bucket, ( + "AZURE_STORAGE_NAME should not match '^[a-z]+\\-[a-z]+\\-\\d+$', " + "we got: '%s'" % (storage_name,) + ) + + elif environment_name == "test": + # store in DB so we don't have files local to the host + assert params.get_param("ir_attachment.location") == "db", ( + "In test instances, files must be stored in the database with " + "'ir_attachment.location' set to 'db'. This is " + "automatically set by the function 'install()'." + ) + + @api.model + def install(self): + self._install("azure") diff --git a/monitoring_log_requests/models/ir_http.py b/monitoring_log_requests/models/ir_http.py index 1bb6c54..fe351c6 100644 --- a/monitoring_log_requests/models/ir_http.py +++ b/monitoring_log_requests/models/ir_http.py @@ -11,7 +11,7 @@ from openerp.http import request as http_request from openerp.tools.config import config -_logger = logging.getLogger('monitoring.http.requests') +_logger = logging.getLogger(u'monitoring.http.requests') class IrHttp(models.AbstractModel): @@ -28,8 +28,8 @@ class IrHttp(models.AbstractModel): return response def _monitoring_blacklist(self, request): - path_info = request.httprequest.environ.get('PATH_INFO') - if path_info.startswith('/longpolling/'): + path_info = request.httprequest.environ.get(u'PATH_INFO') + if path_info.startswith(u'/longpolling/'): return True return False @@ -37,7 +37,7 @@ class IrHttp(models.AbstractModel): return True def _monitoring_info(self, request, response, begin, end): - path = request.httprequest.environ.get('PATH_INFO') + path = request.httprequest.environ.get(u'PATH_INFO') info = { # timing 'start_time': time.strftime("%Y-%m-%d %H:%M:%S", diff --git a/monitoring_prometheus/README.rst b/monitoring_prometheus/README.rst new file mode 100644 index 0000000..aa98ffe --- /dev/null +++ b/monitoring_prometheus/README.rst @@ -0,0 +1,17 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :alt: License + +============================== +Monitoring: Prometheus metrics +============================== + +Add an endpoint */metrics* to allow a Prometheus server to fetch application metrics. +Current available metrics are: + +* Request completion time with 3 differentiators: + * Filestore + * Assets + * Everything else +* Longpolling request count + +No additional configuration is needed, just ensure that the Prometheus server is allowed to communicate with Odoo diff --git a/monitoring_prometheus/__init__.py b/monitoring_prometheus/__init__.py new file mode 100644 index 0000000..5c1d8ef --- /dev/null +++ b/monitoring_prometheus/__init__.py @@ -0,0 +1,3 @@ +from __future__ import absolute_import +from . import controllers +from . import models diff --git a/monitoring_prometheus/__openerp__.py b/monitoring_prometheus/__openerp__.py new file mode 100644 index 0000000..3fa7ae6 --- /dev/null +++ b/monitoring_prometheus/__openerp__.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{ + "name": "Monitoring: Prometheus Metrics", + "version": "9.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "category", + "depends": [ + "base", + "web", + "server_environment", + ], + "website": "http://www.camptocamp.com", + "data": [], + "external_dependencies": { + "python": ["prometheus_client"], + }, + "installable": True, +} diff --git a/monitoring_prometheus/controllers/__init__.py b/monitoring_prometheus/controllers/__init__.py new file mode 100644 index 0000000..8d50263 --- /dev/null +++ b/monitoring_prometheus/controllers/__init__.py @@ -0,0 +1,2 @@ +from __future__ import absolute_import +from . import prometheus_metrics diff --git a/monitoring_prometheus/controllers/prometheus_metrics.py b/monitoring_prometheus/controllers/prometheus_metrics.py new file mode 100644 index 0000000..82a3f9d --- /dev/null +++ b/monitoring_prometheus/controllers/prometheus_metrics.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +from __future__ import absolute_import +import logging + +from openerp.http import Controller, route + +_logger = logging.getLogger(__name__) + +try: + from prometheus_client import generate_latest +except (ImportError, IOError), err: + _logger.warning(err) + + +class PrometheusController(Controller): + @route(u'/metrics', auth=u'public') + def metrics(self): + return generate_latest() diff --git a/monitoring_prometheus/models/__init__.py b/monitoring_prometheus/models/__init__.py new file mode 100644 index 0000000..2369850 --- /dev/null +++ b/monitoring_prometheus/models/__init__.py @@ -0,0 +1,2 @@ +from __future__ import absolute_import +from . import ir_http diff --git a/monitoring_prometheus/models/ir_http.py b/monitoring_prometheus/models/ir_http.py new file mode 100644 index 0000000..62c01bd --- /dev/null +++ b/monitoring_prometheus/models/ir_http.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +# Copyright 2016-2021 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +from __future__ import with_statement +from __future__ import absolute_import +import logging + +from openerp import models +from openerp.http import request + +_logger = logging.getLogger(__name__) + +try: + from prometheus_client import Summary, Counter +except (ImportError, IOError), err: + _logger.warning(err) + + +REQUEST_TIME = Summary( + "request_latency_sec", "Request response time in sec", ["query_type"] +) +LONGPOLLING_COUNT = Counter("longpolling", "Longpolling request count") + + +class IrHttp(models.AbstractModel): + _inherit = "ir.http" + + @classmethod + def _dispatch(cls): + path_info = request.httprequest.environ.get("PATH_INFO") + + if path_info.startswith("/longpolling/"): + LONGPOLLING_COUNT.inc() + return super(IrHttp, cls)._dispatch() + + if path_info.startswith("/metrics"): + return super(IrHttp, cls)._dispatch() + + if path_info.startswith("/web/static"): + label = "assets" + elif path_info.startswith("/web/content"): + label = "filestore" + else: + label = "client" + + res = None + with REQUEST_TIME.labels(label).time(): + res = super(IrHttp, cls)._dispatch() + + return res diff --git a/requirements.txt b/requirements.txt index 8d39c30..0525441 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,3 +9,4 @@ python-keystoneclient==3.22.0 keystoneauth1==3.14.0 # error with 5.x (ConstructorError: could not determine a constructor for the tag '!record') PyYAML==4.2b4 +prometheus_client==0.11.0