9 azure backport (#353)

* fix: backport fix + backport modules to azure

.travis.yml

@@ -27,10 +27,13 @@ virtualenv:
 env:
   matrix:
   - LINT_CHECK="1"
+  - TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="cloud_platform_azure"
+  - TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="cloud_platform_azure"
   - TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="cloud_platform_ovh"
   - TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="cloud_platform_ovh"
-  - TESTS="1" ODOO_REPO="odoo/odoo" EXCLUDE="cloud_platform,cloud_platform_ovh"
-  - TESTS="1" ODOO_REPO="OCA/OCB" EXCLUDE="cloud_platform,cloud_platform_ovh"
+  - TESTS="1" ODOO_REPO="odoo/odoo" EXCLUDE="cloud_platform,cloud_platform_ovh,cloud_platform_azure"
+  - TESTS="1" ODOO_REPO="OCA/OCB" EXCLUDE="cloud_platform,cloud_platform_ovh,cloud_platform_azure"
+
   global:
   - VERSION="9.0" LINT_CHECK="0" TESTS="0"
 

attachment_azure/models/ir_attachment.py

@@ -129,25 +129,39 @@ class IrAttachment(models.Model):
         return str.lower(storage_name)[:63]
 
     @api.model
-    def _get_azure_container(self):
-        container_name = self._get_container_name()
-        blob_service_client = self._get_blob_service_client()
-        container_client = blob_service_client.get_container_client(container_name)
+    def _get_azure_container(self, container_name=None):
+        if not container_name:
+            container_name = self._get_container_name()
         try:
-            # Create the container
-            container_client.create_container()
-        except ResourceExistsError:
-            pass
-        except HttpResponseError as error:
-            _logger.exception("Error during the creation of the Azure container")
-            raise exceptions.UserError(str(error))
+            blob_service_client = self._get_blob_service_client()
+        except exceptions.UserError:
+            _logger.exception(
+                "error accessing to storage '%s' please check credentials ",
+                container_name
+            )
+            return False
+        container_client = blob_service_client.get_container_client(container_name)
+        if not container_client.exists():
+            try:
+                # Create the container
+                container_client.create_container()
+            except HttpResponseError as error:
+                _logger.exception("Error during the creation of the Azure container")
+                raise exceptions.UserError(str(error))
         return container_client
 
     @api.model
     def _store_file_read(self, fname, bin_size=False):
         if fname.startswith("azure://"):
-            container_client = self._get_azure_container()
             key = fname.replace("azure://", "", 1).lower()
+            if '/' in key:
+                container_name, key = key.split('/', 1)
+            else:
+                container_name = None
+            container_client = self._get_azure_container(container_name)
+            # if container cannot be retrived, abort reading from azure storage
+            if not container_client:
+                return ''
             try:
                 blob_client = container_client.get_blob_client(key)
                 if bin_size:
@@ -166,11 +180,13 @@ class IrAttachment(models.Model):
         location = self.env.context.get("storage_location") or self._storage()
         if location == "azure":
             container_client = self._get_azure_container()
+            if not container_client:
+                return ''
+            filename = "azure://%s/%s" % (container_client.container_name, key)
             with io.BytesIO() as file:
                 blob_client = container_client.get_blob_client(key.lower())
                 file.write(bin_data)
                 file.seek(0)
-                filename = "azure://%s" % (key)
                 try:
                     blob_client.upload_blob(file, blob_type="BlockBlob")
                 except ResourceExistsError:
@@ -189,8 +205,15 @@ class IrAttachment(models.Model):
     @api.model
     def _store_file_delete(self, fname):
         if fname.startswith("azure://"):
-            container_client = self._get_azure_container()
+            # if container cannot be retrived, abort reading from azure storage
             key = fname.replace("azure://", "", 1).lower()
+            if '/' in key:
+                container_name, key = key.split('/', 1)
+            else:
+                container_name = None
+            container_client = self._get_azure_container(container_name)
+            if not container_client:
+                return ''
             # delete the file only if it is on the current configured container
             # otherwise, we might delete files used on a different environment
             try:

attachment_swift/tests/test_mock_swift_api.py

@@ -9,9 +9,9 @@ from mock import patch
 
 import keystoneauth1
 
-from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment
-from odoo.addons.attachment_swift.models.ir_attachment import SwiftSessionStore
-from odoo.addons.attachment_swift.swift_uri import SwiftUri
+from openerp.addons.base.tests.test_ir_attachment import TestIrAttachment
+from openerp.addons.attachment_swift.models.ir_attachment import SwiftSessionStore
+from openerp.addons.attachment_swift.swift_uri import SwiftUri
 
 
 class TestAttachmentSwift(TestIrAttachment):

base_attachment_object_storage/models/ir_attachment.py

@@ -6,7 +6,7 @@ import inspect
 import logging
 import os
 import psycopg2
-import odoo
+import openerp
 
 from contextlib import closing, contextmanager
 from openerp import api, exceptions, fields, models, _
@@ -208,7 +208,7 @@ class IrAttachment(models.Model):
         """
         with api.Environment.manage():
             if new_cr:
-                registry = odoo.modules.registry.RegistryManager.get(
+                registry = openerp.modules.registry.RegistryManager.get(
                     self.env.cr.dbname
                 )
                 with closing(registry.cursor()) as cr:

cloud_platform_azure/README.md

@@ -0,0 +1,6 @@
+Cloud Platform Azure
+====================
+
+Install addons specific to the Azure setup.
+
+    * The object storage is Azure blob storage

cloud_platform_azure/__init__.py

@@ -0,0 +1,2 @@
+from __future__ import absolute_import
+from . import models

cloud_platform_azure/__openerp__.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{
+    "name": "Cloud Platform Azure",
+    "summary": "Addons required for the Camptocamp Cloud Platform on Azure",
+    "version": "9.0.1.0.0",
+    "author": "Camptocamp,Odoo Community Association (OCA)",
+    "license": "AGPL-3",
+    "category": "Extra Tools",
+    "depends": [
+        "cloud_platform",
+        "attachment_azure",
+        "monitoring_prometheus",
+    ],
+    "excludes": [
+        "cloud_platform_ovh",
+        "cloud_platform_exoscale",
+    ],
+    "website": "https://www.camptocamp.com",
+    "data": [],
+    "installable": True,
+}

cloud_platform_azure/models/__init__.py

@@ -0,0 +1,2 @@
+from __future__ import absolute_import
+from . import cloud_platform

cloud_platform_azure/models/cloud_platform.py

@@ -0,0 +1,127 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+from __future__ import absolute_import
+import re
+import os
+
+from openerp import models, api
+from openerp.addons.cloud_platform.models.cloud_platform import FilestoreKind
+from openerp.addons.cloud_platform.models.cloud_platform import PlatformConfig
+
+
+AZURE_STORE_KIND = FilestoreKind("azure", "remote")
+
+
+class CloudPlatform(models.AbstractModel):
+    _inherit = "cloud.platform"
+
+    @api.model
+    def _filestore_kinds(self):
+        kinds = super(CloudPlatform, self)._filestore_kinds()
+        kinds["azure"] = AZURE_STORE_KIND
+        return kinds
+
+    @api.model
+    def _platform_kinds(self):
+        kinds = super(CloudPlatform, self)._platform_kinds()
+        kinds.append("azure")
+        return kinds
+
+    @api.model
+    def _config_by_server_env_for_azure(self):
+        fs_kinds = self._filestore_kinds()
+        configs = {
+            "prod": PlatformConfig(filestore=fs_kinds["azure"]),
+            "integration": PlatformConfig(filestore=fs_kinds["azure"]),
+            "labs": PlatformConfig(filestore=fs_kinds["azure"]),
+            "test": PlatformConfig(filestore=fs_kinds["db"]),
+            "dev": PlatformConfig(filestore=fs_kinds["db"]),
+        }
+        return configs
+
+    @api.model
+    def _check_filestore(self, environment_name):
+        params = self.env["ir.config_parameter"].sudo()
+        use_azure = (params.get_param("ir_attachment.location") ==
+                     AZURE_STORE_KIND.name)
+        if environment_name in ("prod", "integration"):
+            # Labs instances use azure by default, but we don't want
+            # to enforce it in case we want to test something with a different
+            # storage. At your own risks!
+            assert use_azure, (
+                "azure must be used on production and integration instances. "
+                "It is activated by setting 'ir_attachment.location.' to 'azure'."
+                " The 'install()' function sets this option "
+                "automatically."
+            )
+        if use_azure:
+            key_sets = [
+                ["AZURE_STORAGE_USE_AAD", "AZURE_STORAGE_ACCOUNT_URL"],
+                ["AZURE_STORAGE_CONNECTION_STRING"],
+                [
+                    "AZURE_STORAGE_ACCOUNT_NAME",
+                    "AZURE_STORAGE_ACCOUNT_URL",
+                    "AZURE_STORAGE_ACCOUNT_KEY",
+                ],
+            ]
+            is_valid = False
+            for key_set in key_sets:
+                if all([os.environ.get(key) for key in key_set]):
+                    is_valid = True
+                    break
+            assert is_valid, (
+                "When ir_attachment.location is set to 'azure', "
+                "at least one of the following enviromnent variable set "
+                "is required : {}".format(
+                    " or ".join(
+                        [" + ".join([key for key in key_set]) for key_set in key_sets]
+                    )
+                )
+            )
+            storage_name = os.environ.get("AZURE_STORAGE_NAME", "")
+            if environment_name in ("prod", "integration", "labs"):
+                assert storage_name, (
+                    "AZURE_STORAGE_NAME environment variable is required when "
+                    "ir_attachment.location is 'azure'.\n"
+                    "Normally, 'azure' is activated on labs, integration "
+                    "and production, but should not be used in dev environment"
+                    " (or using a dedicated dev bucket, never using the "
+                    "integration/prod bucket).\n"
+                    "If you don't actually need a bucket, change the"
+                    " 'ir_attachment.location' parameter."
+                )
+            # A bucket name is defined under the following format
+            # ^[a-z]+\-[a-z]+\-\d+$
+            # Anything other than prod bucket must be suffixed with env name
+            #
+            # Use AZURE_STORAGE_NAME_UNSTRUCTURED to by-pass check
+            # on bucket name structure
+            if os.environ.get("AZURE_STORAGE_NAME_UNSTRUCTURED"):
+                return
+            prod_bucket = bool(re.match(ur"^[a-z]+\-[a-z]+\-\d+$", storage_name))
+            if environment_name == "prod":
+                assert prod_bucket, (
+                    "AZURE_STORAGE_NAME should match '^[a-z]+\\-[a-z]+\\-\\d+$', "
+                    "we got: '%s'" % (storage_name,)
+                )
+            else:
+                # if we are using the prod bucket on another instance
+                # such as an integration, we must be sure to be in read only!
+                assert not prod_bucket, (
+                    "AZURE_STORAGE_NAME should not match '^[a-z]+\\-[a-z]+\\-\\d+$', "
+                    "we got: '%s'" % (storage_name,)
+                )
+
+        elif environment_name == "test":
+            # store in DB so we don't have files local to the host
+            assert params.get_param("ir_attachment.location") == "db", (
+                "In test instances, files must be stored in the database with "
+                "'ir_attachment.location' set to 'db'. This is "
+                "automatically set by the function 'install()'."
+            )
+
+    @api.model
+    def install(self):
+        self._install("azure")

monitoring_log_requests/models/ir_http.py

@@ -11,7 +11,7 @@ from openerp.http import request as http_request
 from openerp.tools.config import config
 
 
-_logger = logging.getLogger('monitoring.http.requests')
+_logger = logging.getLogger(u'monitoring.http.requests')
 
 
 class IrHttp(models.AbstractModel):
@@ -28,8 +28,8 @@ class IrHttp(models.AbstractModel):
         return response
 
     def _monitoring_blacklist(self, request):
-        path_info = request.httprequest.environ.get('PATH_INFO')
-        if path_info.startswith('/longpolling/'):
+        path_info = request.httprequest.environ.get(u'PATH_INFO')
+        if path_info.startswith(u'/longpolling/'):
             return True
         return False
 
@@ -37,7 +37,7 @@ class IrHttp(models.AbstractModel):
         return True
 
     def _monitoring_info(self, request, response, begin, end):
-        path = request.httprequest.environ.get('PATH_INFO')
+        path = request.httprequest.environ.get(u'PATH_INFO')
         info = {
             # timing
             'start_time': time.strftime("%Y-%m-%d %H:%M:%S",

monitoring_prometheus/README.rst

@@ -0,0 +1,17 @@
+.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg
+    :alt: License
+
+==============================
+Monitoring: Prometheus metrics
+==============================
+
+Add an endpoint */metrics* to allow a Prometheus server to fetch application metrics.
+Current available metrics are:
+
+* Request completion time with 3 differentiators:
+  * Filestore
+  * Assets
+  * Everything else
+* Longpolling request count
+
+No additional configuration is needed, just ensure that the Prometheus server is allowed to communicate with Odoo

monitoring_prometheus/__init__.py

@@ -0,0 +1,3 @@
+from __future__ import absolute_import
+from . import controllers
+from . import models

monitoring_prometheus/__openerp__.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{
+    "name": "Monitoring: Prometheus Metrics",
+    "version": "9.0.1.0.0",
+    "author": "Camptocamp,Odoo Community Association (OCA)",
+    "license": "AGPL-3",
+    "category": "category",
+    "depends": [
+        "base",
+        "web",
+        "server_environment",
+    ],
+    "website": "http://www.camptocamp.com",
+    "data": [],
+    "external_dependencies": {
+        "python": ["prometheus_client"],
+    },
+    "installable": True,
+}

monitoring_prometheus/controllers/__init__.py

@@ -0,0 +1,2 @@
+from __future__ import absolute_import
+from . import prometheus_metrics

monitoring_prometheus/controllers/prometheus_metrics.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+from __future__ import absolute_import
+import logging
+
+from openerp.http import Controller, route
+
+_logger = logging.getLogger(__name__)
+
+try:
+    from prometheus_client import generate_latest
+except (ImportError, IOError), err:
+    _logger.warning(err)
+
+
+class PrometheusController(Controller):
+    @route(u'/metrics', auth=u'public')
+    def metrics(self):
+        return generate_latest()

monitoring_prometheus/models/__init__.py

@@ -0,0 +1,2 @@
+from __future__ import absolute_import
+from . import ir_http

monitoring_prometheus/models/ir_http.py

@@ -0,0 +1,50 @@
+# -*- coding: utf-8 -*-
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+from __future__ import with_statement
+from __future__ import absolute_import
+import logging
+
+from openerp import models
+from openerp.http import request
+
+_logger = logging.getLogger(__name__)
+
+try:
+    from prometheus_client import Summary, Counter
+except (ImportError, IOError), err:
+    _logger.warning(err)
+
+
+REQUEST_TIME = Summary(
+    "request_latency_sec", "Request response time in sec", ["query_type"]
+)
+LONGPOLLING_COUNT = Counter("longpolling", "Longpolling request count")
+
+
+class IrHttp(models.AbstractModel):
+    _inherit = "ir.http"
+
+    @classmethod
+    def _dispatch(cls):
+        path_info = request.httprequest.environ.get("PATH_INFO")
+
+        if path_info.startswith("/longpolling/"):
+            LONGPOLLING_COUNT.inc()
+            return super(IrHttp, cls)._dispatch()
+
+        if path_info.startswith("/metrics"):
+            return super(IrHttp, cls)._dispatch()
+
+        if path_info.startswith("/web/static"):
+            label = "assets"
+        elif path_info.startswith("/web/content"):
+            label = "filestore"
+        else:
+            label = "client"
+
+        res = None
+        with REQUEST_TIME.labels(label).time():
+            res = super(IrHttp, cls)._dispatch()
+
+        return res

requirements.txt

@@ -9,3 +9,4 @@ python-keystoneclient==3.22.0
 keystoneauth1==3.14.0
 # error with 5.x (ConstructorError: could not determine a constructor for the tag '!record')
 PyYAML==4.2b4
+prometheus_client==0.11.0