Merge pull request #299 from camptocamp/9.0-bp-cloud-abstract

[9.0] Backport of cloud abstraction

.travis.yml

@@ -27,8 +27,10 @@ virtualenv:
 env:
   matrix:
   - LINT_CHECK="1"
-  - TESTS="1" ODOO_REPO="odoo/odoo"
+  - TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="cloud_platform_ovh"
-  - TESTS="1" ODOO_REPO="OCA/OCB"
+  - TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="cloud_platform_ovh"
+  - TESTS="1" ODOO_REPO="odoo/odoo" EXCLUDE="cloud_platform,cloud_platform_ovh"
+  - TESTS="1" ODOO_REPO="OCA/OCB" EXCLUDE="cloud_platform,cloud_platform_ovh"
   global:
   - VERSION="9.0" LINT_CHECK="0" TESTS="0"
 

README.md

@@ -145,10 +145,7 @@ some parameters such as the `ir_attachment.location` and migrate the existing
 attachments to the object storage.
 
 From `anthem`, it can be called like this:
-    `ctx.env['cloud.platform'].install(cloud_platform_kind)`
+    `ctx.env['cloud.platform'].install()`
-Replacing `cloud_platform_kind` with 'exoscale' or 'ovh'
-
-Or using one of the direct shortcuts:
 
 ### Startup checks
 

cloud_platform/README.rst

@@ -5,7 +5,7 @@ Install addons required for the Camptocamp Cloud platform.
 
 * Provide a quick install that we can call at the setup / migration
   of a database
-* Check if the environment variables are configured correctly according
+* Implements abstract methods to check if the environment variables are configured correctly
-  to the instance's environment (prod, integration, test or dev) to prevent
+  according to the instance's environment (prod, integration, test or dev) to prevent
   data corruption between the environments (such as the integration server
   writing on the production object storage).

cloud_platform/__openerp__.py

@@ -14,8 +14,6 @@
      'session_redis',
      'monitoring_status',
      'logging_json',
-     # 'monitoring_log_requests',
-     'monitoring_statsd',
      'server_environment',  # OCA/server-tools
  ],
  'website': 'http://www.camptocamp.com',

cloud_platform/models/cloud_platform.py

@@ -7,11 +7,9 @@ import os
 import re
 
 from collections import namedtuple
-
 from distutils.util import strtobool
 
 from openerp import api, models, SUPERUSER_ID
-from openerp.tools.config import config
 
 
 _logger = logging.getLogger(__name__)
@@ -27,19 +25,26 @@ PlatformConfig = namedtuple(
 )
 
 
-class FilestoreKind(object):
+FilestoreKind = namedtuple(
-    db = 'db'
+    'FilestoreKind',
-    s3 = 's3'  # or compatible s3 object storage
+    ['name', 'location']
-    swift = 'swift'
+)
-    file = 'file'
-
-
-DefaultConfig = PlatformConfig(filestore=FilestoreKind.db)
 
 
 class CloudPlatform(models.AbstractModel):
     _name = 'cloud.platform'
 
+    @api.model
+    def _default_config(self):
+        return PlatformConfig(self._filestore_kinds()['db'])
+
+    @api.model
+    def _filestore_kinds(self):
+        return {
+            'db': FilestoreKind('db', 'local'),
+            'file': FilestoreKind('file', 'local'),
+        }
+
     @api.model
     def _platform_kinds(self):
         # XXX for backward compatibility, we need this one here, move
@@ -66,7 +71,7 @@ class CloudPlatform(models.AbstractModel):
             None
         )
         configs = configs_getter() if configs_getter else {}
-        return configs.get(environment) or DefaultConfig
+        return configs.get(environment) or self._default_config()
 
     # Due to the addition of the ovh cloud platform
     # This will be moved to cloud_platform_exoscale on v11
@@ -75,130 +80,25 @@ class CloudPlatform(models.AbstractModel):
         self.install('exoscale')
 
     @api.model
-    def install(self, platform_kind):
+    def _install(self, platform_kind):
         assert platform_kind in self._platform_kinds()
         params = self.env['ir.config_parameter'].sudo()
         params.set_param('cloud.platform.kind', platform_kind)
-        environment = config['running_env']
+        environment_name = self._get_running_env()
-        configs = self._config_by_server_env(platform_kind, environment)
+        configs = self._config_by_server_env(platform_kind, environment_name)
-        params.set_param('ir_attachment.location', configs.filestore)
+        params.set_param('ir_attachment.location', configs.filestore.name)
         self.check()
-        if configs.filestore in [FilestoreKind.swift, FilestoreKind.s3]:
+        if configs.filestore.location == 'remote':
             self.env['ir.attachment'].sudo().force_storage()
         _logger.info('cloud platform configured for {}'.format(platform_kind))
 
     @api.model
-    def _check_swift(self, environment_name):
+    def install(self):
-        params = self.env['ir.config_parameter'].sudo()
+        raise NotImplementedError
-        use_swift = (params.get_param('ir_attachment.location') ==
-                     FilestoreKind.swift)
-        if environment_name in ('prod', 'integration'):
-            assert use_swift, (
-                "Swift must be used on production and integration instances. "
-                "It is activated, setting 'ir_attachment.location.' to 'swift'"
-                " The 'install_exoscale()' function sets this option "
-                "automatically."
-            )
-        if use_swift:
-            assert os.environ.get('SWIFT_AUTH_URL'), (
-                "SWIFT_AUTH_URL environment variable is required when "
-                "ir_attachment.location is 'swift'."
-            )
-            assert os.environ.get('SWIFT_ACCOUNT'), (
-                "SWIFT_ACCOUNT environment variable is required when "
-                "ir_attachment.location is 'swift'."
-            )
-            assert os.environ.get('SWIFT_PASSWORD'), (
-                "SWIFT_PASSWORD environment variable is required when "
-                "ir_attachment.location is 'swift'."
-            )
-            container_name = os.environ.get('SWIFT_WRITE_CONTAINER', '')
-            if environment_name in ('prod', 'integration', 'labs'):
-                assert container_name, (
-                    "SWIFT_WRITE_CONTAINER environment variable is required when "
-                    "ir_attachment.location is 'swift'.\n"
-                    "Normally, 'swift' is activated on labs, integration "
-                    "and production, but should not be used in dev environment"
-                    " (or using a dedicated dev bucket, never using the "
-                    "integration/prod bucket).\n"
-                    "If you don't actually need a bucket, change the"
-                    " 'ir_attachment.location' parameter."
-                )
-            prod_container = bool(re.match(r'[a-z0-9-]+-odoo-prod',
-                                           container_name))
-            if environment_name == 'prod':
-                assert prod_container, (
-                    "SWIFT_WRITE_CONTAINER should match '<client>-odoo-prod', "
-                    "we got: '%s'" % (container_name,)
-                )
-            else:
-                # if we are using the prod bucket on another instance
-                # such as an integration, we must be sure to be in read only!
-                assert not prod_container, (
-                    "SWIFT_WRITE_CONTAINER should not match "
-                    "'<client>-odoo-prod', we got: '%s'" % (container_name,)
-                )
-        elif environment_name == 'test':
-            # store in DB so we don't have files local to the host
-            assert params.get_param('ir_attachment.location') == 'db', (
-                "In test instances, files must be stored in the database with "
-                "'ir_attachment.location' set to 'db'. This is "
-                "automatically set by the function 'install_ovh()'."
-            )
 
     @api.model
-    def _check_s3(self, environment_name):
+    def _check_filestore(self, environment_name):
-        params = self.env['ir.config_parameter'].sudo()
+        raise NotImplementedError
-        use_s3 = params.get_param('ir_attachment.location') == FilestoreKind.s3
-        if environment_name in ('prod', 'integration'):
-            assert use_s3, (
-                "S3 must be used on production and integration instances. "
-                "It is activated by setting 'ir_attachment.location.' to 's3'."
-                " The 'install_exoscale()' function sets this option "
-                "automatically."
-            )
-        if use_s3:
-            assert os.environ.get('AWS_ACCESS_KEY_ID'), (
-                "AWS_ACCESS_KEY_ID environment variable is required when "
-                "ir_attachment.location is 's3'."
-            )
-            assert os.environ.get('AWS_SECRET_ACCESS_KEY'), (
-                "AWS_SECRET_ACCESS_KEY environment variable is required when "
-                "ir_attachment.location is 's3'."
-            )
-            bucket_name = os.environ.get('AWS_BUCKETNAME', '')
-            if environment_name in ('prod', 'integration', 'labs'):
-                assert bucket_name, (
-                    "AWS_BUCKETNAME environment variable is required when "
-                    "ir_attachment.location is 's3'.\n"
-                    "Normally, 's3' is activated on labs, integration "
-                    "and production, but should not be used in dev environment"
-                    " (or using a dedicated dev bucket, never using the "
-                    "integration/prod bucket).\n"
-                    "If you don't actually need a bucket, change the"
-                    " 'ir_attachment.location' parameter."
-                )
-            prod_bucket = bool(re.match(r'[a-z-0-9]+-odoo-prod', bucket_name))
-            if environment_name == 'prod':
-                assert prod_bucket, (
-                    "AWS_BUCKETNAME should match '<client>-odoo-prod', "
-                    "we got: '%s'" % (bucket_name,)
-                )
-            else:
-                # if we are using the prod bucket on another instance
-                # such as an integration, we must be sure to be in read only!
-                assert not prod_bucket, (
-                    "AWS_BUCKETNAME should not match '<client>-odoo-prod', "
-                    "we got: '%s'" % (bucket_name,)
-                )
-
-        elif environment_name == 'test':
-            # store in DB so we don't have files local to the host
-            assert params.get_param('ir_attachment.location') == 'db', (
-                "In test instances, files must be stored in the database with "
-                "'ir_attachment.location' set to 'db'. This is "
-                "automatically set by the function 'install_exoscale()'."
-            )
 
     @api.model
     def _check_redis(self, environment_name):
@@ -208,8 +108,11 @@ class CloudPlatform(models.AbstractModel):
                 "This is done by setting ODOO_SESSION_REDIS=1."
             )
             assert (os.environ.get('ODOO_SESSION_REDIS_HOST') or
-                    os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST')), (
+                    os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST') or
-                "ODOO_SESSION_REDIS_HOST or ODOO_SESSION_REDIS_SENTINEL_HOST "
+                    os.environ.get('ODOO_SESSION_REDIS_URL')), (
+                "ODOO_SESSION_REDIS_HOST or "
+                "ODOO_SESSION_REDIS_SENTINEL_HOST or "
+                "ODOO_SESSION_REDIS_URL "
                 "environment variable is required to connect on Redis"
             )
             assert os.environ.get('ODOO_SESSION_REDIS_PREFIX'), (
@@ -235,14 +138,11 @@ class CloudPlatform(models.AbstractModel):
         if not kind:
             _logger.warning(
                 "cloud platform not configured, you should "
-                "probably run 'env['cloud.platform'].install_exoscale()'"
+                "probably run 'env['cloud.platform'].install()'"
             )
             return
-        environment_name = config['running_env']
+        environment_name = self._get_running_env()
-        if kind == 'exoscale':
+        self._check_filestore(environment_name)
-            self._check_s3(environment_name)
-        elif kind == 'ovh':
-            self._check_swift(environment_name)
         self._check_redis(environment_name)
 
     @api.cr

cloud_platform/songs.py

@@ -1,9 +1,5 @@
 # -*- coding: utf-8 -*-
 
 
-def install_exoscale(ctx):
+def install(ctx):
-    ctx.env['cloud.platform'].install_exoscale()
+    ctx.env['cloud.platform'].install()
-
-
-def install_ovh(ctx):
-    ctx.env['cloud.platform'].install('ovh')

cloud_platform_ovh/__openerp__.py

@@ -3,17 +3,22 @@
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
 
-{'name': 'Cloud Platform OVH',
+{
- 'summary': 'Addons required for the Camptocamp Cloud Platform on OVH',
+    "name": "Cloud Platform OVH",
- 'version': '9.0.1.1.0',
+    "summary": "Addons required for the Camptocamp Cloud Platform on OVH",
- 'author': 'Camptocamp,Odoo Community Association (OCA)',
+    "version": "9.0.1.1.0",
- 'license': 'AGPL-3',
+    "author": "Camptocamp,Odoo Community Association (OCA)",
- 'category': 'Extra Tools',
+    "license": "AGPL-3",
- 'depends': [
+    "category": "Extra Tools",
-     'cloud_platform',
+    "depends": [
-     'attachment_swift',
+        "cloud_platform",
+        "attachment_swift",
+        "monitoring_statsd",
     ],
- 'website': 'https://www.camptocamp.com',
+    "excludes": [
- 'data': [],
+        "cloud_platform_exoscale",
- 'installable': True,
+    ],
+    "website": "https://www.camptocamp.com",
+    "data": [],
+    "installable": True,
 }

cloud_platform_ovh/models/cloud_platform.py

@@ -2,26 +2,27 @@
 # Copyright 2017 Camptocamp SA
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
-import logging
+import re
+import os
 
 from openerp import api, models
 
-_logger = logging.getLogger(__name__)
+from openerp.addons.cloud_platform.models.cloud_platform import FilestoreKind
+from openerp.addons.cloud_platform.models.cloud_platform import PlatformConfig
 
-try:
+
-    from openerp.addons.cloud_platform.models.cloud_platform \
+SWIFT_STORE_KIND = FilestoreKind('swift', 'remote')
-        import FilestoreKind
-    from openerp.addons.cloud_platform.models.cloud_platform \
-        import PlatformConfig
-except ImportError:
-    FilestoreKind = None
-    PlatformConfig = None
-    _logger.debug("Cannot 'import from cloud_platform'")
 
 
 class CloudPlatform(models.AbstractModel):
     _inherit = 'cloud.platform'
 
+    @api.model
+    def _filestore_kinds(self):
+        kinds = super(CloudPlatform, self)._filestore_kinds()
+        kinds['swift'] = SWIFT_STORE_KIND
+        return kinds
+
     @api.model
     def _platform_kinds(self):
         kinds = super(CloudPlatform, self)._platform_kinds()
@@ -30,14 +31,85 @@ class CloudPlatform(models.AbstractModel):
 
     @api.model
     def _config_by_server_env_for_ovh(self):
+        fs_kinds = self._filestore_kinds()
         configs = {
-            'prod': PlatformConfig(filestore=FilestoreKind.swift),
+            'prod': PlatformConfig(filestore=fs_kinds['swift']),
-            'integration': PlatformConfig(filestore=FilestoreKind.swift),
+            'integration': PlatformConfig(filestore=fs_kinds['swift']),
-            'test': PlatformConfig(filestore=FilestoreKind.db),
+            'labs': PlatformConfig(filestore=fs_kinds['swift']),
-            'dev': PlatformConfig(filestore=FilestoreKind.db),
+            'test': PlatformConfig(filestore=fs_kinds['db']),
+            'dev': PlatformConfig(filestore=fs_kinds['db']),
         }
         return configs
 
     @api.model
-    def install_ovh(self):
+    def _check_filestore(self, environment_name):
-        self.install('ovh')
+        params = self.env['ir.config_parameter'].sudo()
+        use_swift = (params.get_param('ir_attachment.location') ==
+                     SWIFT_STORE_KIND.name)
+        if environment_name in ('prod', 'integration'):
+            # Labs instances use swift by default, but we don't want
+            # to enforce it in case we want to test something with a different
+            # storage. At your own risks!
+            assert use_swift, (
+                "Swift must be used on production and integration instances. "
+                "It is activated, setting 'ir_attachment.location.' to 'swift'"
+                " The 'install()' function sets this option "
+                "automatically."
+            )
+        if use_swift:
+            assert os.environ.get('SWIFT_AUTH_URL'), (
+                "SWIFT_AUTH_URL environment variable is required when "
+                "ir_attachment.location is 'swift'."
+            )
+            assert os.environ.get('SWIFT_ACCOUNT'), (
+                "SWIFT_ACCOUNT environment variable is required when "
+                "ir_attachment.location is 'swift'."
+            )
+            assert os.environ.get('SWIFT_PASSWORD'), (
+                "SWIFT_PASSWORD environment variable is required when "
+                "ir_attachment.location is 'swift'."
+            )
+            container_name = os.environ.get('SWIFT_WRITE_CONTAINER', '')
+            if environment_name in ('prod', 'integration', 'labs'):
+                assert container_name, (
+                    "SWIFT_WRITE_CONTAINER environment variable is required when "
+                    "ir_attachment.location is 'swift'.\n"
+                    "Normally, 'swift' is activated on labs, integration "
+                    "and production, but should not be used in dev environment"
+                    " (or using a dedicated dev bucket, never using the "
+                    "integration/prod bucket).\n"
+                    "If you don't actually need a bucket, change the"
+                    " 'ir_attachment.location' parameter."
+                )
+            prod_container = bool(re.match(r'[a-z0-9-]+-odoo-prod',
+                                           container_name))
+            # A bucket name is defined under the following format
+            # <client>-odoo-<env>
+            #
+            # Use SWIFT_WRITE_CONTAINER_UNSTRUCTURED to by-pass check on bucket name
+            # structure
+            if os.environ.get('SWIFT_WRITE_CONTAINER_UNSTRUCTURED'):
+                return
+            if environment_name == 'prod':
+                assert prod_container, (
+                    "SWIFT_WRITE_CONTAINER should match '<client>-odoo-prod', "
+                    "we got: '%s'" % (container_name,)
+                )
+            else:
+                # if we are using the prod bucket on another instance
+                # such as an integration, we must be sure to be in read only!
+                assert not prod_container, (
+                    "SWIFT_WRITE_CONTAINER should not match "
+                    "'<client>-odoo-prod', we got: '%s'" % (container_name,)
+                )
+        elif environment_name == 'test':
+            # store in DB so we don't have files local to the host
+            assert params.get_param('ir_attachment.location') == 'db', (
+                "In test instances, files must be stored in the database with "
+                "'ir_attachment.location' set to 'db'. This is "
+                "automatically set by the function 'install()'."
+            )
+
+    @api.model
+    def install(self):
+        self._install('ovh')