From 899f390f66698fe7d0e94957818e30ff7eda1742 Mon Sep 17 00:00:00 2001 From: Patrick Tombez <35060345+p-tombez@users.noreply.github.com> Date: Wed, 4 Aug 2021 11:29:31 +0200 Subject: [PATCH] [13.0][IMP] Make cloud_platform fully abstract + update related modules (#244) --- .travis.yml | 8 +- cloud_platform/README.rst | 4 +- cloud_platform/__manifest__.py | 24 ++- cloud_platform/models/cloud_platform.py | 172 +++--------------- cloud_platform/songs.py | 8 +- cloud_platform_exoscale/__manifest__.py | 33 ++-- .../models/cloud_platform.py | 100 ++++++++-- cloud_platform_ovh/__manifest__.py | 33 ++-- cloud_platform_ovh/models/cloud_platform.py | 105 +++++++++-- 9 files changed, 257 insertions(+), 230 deletions(-) diff --git a/.travis.yml b/.travis.yml index 17dc6cc..99d2eb0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,12 @@ addons: env: matrix: - LINT_CHECK="1" - - TESTS="1" ODOO_REPO="odoo/odoo" - - TESTS="1" ODOO_REPO="OCA/OCB" + - TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="cloud_platform_exoscale" + - TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="cloud_platform_exoscale" + - TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="cloud_platform_ovh" + - TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="cloud_platform_ovh" + - TESTS="1" ODOO_REPO="odoo/odoo" EXCLUDE="cloud_platform,cloud_platform_ovh,cloud_platform_exoscale" + - TESTS="1" ODOO_REPO="OCA/OCB" EXCLUDE="cloud_platform,cloud_platform_ovh,cloud_platform_exoscale" global: - VERSION="15.0" LINT_CHECK="0" TESTS="0" diff --git a/cloud_platform/README.rst b/cloud_platform/README.rst index 48e80ff..c6491ff 100644 --- a/cloud_platform/README.rst +++ b/cloud_platform/README.rst @@ -6,7 +6,7 @@ common to all platform providers. * Provide a quick install that we can call at the setup / migration of a database -* Check if the environment variables are configured correctly according - to the instance's environment (prod, integration, test or dev) to prevent +* Implements abstract methods to check if the environment variables are configured correctly + according to the instance's environment (prod, integration, test or dev) to prevent data corruption between the environments (such as the integration server writing on the production object storage). diff --git a/cloud_platform/__manifest__.py b/cloud_platform/__manifest__.py index 48f841f..bee5ab6 100644 --- a/cloud_platform/__manifest__.py +++ b/cloud_platform/__manifest__.py @@ -2,19 +2,17 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -{"name": "Cloud Platform", - "summary": "Addons required for the Camptocamp Cloud Platform", - "version": "15.0.1.0.0", - "author": "Camptocamp,Odoo Community Association (OCA)", - "license": "AGPL-3", - "category": "Extra Tools", - "depends": [ - "session_redis", - "monitoring_status", - "logging_json", - # "monitoring_log_requests", - "monitoring_statsd", - "server_environment", # OCA/server-tools +{'name': 'Cloud Platform', + 'summary': 'Addons required for the Camptocamp Cloud Platform', + 'version': "15.0.1.0.0", + 'author': 'Camptocamp,Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'Extra Tools', + 'depends': [ + 'session_redis', + 'monitoring_status', + 'logging_json', + 'server_environment', # OCA/server-tools ], "website": "https://www.camptocamp.com", "data": [], diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py index 006a9e2..edc1f5e 100644 --- a/cloud_platform/models/cloud_platform.py +++ b/cloud_platform/models/cloud_platform.py @@ -6,7 +6,6 @@ import os import re from collections import namedtuple - from distutils.util import strtobool from odoo import api, models @@ -26,20 +25,27 @@ PlatformConfig = namedtuple( ) -class FilestoreKind(object): - db = 'db' - s3 = 's3' # or compatible s3 object storage - swift = 'swift' - file = 'file' - - -DefaultConfig = PlatformConfig(filestore=FilestoreKind.db) +FilestoreKind = namedtuple( + 'FilestoreKind', + ['name', 'location'] +) class CloudPlatform(models.AbstractModel): _name = 'cloud.platform' _description = 'cloud.platform' + @api.model + def _default_config(self): + return PlatformConfig(self._filestore_kinds()['db']) + + @api.model + def _filestore_kinds(self): + return { + 'db': FilestoreKind('db', 'local'), + 'file': FilestoreKind('file', 'local'), + } + @api.model def _platform_kinds(self): return [] @@ -52,7 +58,7 @@ class CloudPlatform(models.AbstractModel): None ) configs = configs_getter() if configs_getter else {} - return configs.get(environment) or DefaultConfig + return configs.get(environment) or self._default_config() def _get_running_env(self): environment_name = config['running_env'] @@ -63,150 +69,25 @@ class CloudPlatform(models.AbstractModel): return environment_name @api.model - def install(self, platform_kind): + def _install(self, platform_kind): assert platform_kind in self._platform_kinds() params = self.env['ir.config_parameter'].sudo() params.set_param('cloud.platform.kind', platform_kind) environment_name = self._get_running_env() configs = self._config_by_server_env(platform_kind, environment_name) - params.set_param('ir_attachment.location', configs.filestore) + params.set_param('ir_attachment.location', configs.filestore.name) self.check() - if configs.filestore in [FilestoreKind.swift, FilestoreKind.s3]: + if configs.filestore.location == 'remote': self.env['ir.attachment'].sudo().force_storage() _logger.info('cloud platform configured for {}'.format(platform_kind)) @api.model - def _check_swift(self, environment_name): - params = self.env['ir.config_parameter'].sudo() - use_swift = (params.get_param('ir_attachment.location') == - FilestoreKind.swift) - if environment_name in ('prod', 'integration'): - # Labs instances use swift or s3 by default, but we don't want - # to enforce it in case we want to test something with a different - # storage. At your own risks! - assert use_swift, ( - "Swift must be used on production and integration instances. " - "It is activated, setting 'ir_attachment.location.' to 'swift'" - " The 'install_exoscale()' function sets this option " - "automatically." - ) - if use_swift: - assert os.environ.get('SWIFT_AUTH_URL'), ( - "SWIFT_AUTH_URL environment variable is required when " - "ir_attachment.location is 'swift'." - ) - assert os.environ.get('SWIFT_ACCOUNT'), ( - "SWIFT_ACCOUNT environment variable is required when " - "ir_attachment.location is 'swift'." - ) - assert os.environ.get('SWIFT_PASSWORD'), ( - "SWIFT_PASSWORD environment variable is required when " - "ir_attachment.location is 'swift'." - ) - container_name = os.environ.get('SWIFT_WRITE_CONTAINER', '') - if environment_name in ('prod', 'integration', 'labs'): - assert container_name, ( - "SWIFT_WRITE_CONTAINER environment variable is required when " - "ir_attachment.location is 'swift'.\n" - "Normally, 'swift' is activated on labs, integration " - "and production, but should not be used in dev environment" - " (or using a dedicated dev bucket, never using the " - "integration/prod bucket).\n" - "If you don't actually need a bucket, change the" - " 'ir_attachment.location' parameter." - ) - prod_container = bool(re.match(r'[a-z0-9-]+-odoo-prod', - container_name)) - # A bucket name is defined under the following format - # -odoo- - # - # Use SWIFT_WRITE_CONTAINER_UNSTRUCTURED to by-pass check on bucket name - # structure - if os.environ.get('SWIFT_WRITE_CONTAINER_UNSTRUCTURED'): - return - if environment_name == 'prod': - assert prod_container, ( - "SWIFT_WRITE_CONTAINER should match '-odoo-prod', " - "we got: '%s'" % (container_name,) - ) - else: - # if we are using the prod bucket on another instance - # such as an integration, we must be sure to be in read only! - assert not prod_container, ( - "SWIFT_WRITE_CONTAINER should not match " - "'-odoo-prod', we got: '%s'" % (container_name,) - ) - elif environment_name == 'test': - # store in DB so we don't have files local to the host - assert params.get_param('ir_attachment.location') == 'db', ( - "In test instances, files must be stored in the database with " - "'ir_attachment.location' set to 'db'. This is " - "automatically set by the function 'install_ovh()'." - ) + def install(self): + raise NotImplementedError @api.model - def _check_s3(self, environment_name): - params = self.env['ir.config_parameter'].sudo() - use_s3 = params.get_param('ir_attachment.location') == FilestoreKind.s3 - if environment_name in ('prod', 'integration'): - # Labs instances use swift or s3 by default, but we don't want - # to enforce it in case we want to test something with a different - # storage. At your own risks! - assert use_s3, ( - "S3 must be used on production and integration instances. " - "It is activated by setting 'ir_attachment.location.' to 's3'." - " The 'install_exoscale()' function sets this option " - "automatically." - ) - if use_s3: - assert os.environ.get('AWS_ACCESS_KEY_ID'), ( - "AWS_ACCESS_KEY_ID environment variable is required when " - "ir_attachment.location is 's3'." - ) - assert os.environ.get('AWS_SECRET_ACCESS_KEY'), ( - "AWS_SECRET_ACCESS_KEY environment variable is required when " - "ir_attachment.location is 's3'." - ) - bucket_name = os.environ.get('AWS_BUCKETNAME', '') - if environment_name in ('prod', 'integration', 'labs'): - assert bucket_name, ( - "AWS_BUCKETNAME environment variable is required when " - "ir_attachment.location is 's3'.\n" - "Normally, 's3' is activated on labs, integration " - "and production, but should not be used in dev environment" - " (or using a dedicated dev bucket, never using the " - "integration/prod bucket).\n" - "If you don't actually need a bucket, change the" - " 'ir_attachment.location' parameter." - ) - # A bucket name is defined under the following format - # -odoo- - # - # Use AWS_BUCKETNAME_UNSTRUCTURED to by-pass check on bucket name - # structure - if os.environ.get('AWS_BUCKETNAME_UNSTRUCTURED'): - return - prod_bucket = bool(re.match(r'[a-z-0-9]+-odoo-prod', bucket_name)) - if environment_name == 'prod': - assert prod_bucket, ( - "AWS_BUCKETNAME should match '-odoo-prod', " - "we got: '%s'" % (bucket_name,) - ) - else: - # if we are using the prod bucket on another instance - # such as an integration, we must be sure to be in read only! - assert not prod_bucket, ( - "AWS_BUCKETNAME should not match '-odoo-prod', " - "we got: '%s'" % (bucket_name,) - ) - - elif environment_name == 'test': - # store in DB so we don't have files local to the host - assert params.get_param('ir_attachment.location') == 'db', ( - "In test instances, files must be stored in the database with " - "'ir_attachment.location' set to 'db'. This is " - "automatically set by the function 'install_exoscale()'." - ) + def _check_filestore(self, environment_name): + raise NotImplementedError @api.model def _check_redis(self, environment_name): @@ -243,14 +124,11 @@ class CloudPlatform(models.AbstractModel): if not kind: _logger.warning( "cloud platform not configured, you should " - "probably run 'env['cloud.platform'].install_exoscale()'" + "probably run 'env['cloud.platform'].install()'" ) return environment_name = self._get_running_env() - if kind == 'exoscale': - self._check_s3(environment_name) - elif kind == 'ovh': - self._check_swift(environment_name) + self._check_filestore(environment_name) self._check_redis(environment_name) def _register_hook(self): diff --git a/cloud_platform/songs.py b/cloud_platform/songs.py index 7d3b4eb..043fc7b 100644 --- a/cloud_platform/songs.py +++ b/cloud_platform/songs.py @@ -1,7 +1,3 @@ -def install_exoscale(ctx): - ctx.env['cloud.platform'].install('exoscale') - - -def install_ovh(ctx): - ctx.env['cloud.platform'].install('ovh') +def install(ctx): + ctx.env['cloud.platform'].install() diff --git a/cloud_platform_exoscale/__manifest__.py b/cloud_platform_exoscale/__manifest__.py index 28504f9..8cca042 100644 --- a/cloud_platform_exoscale/__manifest__.py +++ b/cloud_platform_exoscale/__manifest__.py @@ -2,17 +2,22 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -{"name": "Cloud Platform Exoscale", - "summary": "Addons required for the Camptocamp Cloud Platform on Exoscale", - "version": "15.0.1.0.0", - "author": "Camptocamp,Odoo Community Association (OCA)", - "license": "AGPL-3", - "category": "Extra Tools", - "depends": [ - "cloud_platform", - "attachment_s3", - ], - "website": "https://www.camptocamp.com", - "data": [], - "installable": True, - } +{ + "name": "Cloud Platform Exoscale", + "summary": "Addons required for the Camptocamp Cloud Platform on Exoscale", + "version": "15.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Extra Tools", + "depends": [ + "cloud_platform", + "attachment_s3", + "monitoring_statsd", + ], + "excludes": [ + "cloud_platform_ovh", + ], + "website": "https://www.camptocamp.com", + "data": [], + "installable": True, +} diff --git a/cloud_platform_exoscale/models/cloud_platform.py b/cloud_platform_exoscale/models/cloud_platform.py index 7a9f840..3a29942 100644 --- a/cloud_platform_exoscale/models/cloud_platform.py +++ b/cloud_platform_exoscale/models/cloud_platform.py @@ -1,24 +1,26 @@ # Copyright 2016-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -import logging +import re +import os from odoo import models, api +from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind +from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig -_logger = logging.getLogger(__name__) -try: - from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind - from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig -except ImportError: - FilestoreKind = None - PlatformConfig = None - _logger.debug("Cannot 'import from cloud_platform'") +S3_STORE_KIND = FilestoreKind('s3', 'remote') class CloudPlatform(models.AbstractModel): _inherit = 'cloud.platform' + @api.model + def _filestore_kinds(self): + kinds = super(CloudPlatform, self)._filestore_kinds() + kinds['s3'] = S3_STORE_KIND + return kinds + @api.model def _platform_kinds(self): kinds = super(CloudPlatform, self)._platform_kinds() @@ -27,15 +29,81 @@ class CloudPlatform(models.AbstractModel): @api.model def _config_by_server_env_for_exoscale(self): + fs_kinds = self._filestore_kinds() configs = { - 'prod': PlatformConfig(filestore=FilestoreKind.s3), - 'integration': PlatformConfig(filestore=FilestoreKind.s3), - 'labs': PlatformConfig(filestore=FilestoreKind.s3), - 'test': PlatformConfig(filestore=FilestoreKind.db), - 'dev': PlatformConfig(filestore=FilestoreKind.db), + 'prod': PlatformConfig(filestore=fs_kinds['s3']), + 'integration': PlatformConfig(filestore=fs_kinds['s3']), + 'labs': PlatformConfig(filestore=fs_kinds['s3']), + 'test': PlatformConfig(filestore=fs_kinds['db']), + 'dev': PlatformConfig(filestore=fs_kinds['db']), } return configs @api.model - def install_exoscale(self): - self.install('exoscale') + def _check_filestore(self, environment_name): + params = self.env['ir.config_parameter'].sudo() + use_s3 = (params.get_param('ir_attachment.location') == + S3_STORE_KIND.name) + if environment_name in ('prod', 'integration'): + # Labs instances use s3 by default, but we don't want + # to enforce it in case we want to test something with a different + # storage. At your own risks! + assert use_s3, ( + "S3 must be used on production and integration instances. " + "It is activated by setting 'ir_attachment.location.' to 's3'." + " The 'install()' function sets this option " + "automatically." + ) + if use_s3: + assert os.environ.get('AWS_ACCESS_KEY_ID'), ( + "AWS_ACCESS_KEY_ID environment variable is required when " + "ir_attachment.location is 's3'." + ) + assert os.environ.get('AWS_SECRET_ACCESS_KEY'), ( + "AWS_SECRET_ACCESS_KEY environment variable is required when " + "ir_attachment.location is 's3'." + ) + bucket_name = os.environ.get('AWS_BUCKETNAME', '') + if environment_name in ('prod', 'integration', 'labs'): + assert bucket_name, ( + "AWS_BUCKETNAME environment variable is required when " + "ir_attachment.location is 's3'.\n" + "Normally, 's3' is activated on labs, integration " + "and production, but should not be used in dev environment" + " (or using a dedicated dev bucket, never using the " + "integration/prod bucket).\n" + "If you don't actually need a bucket, change the" + " 'ir_attachment.location' parameter." + ) + # A bucket name is defined under the following format + # -odoo- + # + # Use AWS_BUCKETNAME_UNSTRUCTURED to by-pass check on bucket name + # structure + if os.environ.get('AWS_BUCKETNAME_UNSTRUCTURED'): + return + prod_bucket = bool(re.match(r'[a-z-0-9]+-odoo-prod', bucket_name)) + if environment_name == 'prod': + assert prod_bucket, ( + "AWS_BUCKETNAME should match '-odoo-prod', " + "we got: '%s'" % (bucket_name,) + ) + else: + # if we are using the prod bucket on another instance + # such as an integration, we must be sure to be in read only! + assert not prod_bucket, ( + "AWS_BUCKETNAME should not match '-odoo-prod', " + "we got: '%s'" % (bucket_name,) + ) + + elif environment_name == 'test': + # store in DB so we don't have files local to the host + assert params.get_param('ir_attachment.location') == 'db', ( + "In test instances, files must be stored in the database with " + "'ir_attachment.location' set to 'db'. This is " + "automatically set by the function 'install()'." + ) + + @api.model + def install(self): + self._install('exoscale') diff --git a/cloud_platform_ovh/__manifest__.py b/cloud_platform_ovh/__manifest__.py index 5dde01d..9945f50 100644 --- a/cloud_platform_ovh/__manifest__.py +++ b/cloud_platform_ovh/__manifest__.py @@ -2,17 +2,22 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -{"name": "Cloud Platform OVH", - "summary": "Addons required for the Camptocamp Cloud Platform on OVH", - "version": "15.0.1.0.0", - "author": "Camptocamp,Odoo Community Association (OCA)", - "license": "AGPL-3", - "category": "Extra Tools", - "depends": [ - "cloud_platform", - "attachment_swift", - ], - "website": "https://www.camptocamp.com", - "data": [], - "installable": True, - } +{ + "name": "Cloud Platform OVH", + "summary": "Addons required for the Camptocamp Cloud Platform on OVH", + "version": "15.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Extra Tools", + "depends": [ + "cloud_platform", + "attachment_swift", + "monitoring_statsd", + ], + "excludes": [ + "cloud_platform_exoscale", + ], + "website": "https://www.camptocamp.com", + "data": [], + "installable": True, +} diff --git a/cloud_platform_ovh/models/cloud_platform.py b/cloud_platform_ovh/models/cloud_platform.py index 59cfc6c..6c73a37 100644 --- a/cloud_platform_ovh/models/cloud_platform.py +++ b/cloud_platform_ovh/models/cloud_platform.py @@ -1,24 +1,27 @@ # Copyright 2017-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -import logging +import re +import os from odoo import api, models -_logger = logging.getLogger(__name__) +from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind +from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig -try: - from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind - from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig -except ImportError: - FilestoreKind = None - PlatformConfig = None - _logger.debug("Cannot 'import from cloud_platform'") + +SWIFT_STORE_KIND = FilestoreKind('s3', 'remote') class CloudPlatform(models.AbstractModel): _inherit = 'cloud.platform' + @api.model + def _filestore_kinds(self): + kinds = super(CloudPlatform, self)._filestore_kinds() + kinds['swift'] = SWIFT_STORE_KIND + return kinds + @api.model def _platform_kinds(self): kinds = super()._platform_kinds() @@ -27,15 +30,85 @@ class CloudPlatform(models.AbstractModel): @api.model def _config_by_server_env_for_ovh(self): + fs_kinds = self._filestore_kinds() configs = { - 'prod': PlatformConfig(filestore=FilestoreKind.swift), - 'integration': PlatformConfig(filestore=FilestoreKind.swift), - 'labs': PlatformConfig(filestore=FilestoreKind.swift), - 'test': PlatformConfig(filestore=FilestoreKind.db), - 'dev': PlatformConfig(filestore=FilestoreKind.db), + 'prod': PlatformConfig(filestore=fs_kinds['swift']), + 'integration': PlatformConfig(filestore=fs_kinds['swift']), + 'labs': PlatformConfig(filestore=fs_kinds['swift']), + 'test': PlatformConfig(filestore=fs_kinds['db']), + 'dev': PlatformConfig(filestore=fs_kinds['db']), } return configs @api.model - def install_ovh(self): - self.install('ovh') + def _check_filestore(self, environment_name): + params = self.env['ir.config_parameter'].sudo() + use_swift = (params.get_param('ir_attachment.location') == + SWIFT_STORE_KIND.name) + if environment_name in ('prod', 'integration'): + # Labs instances use swift by default, but we don't want + # to enforce it in case we want to test something with a different + # storage. At your own risks! + assert use_swift, ( + "Swift must be used on production and integration instances. " + "It is activated, setting 'ir_attachment.location.' to 'swift'" + " The 'install()' function sets this option " + "automatically." + ) + if use_swift: + assert os.environ.get('SWIFT_AUTH_URL'), ( + "SWIFT_AUTH_URL environment variable is required when " + "ir_attachment.location is 'swift'." + ) + assert os.environ.get('SWIFT_ACCOUNT'), ( + "SWIFT_ACCOUNT environment variable is required when " + "ir_attachment.location is 'swift'." + ) + assert os.environ.get('SWIFT_PASSWORD'), ( + "SWIFT_PASSWORD environment variable is required when " + "ir_attachment.location is 'swift'." + ) + container_name = os.environ.get('SWIFT_WRITE_CONTAINER', '') + if environment_name in ('prod', 'integration', 'labs'): + assert container_name, ( + "SWIFT_WRITE_CONTAINER environment variable is required when " + "ir_attachment.location is 'swift'.\n" + "Normally, 'swift' is activated on labs, integration " + "and production, but should not be used in dev environment" + " (or using a dedicated dev bucket, never using the " + "integration/prod bucket).\n" + "If you don't actually need a bucket, change the" + " 'ir_attachment.location' parameter." + ) + prod_container = bool(re.match(r'[a-z0-9-]+-odoo-prod', + container_name)) + # A bucket name is defined under the following format + # -odoo- + # + # Use SWIFT_WRITE_CONTAINER_UNSTRUCTURED to by-pass check on bucket name + # structure + if os.environ.get('SWIFT_WRITE_CONTAINER_UNSTRUCTURED'): + return + if environment_name == 'prod': + assert prod_container, ( + "SWIFT_WRITE_CONTAINER should match '-odoo-prod', " + "we got: '%s'" % (container_name,) + ) + else: + # if we are using the prod bucket on another instance + # such as an integration, we must be sure to be in read only! + assert not prod_container, ( + "SWIFT_WRITE_CONTAINER should not match " + "'-odoo-prod', we got: '%s'" % (container_name,) + ) + elif environment_name == 'test': + # store in DB so we don't have files local to the host + assert params.get_param('ir_attachment.location') == 'db', ( + "In test instances, files must be stored in the database with " + "'ir_attachment.location' set to 'db'. This is " + "automatically set by the function 'install()'." + ) + + @api.model + def install(self): + self._install('ovh')