From d85722fdf15d3b1080665e0554368d4ba91b6292 Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Wed, 8 May 2019 11:43:43 +0200 Subject: [PATCH] Add cloud_platform --- cloud_platform/README.rst | 11 ++ cloud_platform/__init__.py | 3 + cloud_platform/__openerp__.py | 22 +++ cloud_platform/models/__init__.py | 2 + cloud_platform/models/cloud_platform.py | 248 ++++++++++++++++++++++++ oca_dependencies.txt | 1 + 6 files changed, 287 insertions(+) create mode 100644 cloud_platform/README.rst create mode 100644 cloud_platform/__init__.py create mode 100644 cloud_platform/__openerp__.py create mode 100644 cloud_platform/models/__init__.py create mode 100644 cloud_platform/models/cloud_platform.py create mode 100644 oca_dependencies.txt diff --git a/cloud_platform/README.rst b/cloud_platform/README.rst new file mode 100644 index 0000000..5e1e75c --- /dev/null +++ b/cloud_platform/README.rst @@ -0,0 +1,11 @@ +Cloud Platform +============== + +Install addons required for the Camptocamp Cloud platform. + +* Provide a quick install that we can call at the setup / migration + of a database +* Check if the environment variables are configured correctly according + to the instance's environment (prod, integration, test or dev) to prevent + data corruption between the environments (such as the integration server + writing on the production object storage). diff --git a/cloud_platform/__init__.py b/cloud_platform/__init__.py new file mode 100644 index 0000000..cde864b --- /dev/null +++ b/cloud_platform/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from . import models diff --git a/cloud_platform/__openerp__.py b/cloud_platform/__openerp__.py new file mode 100644 index 0000000..b9ebbfc --- /dev/null +++ b/cloud_platform/__openerp__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{'name': 'Cloud Platform', + 'summary': 'Addons required for the Camptocamp Cloud Platform', + 'version': '8.0.1.0.0', + 'author': 'Camptocamp,Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'Extra Tools', + 'depends': [ + 'base_attachment_object_storage', + 'session_redis', + 'monitoring_status', + 'logging_json', + 'server_environment', # OCA/server-tools + ], + 'website': 'http://www.camptocamp.com', + 'data': [], + 'installable': True, + } diff --git a/cloud_platform/models/__init__.py b/cloud_platform/models/__init__.py new file mode 100644 index 0000000..5f2c99d --- /dev/null +++ b/cloud_platform/models/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +from . import cloud_platform diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py new file mode 100644 index 0000000..f038f3d --- /dev/null +++ b/cloud_platform/models/cloud_platform.py @@ -0,0 +1,248 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import logging +import os +import re + +from collections import namedtuple + +from distutils.util import strtobool + +from openerp import SUPERUSER_ID +from openerp.osv import osv +from openerp.tools.config import config + + +_logger = logging.getLogger(__name__) + + +def is_true(strval): + return bool(strtobool(strval or '0'.lower())) + + +PlatformConfig = namedtuple( + 'PlatformConfig', + 'filestore' +) + + +class FilestoreKind(object): + db = 'db' + s3 = 's3' # or compatible s3 object storage + swift = 'swift' + file = 'file' + + +class CloudPlatform(osv.osv_abstract): + _name = 'cloud.platform' + + def _platform_kinds(self): + # XXX for backward compatibility, we need this one here, move + # it in cloud_platform_exoscale in V11 + return ['exoscale'] + + # XXX for backward compatibility, we need this one here, move + # it in cloud_platform_exoscale in V11 + def _config_by_server_env_for_exoscale(self): + configs = { + 'prod': PlatformConfig(filestore=FilestoreKind.s3), + 'integration': PlatformConfig(filestore=FilestoreKind.s3), + 'test': PlatformConfig(filestore=FilestoreKind.db), + 'dev': PlatformConfig(filestore=FilestoreKind.db), + } + return configs + + def _config_by_server_env(self, platform_kind, environment): + configs_getter = getattr( + self, + '_config_by_server_env_for_%s' % platform_kind, + None + ) + configs = configs_getter() if configs_getter else {} + return configs.get(environment) or FilestoreKind.db + + # Due to the addition of the ovh cloud platform + # This will be moved to cloud_platform_exoscale on v11 + def install_exoscale(self, cr, uid, context=None): + self.install(cr, uid, 'exoscale', context) + + def install(self, cr, uid, platform_kind, context=None): + assert platform_kind in self._platform_kinds() + params = self.pool.get('ir.config_parameter') + params.set_param( + cr, SUPERUSER_ID, + 'cloud.platform.kind', platform_kind, + context=context + ) + environment = config['running_env'] + configs = self._config_by_server_env(platform_kind, environment) + params.set_param( + cr, SUPERUSER_ID, + 'ir_attachment.location', configs.filestore, + context=context + ) + self.check(cr, uid, context) + if configs.filestore in [FilestoreKind.swift, FilestoreKind.s3]: + self.pool.get('ir.attachment').force_storage( + cr, SUPERUSER_ID, context=context + ) + _logger.info('cloud platform configured for {}'.format(platform_kind)) + + def _check_swift(self, cr, uid, environment_name, context=None): + params = self.pool.get('ir.config_parameter') + use_swift = ( + params.get_param( + cr, SUPERUSER_ID, 'ir_attachment.location', context=context + ) == FilestoreKind.swift + ) + if environment_name in ('prod', 'integration'): + assert use_swift, ( + "Swift must be used on production and integration instances. " + "It is activated, setting 'ir_attachment.location.' to 'swift'" + " The 'install_exoscale()' function sets this option " + "automatically." + ) + if use_swift: + assert os.environ.get('SWIFT_AUTH_URL'), ( + "SWIFT_AUTH_URL environment variable is required when " + "ir_attachment.location is 'swift'." + ) + assert os.environ.get('SWIFT_ACCOUNT'), ( + "SWIFT_ACCOUNT environment variable is required when " + "ir_attachment.location is 'swift'." + ) + assert os.environ.get('SWIFT_PASSWORD'), ( + "SWIFT_PASSWORD environment variable is required when " + "ir_attachment.location is 'swift'." + ) + container_name = os.environ['SWIFT_WRITE_CONTAINER'] + if environment_name in ('integration', 'prod'): + assert container_name, ( + "SWIFT_WRITE_CONTAINER must not be empty for prod " + "and integration" + ) + prod_container = bool(re.match(r'[a-z0-9_-]+-odoo-prod', + container_name)) + if environment_name == 'prod': + assert prod_container, ( + "SWIFT_WRITE_CONTAINER should match '-odoo-prod', " + "we got: '%s'" % (container_name,) + ) + else: + # if we are using the prod bucket on another instance + # such as an integration, we must be sure to be in read only! + assert not prod_container, ( + "SWIFT_WRITE_CONTAINER should not match " + "'-odoo-prod', we got: '%s'" % (container_name,) + ) + elif environment_name == 'test': + # store in DB so we don't have files local to the host + assert params.get_param(cr, SUPERUSER_ID, 'ir_attachment.location', + context=context) == 'db', ( + "In test instances, files must be stored in the database with " + "'ir_attachment.location' set to 'db'. This is " + "automatically set by the function 'install_ovh()'." + ) + + def _check_s3(self, cr, uid, environment_name, context=None): + params = self.pool.get('ir.config_parameter') + use_s3 = params.get_param( + cr, SUPERUSER_ID, 'ir_attachment.location', context=context + ) == FilestoreKind.s3 + if environment_name in ('prod', 'integration'): + assert use_s3, ( + "S3 must be used on production and integration instances. " + "It is activated by setting 'ir_attachment.location.' to 's3'." + " The 'install_exoscale()' function sets this option " + "automatically." + ) + if use_s3: + assert os.environ.get('AWS_ACCESS_KEY_ID'), ( + "AWS_ACCESS_KEY_ID environment variable is required when " + "ir_attachment.location is 's3'." + ) + assert os.environ.get('AWS_SECRET_ACCESS_KEY'), ( + "AWS_SECRET_ACCESS_KEY environment variable is required when " + "ir_attachment.location is 's3'." + ) + assert os.environ.get('AWS_BUCKETNAME'), ( + "AWS_BUCKETNAME environment variable is required when " + "ir_attachment.location is 's3'.\n" + "Normally, 's3' is activated on integration and production, " + "but should not be used in dev environment (or at least " + "not with a dev bucket, but never the " + "integration/prod bucket)." + ) + bucket_name = os.environ['AWS_BUCKETNAME'] + prod_bucket = bool(re.match(r'[a-z0-9_-]+-odoo-prod', bucket_name)) + if environment_name == 'prod': + assert prod_bucket, ( + "AWS_BUCKETNAME should match '-odoo-prod', " + "we got: '%s'" % (bucket_name,) + ) + else: + # if we are using the prod bucket on another instance + # such as an integration, we must be sure to be in read only! + assert not prod_bucket, ( + "AWS_BUCKETNAME should not match '-odoo-prod', " + "we got: '%s'" % (bucket_name,) + ) + + elif environment_name == 'test': + # store in DB so we don't have files local to the host + assert params.get_param(cr, SUPERUSER_ID, 'ir_attachment.location', + context=context) == 'db', ( + "In test instances, files must be stored in the database with " + "'ir_attachment.location' set to 'db'. This is " + "automatically set by the function 'install_exoscale()'." + ) + + def _check_redis(self, cr, uid, environment_name, context=None): + if environment_name in ('prod', 'integration', 'test'): + assert is_true(os.environ.get('ODOO_SESSION_REDIS')), ( + "Redis must be activated on prod, integration, test instances." + "This is done by setting ODOO_SESSION_REDIS=1." + ) + assert (os.environ.get('ODOO_SESSION_REDIS_HOST') or + os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST')), ( + "ODOO_SESSION_REDIS_HOST or ODOO_SESSION_REDIS_SENTINEL_HOST " + "environment variable is required to connect on Redis" + ) + assert os.environ.get('ODOO_SESSION_REDIS_PREFIX'), ( + "ODOO_SESSION_REDIS_PREFIX environment variable is required " + "to store sessions on Redis" + ) + + prefix = os.environ['ODOO_SESSION_REDIS_PREFIX'] + assert re.match(r'[a-z0-9_-]+-odoo-[a-z]+', prefix), ( + "ODOO_SESSION_REDIS_PREFIX must match '-odoo-'" + ", we got: '%s'" % (prefix,) + ) + + def check(self, cr, uid, context=None): + if is_true(os.environ.get('ODOO_CLOUD_PLATFORM_UNSAFE')): + _logger.warning( + "cloud platform checks disabled, this is not safe" + ) + return + params = self.pool.get('ir.config_parameter') + kind = params.get_param(cr, SUPERUSER_ID, + 'cloud.platform.kind', context=None) + if not kind: + _logger.warning( + "cloud platform not configured, you should " + "probably run 'env['cloud.platform'].install_exoscale()'" + ) + return + environment_name = config['running_env'] + if kind == 'exoscale': + self._check_s3(cr, uid, environment_name, context) + elif kind == 'ovh': + self._check_swift(cr, uid, environment_name, context) + self._check_redis(cr, uid, environment_name, context) + + def _register_hook(self, cr): + super(CloudPlatform, self)._register_hook(cr) + self.pool.get('cloud.platform').check(cr, SUPERUSER_ID) diff --git a/oca_dependencies.txt b/oca_dependencies.txt new file mode 100644 index 0000000..9c8c917 --- /dev/null +++ b/oca_dependencies.txt @@ -0,0 +1 @@ +server-tools