diff --git a/README.md b/README.md index 54538fc..12d7f6d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,94 @@ -Odoo Monitoring -=============== +# Cloud Platform -Camptocamp odoo addons for monitoring / usage analysis. +Camptocamp odoo addons used on our Cloud Platform. + +## Introduction + +On the platform we want to achieve having: + +* no data stored on the local filesystem so we can move an instance + between hosts and even have several running front-ends +* metrics read from the logs or sent to Prometheus to monitor the instances + +For the storage, we store all the attachments on a object storage such as S3 or a S3 compatible one, and we store the werkzeug sessions on Redis. + +For the metrics, we produce logs as json (TODO) that is sent +to ELK and send data to Prometheus/Grafana using statsd. + +## Setup + +### Python dependencies + +Libraries that must be added in ``requirements.txt``: + +``` +boto==2.42.0 +redis==2.10.5 +``` + +### Server Environment + +The server environments in `server_environment_files` must be at least: + +* `prod` +* `integration` +* `test` +* `dev` + +The exact naming is important, because the `cloud_platform` addon rely on these keys to know and check the running environment. + + +### Attachments in the Object Storage + +* prod: stored RW in the object storage + * `AWS_HOST`: depends of the platform + * `AWS_ACCESS_KEY_ID`: depends of the platform + * `AWS_SECRET_ACCESS_KEY`: depends of the platform + * `AWS_BUCKETNAME`: `-odoo-prod` +* integration: stored RO in the production object storage, fallback + on database for the writes + * `AWS_HOST`: depends of the platform + * `AWS_ACCESS_KEY_ID`: depends of the platform + * `AWS_SECRET_ACCESS_KEY`: depends of the platform + * `AWS_BUCKETNAME`: `-odoo-prod` (this is normal, we read only the data) + * `AWS_ATTACHMENT_READONLY`: `1` +* test: attachments are stored in database + +Besides, the + * `ir.config_parameter` `ir_attachment.location`: `s3://` ( + +### Sessions in Redis + +* prod: + * `ODOO_SESSION_REDIS`: 1 + * `ODOO_SESSION_REDIS_HOST`: depends of the platform + * `ODOO_SESSION_REDIS_PASSWORD`: depends of the platform + * `ODOO_SESSION_REDIS_PREFIX`: `-odoo-prod` +* integration: + * `ODOO_SESSION_REDIS`: 1 + * `ODOO_SESSION_REDIS_HOST`: depends of the platform + * `ODOO_SESSION_REDIS_PASSWORD`: depends of the platform + * `ODOO_SESSION_REDIS_PREFIX`: `-odoo-integration` +* test: + * `ODOO_SESSION_REDIS`: 1 + * `ODOO_SESSION_REDIS_HOST`: depends of the platform + * `ODOO_SESSION_REDIS_PASSWORD`: depends of the platform + * `ODOO_SESSION_REDIS_PREFIX`: `-odoo-test` + * `ODOO_SESSION_REDIS_EXPIRATION`: `86400` (1 day) + +### Automatic Configuration + +Calling `ctx.env['cloud.platform'].install_exoscale()` in an +`anthem` song will configure some parameters such as the +`ir_attachment.location` and migrate the existing attachments to the +object storage. + + +### Startup checks + +At loading of the database, the addon will check if the environment variables +for Redis and the object storage are set as expected for the loaded +environment. It will refuse to start if anything is badly configured. + +The checks can be bypassed with the environment variable +`ODOO_CLOUD_PLATFORM_UNSAFE` set to `1`. diff --git a/cloud_platform/README.rst b/cloud_platform/README.rst new file mode 100644 index 0000000..5e1e75c --- /dev/null +++ b/cloud_platform/README.rst @@ -0,0 +1,11 @@ +Cloud Platform +============== + +Install addons required for the Camptocamp Cloud platform. + +* Provide a quick install that we can call at the setup / migration + of a database +* Check if the environment variables are configured correctly according + to the instance's environment (prod, integration, test or dev) to prevent + data corruption between the environments (such as the integration server + writing on the production object storage). diff --git a/cloud_platform/__init__.py b/cloud_platform/__init__.py new file mode 100644 index 0000000..cde864b --- /dev/null +++ b/cloud_platform/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from . import models diff --git a/cloud_platform/__openerp__.py b/cloud_platform/__openerp__.py new file mode 100644 index 0000000..cd0042e --- /dev/null +++ b/cloud_platform/__openerp__.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{'name': 'Cloud Platform', + 'summary': 'Addons required for the Camptocamp Cloud Platform', + 'version': '9.0.1.0.0', + 'author': 'Camptocamp', + 'license': 'AGPL-3', + 'category': 'Extra Tools', + 'depends': [ + 'attachment_s3', + #'monitoring_log_requests', + 'server_environment', # OCA/server-tools + ], + 'website': 'http://www.camptocamp.com', + 'data': [], + 'installable': True, + } diff --git a/cloud_platform/models/__init__.py b/cloud_platform/models/__init__.py new file mode 100644 index 0000000..5f2c99d --- /dev/null +++ b/cloud_platform/models/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +from . import cloud_platform diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py new file mode 100644 index 0000000..c9204e4 --- /dev/null +++ b/cloud_platform/models/cloud_platform.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import logging +import os +import re + +from collections import namedtuple + +from distutils.util import strtobool + +from openerp import api, models, SUPERUSER_ID +from openerp.tools.config import config + + +_logger = logging.getLogger(__name__) + + +def is_true(strval): + return bool(strtobool(strval or '0'.lower())) + + +PlatformConfig = namedtuple( + 'PlatformConfig', + 'filestore filestore_readonly' +) + + +class FilestoreKind(object): + db = 'db' + s3 = 's3://' # or compatible s3 object storage + file = 'file' + + + +class CloudPlatform(models.AbstractModel): + _name = 'cloud.platform' + + @api.model + def _config_by_server_env(self, environment): + configs = { + 'prod': PlatformConfig(filestore=FilestoreKind.s3, + filestore_readonly=False), + 'integration': PlatformConfig(filestore=FilestoreKind.s3, + filestore_readonly=True), + 'test': PlatformConfig(filestore=FilestoreKind.db, + filestore_readonly=True), + 'dev': PlatformConfig(filestore=FilestoreKind.db, + filestore_readonly=True), + } + return configs.get(environment) or configs['dev'] + + @api.model + def install_exoscale(self): + params = self.env['ir.config_parameter'].sudo() + params.set_param('cloud.platform.kind', 'exoscale') + environment = config['running_env'] + configs = self._config_by_server_env(environment) + params.set_param('ir_attachment.location', configs.filestore) + self.check(self.env) + if configs.filestore == FilestoreKind.s3: + self.env['ir.attachment'].sudo().force_storage() + _logger.info('cloud platform configured for exoscale') + + @api.model + def _check_s3(self, environment_name): + params = self.env['ir.config_parameter'].sudo() + attachment_readonly = is_true( + os.environ.get('AWS_ATTACHMENT_READONLY') + ) + if environment_name in ('prod', 'integration'): + assert os.environ.get('AWS_ACCESS_KEY_ID') + assert os.environ.get('AWS_SECRET_ACCESS_KEY') + assert os.environ.get('AWS_BUCKETNAME') + bucket_name = os.environ['AWS_BUCKETNAME'] + assert re.match(r'[a-z]+-odoo-prod', bucket_name), ( + "AWS_BUCKETNAME should match '-odoo-prod', " + "we got: '%s'" % (bucket_name,) + ) + assert params.get_param('ir_attachment.location') == 's3://' + # on the integration, we read the filestore from the production + # s3, but we must be readonly! + if environment_name == 'integration': + assert attachment_readonly + else: + assert not attachment_readonly + elif environment_name == 'test': + assert params.get_param('ir_attachment.location') == 'db' + else: + assert params.get_param('ir_attachment.location') in ('db', 'file') + + @api.model + def _check_redis(self, environment_name): + assert is_true(os.environ.get('ODOO_SESSION_REDIS')) + assert os.environ.get('ODOO_SESSION_REDIS_HOST') + assert os.environ.get('ODOO_SESSION_REDIS_PREFIX') + prefix = os.environ['ODOO_SESSION_REDIS_PREFIX'] + if environment_name in ('prod', 'integration', 'test'): + assert re.match(r'[a-z]+-odoo-%s' % (environment_name,), prefix), ( + "ODOO_SESSION_REDIS_PREFIX should match '-odoo-%s', " + "we got: '%s'" % (environment_name, prefix) + ) + + @api.model + def check(self): + if is_true(os.environ.get('ODOO_CLOUD_PLATFORM_UNSAFE')): + _logger.warning( + "cloud platform checks disabled, this is not safe" + ) + return + params = self.env['ir.config_parameter'].sudo() + kind = params.get_param('cloud.platform.kind') + if not kind: + _logger.warning( + "cloud platform not configured, you should " + "probably run 'env['cloud.platform'].install_exoscale()'" + ) + return + environment_name = config['running_env'] + self._check_s3(environment_name) + self._check_redis(environment_name) + + @api.cr + def _register_hook(self, cr): + env = api.Environment(cr, SUPERUSER_ID, {}) + env['cloud.platform'].check() diff --git a/cloud_platform/songs.py b/cloud_platform/songs.py new file mode 100644 index 0000000..ce5736b --- /dev/null +++ b/cloud_platform/songs.py @@ -0,0 +1,4 @@ +# -*- coding: utf-8 -*- + +def install_exoscale(ctx): + ctx.env['cloud.platform'].install_exoscale()