diff --git a/README.md b/README.md index bf11379..69525f5 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,15 @@ On the platform we want to achieve having: * Logs sent to ElasticSearch-Kibana structured as JSON for better searching For the storage, we store all the attachments on a object storage such as S3 or -a S3 compatible one, and we store the werkzeug sessions on Redis. + Swift, and we store the werkzeug sessions on Redis. + +Two providers are available for the Cloud Platform, Exoscale based in +Switzerland and OVH in France. + +The main difference between the two is the Object Store they use : + +* Exoscale uses S3 +* OVH uses Swift ## Setup @@ -21,7 +29,10 @@ a S3 compatible one, and we store the werkzeug sessions on Redis. Libraries that must be added in ``requirements.txt``: ``` +# For S3 object store boto==2.42.0 +# For Swift object store +python-swiftclient==3.0.0 redis==2.10.5 python-json-logger==0.1.5 statsd==3.2.1 @@ -31,7 +42,7 @@ statsd==3.2.1 The `--load` option of Odoo must contains the following addons: -* `attachment_s3` +* `attachment_s3` or `attachment_swift` depending of the provider used. * `session_redis` * `logging_json` @@ -51,7 +62,7 @@ The server environments in `server_environment_files` must be at least: The exact naming is important, because the `cloud_platform` addon rely on these keys to know and check the running environment. -### Attachments in the Object Storage +### Attachments in the Object Storage S3 * prod: stored RW in the object storage * `AWS_HOST`: depends of the platform @@ -69,6 +80,25 @@ Besides, the attachment location should be set to `s3` (but this is automatically done by the `install` methods of the `cloud_platform` module. * `ir.config_parameter` `ir_attachment.location`: `s3` + +### Attachments in the Object Storage Swift + +* prod: stored RW in the object storage + * `SWIFT_HOST`: depends of the platform + * `SWIFT_ACCOUNT`: depends of the platform + * `SWIFT_PASSWORD`: depends of the platform + * `SWIFT_WRITE_CONTAINER`: `-odoo-prod` +* integration: + * `SWIFT_HOST`: depends of the platform + * `SWIFT_ACCOUNT`: depends of the platform + * `SWIFT_PASSWORD`: depends of the platform + * `SWIFT_WRITE_CONTAINER`: `-odoo-integration` +* test: attachments are stored in database + +Besides, the attachment location should be set to `swift` (but this is +automatically done by the `install` methods of the `cloud_platform` module. + * `ir.config_parameter` `ir_attachment.location`: `swift` + ### Sessions in Redis * prod: @@ -108,8 +138,11 @@ Should be active at least on the production server ### Automatic Configuration -Calling `ctx.env['cloud.platform'].install_exoscale()` in an -`anthem` song will configure some parameters such as the +Calling + `ctx.env['cloud.platform'].install_exoscale()` +or + `ctx.env['cloud.platform'].install_ovh()` +in an `anthem` song will configure some parameters such as the `ir_attachment.location` and migrate the existing attachments to the object storage. diff --git a/cloud_platform/README.rst b/cloud_platform/README.rst index 5e1e75c..48e80ff 100644 --- a/cloud_platform/README.rst +++ b/cloud_platform/README.rst @@ -1,7 +1,8 @@ Cloud Platform ============== -Install addons required for the Camptocamp Cloud platform. +Install addons required for the Camptocamp Cloud platform, and that are +common to all platform providers. * Provide a quick install that we can call at the setup / migration of a database diff --git a/cloud_platform/__manifest__.py b/cloud_platform/__manifest__.py index 4df4e98..2e43c33 100644 --- a/cloud_platform/__manifest__.py +++ b/cloud_platform/__manifest__.py @@ -10,7 +10,6 @@ 'license': 'AGPL-3', 'category': 'Extra Tools', 'depends': [ - 'attachment_s3', 'session_redis', 'monitoring_status', 'logging_json', diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py index 5355512..1c38ffd 100644 --- a/cloud_platform/models/cloud_platform.py +++ b/cloud_platform/models/cloud_platform.py @@ -30,6 +30,7 @@ PlatformConfig = namedtuple( class FilestoreKind(object): db = 'db' s3 = 's3' # or compatible s3 object storage + swift = 'swift' file = 'file' @@ -46,6 +47,8 @@ class CloudPlatform(models.AbstractModel): } return configs.get(environment) or configs['dev'] + # Due to the addition of the ovh cloud platform + # This will be moved to cloud_platform_exoscale on v11 @api.model def install_exoscale(self): params = self.env['ir.config_parameter'].sudo() @@ -58,6 +61,54 @@ class CloudPlatform(models.AbstractModel): self.env['ir.attachment'].sudo().force_storage() _logger.info('cloud platform configured for exoscale') + @api.model + def _check_swift(self, environment_name): + params = self.env['ir.config_parameter'].sudo() + use_swift = (params.get_param('ir_attachment.location') == + FilestoreKind.swift) + if environment_name in ('prod', 'integration'): + assert use_swift, ( + "Swift must be used on production and integration instances. " + "It is activated, setting 'ir_attachment.location.' to 'swift'" + " The 'install_exoscale()' function sets this option " + "automatically." + ) + if use_swift: + assert os.environ.get('SWIFT_HOST'), ( + "SWIFT_HOST environment variable is required when " + "ir_attachment.location is 'swift'." + ) + assert os.environ.get('SWIFT_ACCOUNT'), ( + "SWIFT_ACCOUNT environment variable is required when " + "ir_attachment.location is 'swift'." + ) + assert os.environ.get('SWIFT_PASSWORD'), ( + "SWIFT_PASSWORD environment variable is required when " + "ir_attachment.location is 'swift'." + ) + container_name = os.environ['SWIFT_WRITE_CONTAINER'] + prod_container = bool(re.match(r'[a-z]+-odoo-prod', + container_name)) + if environment_name == 'prod': + assert prod_container, ( + "SWIFT_WRITE_CONTAINER should match '-odoo-prod', " + "we got: '%s'" % (container_name,) + ) + else: + # if we are using the prod bucket on another instance + # such as an integration, we must be sure to be in read only! + assert not prod_container, ( + "SWIFT_WRITE_CONTAINER should not match " + "'-odoo-prod', we got: '%s'" % (container_name,) + ) + elif environment_name == 'test': + # store in DB so we don't have files local to the host + assert params.get_param('ir_attachment.location') == 'db', ( + "In test instances, files must be stored in the database with " + "'ir_attachment.location' set to 'db'. This is " + "automatically set by the function 'install_ovh()'." + ) + @api.model def _check_s3(self, environment_name): params = self.env['ir.config_parameter'].sudo() @@ -148,7 +199,10 @@ class CloudPlatform(models.AbstractModel): ) return environment_name = config['running_env'] - self._check_s3(environment_name) + if kind == 'exoscale': + self._check_s3(environment_name) + elif kind == 'ovh': + self._check_swift(environment_name) self._check_redis(environment_name) @api.model_cr diff --git a/cloud_platform/songs.py b/cloud_platform/songs.py index d8ef9b5..ac62e7e 100644 --- a/cloud_platform/songs.py +++ b/cloud_platform/songs.py @@ -3,3 +3,7 @@ def install_exoscale(ctx): ctx.env['cloud.platform'].install_exoscale() + + +def install_ovh(ctx): + ctx.env['cloud.platform'].install_ovh() diff --git a/cloud_platform_exoscale/README.md b/cloud_platform_exoscale/README.md new file mode 100644 index 0000000..f2931ea --- /dev/null +++ b/cloud_platform_exoscale/README.md @@ -0,0 +1,6 @@ +Cloud Platform Exoscale +======================= + +Install addons specific to the Exoscale setup. + + * The object storage is S3 diff --git a/cloud_platform_exoscale/__init__.py b/cloud_platform_exoscale/__init__.py new file mode 100644 index 0000000..0650744 --- /dev/null +++ b/cloud_platform_exoscale/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/cloud_platform_exoscale/__manifest__.py b/cloud_platform_exoscale/__manifest__.py new file mode 100644 index 0000000..a74abfc --- /dev/null +++ b/cloud_platform_exoscale/__manifest__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{'name': 'Cloud Platform Exoscale', + 'summary': 'Addons required for the Camptocamp Cloud Platform on Exoscale', + 'version': '10.0.1.1.0', + 'author': 'Camptocamp,Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'Extra Tools', + 'depends': [ + 'cloud_platform', + 'attachment_s3', + ], + 'website': 'http://www.camptocamp.com', + 'data': [], + 'installable': True, + } diff --git a/cloud_platform_exoscale/models/__init__.py b/cloud_platform_exoscale/models/__init__.py new file mode 100644 index 0000000..5d08f36 --- /dev/null +++ b/cloud_platform_exoscale/models/__init__.py @@ -0,0 +1 @@ +from . import cloud_platform diff --git a/cloud_platform_exoscale/models/cloud_platform.py b/cloud_platform_exoscale/models/cloud_platform.py new file mode 100644 index 0000000..4e58852 --- /dev/null +++ b/cloud_platform_exoscale/models/cloud_platform.py @@ -0,0 +1,13 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import logging + +from odoo import models + +_logger = logging.getLogger(__name__) + + +class CloudPlatform(models.AbstractModel): + _inherit = 'cloud.platform' diff --git a/cloud_platform_ovh/README.md b/cloud_platform_ovh/README.md new file mode 100644 index 0000000..c350eba --- /dev/null +++ b/cloud_platform_ovh/README.md @@ -0,0 +1,7 @@ +Cloud Platform OVH +================== + +Install addons specific to the OVH setup. + + * The object storage is Swift + diff --git a/cloud_platform_ovh/__init__.py b/cloud_platform_ovh/__init__.py new file mode 100644 index 0000000..0650744 --- /dev/null +++ b/cloud_platform_ovh/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/cloud_platform_ovh/__manifest__.py b/cloud_platform_ovh/__manifest__.py new file mode 100644 index 0000000..6ea2be1 --- /dev/null +++ b/cloud_platform_ovh/__manifest__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{'name': 'Cloud Platform OVH', + 'summary': 'Addons required for the Camptocamp Cloud Platform on OVH', + 'version': '10.0.1.1.0', + 'author': 'Camptocamp,Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'Extra Tools', + 'depends': [ + 'cloud_platform', + 'attachment_swift', + ], + 'website': 'http://www.camptocamp.com', + 'data': [], + 'installable': True, + } diff --git a/cloud_platform_ovh/models/__init__.py b/cloud_platform_ovh/models/__init__.py new file mode 100644 index 0000000..5d08f36 --- /dev/null +++ b/cloud_platform_ovh/models/__init__.py @@ -0,0 +1 @@ +from . import cloud_platform diff --git a/cloud_platform_ovh/models/cloud_platform.py b/cloud_platform_ovh/models/cloud_platform.py new file mode 100644 index 0000000..9bbed5e --- /dev/null +++ b/cloud_platform_ovh/models/cloud_platform.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import logging + +from odoo import api, models +from odoo.tools.config import config + +_logger = logging.getLogger(__name__) + +try: + from ...cloud_platform.models.cloud_platform import FilestoreKind + from ...cloud_platform.models.cloud_platform import PlatformConfig +except ImportError: + FilestoreKind = None + PlatformConfig = None + _logger.debug("Cannot 'import from cloud_platform'") + + +class CloudPlatform(models.AbstractModel): + _inherit = 'cloud.platform' + + @api.model + def _config_by_server_env(self, environment): + configs = { + 'prod': PlatformConfig(filestore=FilestoreKind.swift), + 'integration': PlatformConfig(filestore=FilestoreKind.swift), + 'test': PlatformConfig(filestore=FilestoreKind.db), + 'dev': PlatformConfig(filestore=FilestoreKind.db), + } + return configs.get(environment) or configs['dev'] + + @api.model + def install_ovh(self): + params = self.env['ir.config_parameter'].sudo() + params.set_param('cloud.platform.kind', 'ovh') + environment = config['running_env'] + configs = self._config_by_server_env(environment) + params.set_param('ir_attachment.location', configs.filestore) + self.check() + if configs.filestore == FilestoreKind.swift: + self.env['ir.attachment'].sudo().force_storage() + _logger.info('cloud platform configured for ovh')