[13.0][IMP] Make cloud_platform fully abstract + update related modules (#244)

This commit is contained in:
Patrick Tombez
2021-11-03 20:23:22 +01:00
committed by Yannick Vaucher
co-authored by Yannick Vaucher
parent 35ad1a866e
commit 899f390f66
9 changed files with 257 additions and 230 deletions
+6 -2
View File
@@ -17,8 +17,12 @@ addons:
env:
matrix:
- LINT_CHECK="1"
- TESTS="1" ODOO_REPO="odoo/odoo"
- TESTS="1" ODOO_REPO="OCA/OCB"
- TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="cloud_platform_exoscale"
- TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="cloud_platform_exoscale"
- TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="cloud_platform_ovh"
- TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="cloud_platform_ovh"
- TESTS="1" ODOO_REPO="odoo/odoo" EXCLUDE="cloud_platform,cloud_platform_ovh,cloud_platform_exoscale"
- TESTS="1" ODOO_REPO="OCA/OCB" EXCLUDE="cloud_platform,cloud_platform_ovh,cloud_platform_exoscale"
global:
- VERSION="15.0" LINT_CHECK="0" TESTS="0"
+2 -2
View File
@@ -6,7 +6,7 @@ common to all platform providers.
* Provide a quick install that we can call at the setup / migration
of a database
* Check if the environment variables are configured correctly according
to the instance's environment (prod, integration, test or dev) to prevent
* Implements abstract methods to check if the environment variables are configured correctly
according to the instance's environment (prod, integration, test or dev) to prevent
data corruption between the environments (such as the integration server
writing on the production object storage).
+11 -13
View File
@@ -2,19 +2,17 @@
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
{"name": "Cloud Platform",
"summary": "Addons required for the Camptocamp Cloud Platform",
"version": "15.0.1.0.0",
"author": "Camptocamp,Odoo Community Association (OCA)",
"license": "AGPL-3",
"category": "Extra Tools",
"depends": [
"session_redis",
"monitoring_status",
"logging_json",
# "monitoring_log_requests",
"monitoring_statsd",
"server_environment", # OCA/server-tools
{'name': 'Cloud Platform',
'summary': 'Addons required for the Camptocamp Cloud Platform',
'version': "15.0.1.0.0",
'author': 'Camptocamp,Odoo Community Association (OCA)',
'license': 'AGPL-3',
'category': 'Extra Tools',
'depends': [
'session_redis',
'monitoring_status',
'logging_json',
'server_environment', # OCA/server-tools
],
"website": "https://www.camptocamp.com",
"data": [],
+25 -147
View File
@@ -6,7 +6,6 @@ import os
import re
from collections import namedtuple
from distutils.util import strtobool
from odoo import api, models
@@ -26,20 +25,27 @@ PlatformConfig = namedtuple(
)
class FilestoreKind(object):
db = 'db'
s3 = 's3' # or compatible s3 object storage
swift = 'swift'
file = 'file'
DefaultConfig = PlatformConfig(filestore=FilestoreKind.db)
FilestoreKind = namedtuple(
'FilestoreKind',
['name', 'location']
)
class CloudPlatform(models.AbstractModel):
_name = 'cloud.platform'
_description = 'cloud.platform'
@api.model
def _default_config(self):
return PlatformConfig(self._filestore_kinds()['db'])
@api.model
def _filestore_kinds(self):
return {
'db': FilestoreKind('db', 'local'),
'file': FilestoreKind('file', 'local'),
}
@api.model
def _platform_kinds(self):
return []
@@ -52,7 +58,7 @@ class CloudPlatform(models.AbstractModel):
None
)
configs = configs_getter() if configs_getter else {}
return configs.get(environment) or DefaultConfig
return configs.get(environment) or self._default_config()
def _get_running_env(self):
environment_name = config['running_env']
@@ -63,150 +69,25 @@ class CloudPlatform(models.AbstractModel):
return environment_name
@api.model
def install(self, platform_kind):
def _install(self, platform_kind):
assert platform_kind in self._platform_kinds()
params = self.env['ir.config_parameter'].sudo()
params.set_param('cloud.platform.kind', platform_kind)
environment_name = self._get_running_env()
configs = self._config_by_server_env(platform_kind, environment_name)
params.set_param('ir_attachment.location', configs.filestore)
params.set_param('ir_attachment.location', configs.filestore.name)
self.check()
if configs.filestore in [FilestoreKind.swift, FilestoreKind.s3]:
if configs.filestore.location == 'remote':
self.env['ir.attachment'].sudo().force_storage()
_logger.info('cloud platform configured for {}'.format(platform_kind))
@api.model
def _check_swift(self, environment_name):
params = self.env['ir.config_parameter'].sudo()
use_swift = (params.get_param('ir_attachment.location') ==
FilestoreKind.swift)
if environment_name in ('prod', 'integration'):
# Labs instances use swift or s3 by default, but we don't want
# to enforce it in case we want to test something with a different
# storage. At your own risks!
assert use_swift, (
"Swift must be used on production and integration instances. "
"It is activated, setting 'ir_attachment.location.' to 'swift'"
" The 'install_exoscale()' function sets this option "
"automatically."
)
if use_swift:
assert os.environ.get('SWIFT_AUTH_URL'), (
"SWIFT_AUTH_URL environment variable is required when "
"ir_attachment.location is 'swift'."
)
assert os.environ.get('SWIFT_ACCOUNT'), (
"SWIFT_ACCOUNT environment variable is required when "
"ir_attachment.location is 'swift'."
)
assert os.environ.get('SWIFT_PASSWORD'), (
"SWIFT_PASSWORD environment variable is required when "
"ir_attachment.location is 'swift'."
)
container_name = os.environ.get('SWIFT_WRITE_CONTAINER', '')
if environment_name in ('prod', 'integration', 'labs'):
assert container_name, (
"SWIFT_WRITE_CONTAINER environment variable is required when "
"ir_attachment.location is 'swift'.\n"
"Normally, 'swift' is activated on labs, integration "
"and production, but should not be used in dev environment"
" (or using a dedicated dev bucket, never using the "
"integration/prod bucket).\n"
"If you don't actually need a bucket, change the"
" 'ir_attachment.location' parameter."
)
prod_container = bool(re.match(r'[a-z0-9-]+-odoo-prod',
container_name))
# A bucket name is defined under the following format
# <client>-odoo-<env>
#
# Use SWIFT_WRITE_CONTAINER_UNSTRUCTURED to by-pass check on bucket name
# structure
if os.environ.get('SWIFT_WRITE_CONTAINER_UNSTRUCTURED'):
return
if environment_name == 'prod':
assert prod_container, (
"SWIFT_WRITE_CONTAINER should match '<client>-odoo-prod', "
"we got: '%s'" % (container_name,)
)
else:
# if we are using the prod bucket on another instance
# such as an integration, we must be sure to be in read only!
assert not prod_container, (
"SWIFT_WRITE_CONTAINER should not match "
"'<client>-odoo-prod', we got: '%s'" % (container_name,)
)
elif environment_name == 'test':
# store in DB so we don't have files local to the host
assert params.get_param('ir_attachment.location') == 'db', (
"In test instances, files must be stored in the database with "
"'ir_attachment.location' set to 'db'. This is "
"automatically set by the function 'install_ovh()'."
)
def install(self):
raise NotImplementedError
@api.model
def _check_s3(self, environment_name):
params = self.env['ir.config_parameter'].sudo()
use_s3 = params.get_param('ir_attachment.location') == FilestoreKind.s3
if environment_name in ('prod', 'integration'):
# Labs instances use swift or s3 by default, but we don't want
# to enforce it in case we want to test something with a different
# storage. At your own risks!
assert use_s3, (
"S3 must be used on production and integration instances. "
"It is activated by setting 'ir_attachment.location.' to 's3'."
" The 'install_exoscale()' function sets this option "
"automatically."
)
if use_s3:
assert os.environ.get('AWS_ACCESS_KEY_ID'), (
"AWS_ACCESS_KEY_ID environment variable is required when "
"ir_attachment.location is 's3'."
)
assert os.environ.get('AWS_SECRET_ACCESS_KEY'), (
"AWS_SECRET_ACCESS_KEY environment variable is required when "
"ir_attachment.location is 's3'."
)
bucket_name = os.environ.get('AWS_BUCKETNAME', '')
if environment_name in ('prod', 'integration', 'labs'):
assert bucket_name, (
"AWS_BUCKETNAME environment variable is required when "
"ir_attachment.location is 's3'.\n"
"Normally, 's3' is activated on labs, integration "
"and production, but should not be used in dev environment"
" (or using a dedicated dev bucket, never using the "
"integration/prod bucket).\n"
"If you don't actually need a bucket, change the"
" 'ir_attachment.location' parameter."
)
# A bucket name is defined under the following format
# <client>-odoo-<env>
#
# Use AWS_BUCKETNAME_UNSTRUCTURED to by-pass check on bucket name
# structure
if os.environ.get('AWS_BUCKETNAME_UNSTRUCTURED'):
return
prod_bucket = bool(re.match(r'[a-z-0-9]+-odoo-prod', bucket_name))
if environment_name == 'prod':
assert prod_bucket, (
"AWS_BUCKETNAME should match '<client>-odoo-prod', "
"we got: '%s'" % (bucket_name,)
)
else:
# if we are using the prod bucket on another instance
# such as an integration, we must be sure to be in read only!
assert not prod_bucket, (
"AWS_BUCKETNAME should not match '<client>-odoo-prod', "
"we got: '%s'" % (bucket_name,)
)
elif environment_name == 'test':
# store in DB so we don't have files local to the host
assert params.get_param('ir_attachment.location') == 'db', (
"In test instances, files must be stored in the database with "
"'ir_attachment.location' set to 'db'. This is "
"automatically set by the function 'install_exoscale()'."
)
def _check_filestore(self, environment_name):
raise NotImplementedError
@api.model
def _check_redis(self, environment_name):
@@ -243,14 +124,11 @@ class CloudPlatform(models.AbstractModel):
if not kind:
_logger.warning(
"cloud platform not configured, you should "
"probably run 'env['cloud.platform'].install_exoscale()'"
"probably run 'env['cloud.platform'].install()'"
)
return
environment_name = self._get_running_env()
if kind == 'exoscale':
self._check_s3(environment_name)
elif kind == 'ovh':
self._check_swift(environment_name)
self._check_filestore(environment_name)
self._check_redis(environment_name)
def _register_hook(self):
+2 -6
View File
@@ -1,7 +1,3 @@
def install_exoscale(ctx):
ctx.env['cloud.platform'].install('exoscale')
def install_ovh(ctx):
ctx.env['cloud.platform'].install('ovh')
def install(ctx):
ctx.env['cloud.platform'].install()
+7 -2
View File
@@ -2,7 +2,8 @@
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
{"name": "Cloud Platform Exoscale",
{
"name": "Cloud Platform Exoscale",
"summary": "Addons required for the Camptocamp Cloud Platform on Exoscale",
"version": "15.0.1.0.0",
"author": "Camptocamp,Odoo Community Association (OCA)",
@@ -11,8 +12,12 @@
"depends": [
"cloud_platform",
"attachment_s3",
"monitoring_statsd",
],
"excludes": [
"cloud_platform_ovh",
],
"website": "https://www.camptocamp.com",
"data": [],
"installable": True,
}
}
@@ -1,24 +1,26 @@
# Copyright 2016-2019 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
import logging
import re
import os
from odoo import models, api
from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind
from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig
_logger = logging.getLogger(__name__)
try:
from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind
from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig
except ImportError:
FilestoreKind = None
PlatformConfig = None
_logger.debug("Cannot 'import from cloud_platform'")
S3_STORE_KIND = FilestoreKind('s3', 'remote')
class CloudPlatform(models.AbstractModel):
_inherit = 'cloud.platform'
@api.model
def _filestore_kinds(self):
kinds = super(CloudPlatform, self)._filestore_kinds()
kinds['s3'] = S3_STORE_KIND
return kinds
@api.model
def _platform_kinds(self):
kinds = super(CloudPlatform, self)._platform_kinds()
@@ -27,15 +29,81 @@ class CloudPlatform(models.AbstractModel):
@api.model
def _config_by_server_env_for_exoscale(self):
fs_kinds = self._filestore_kinds()
configs = {
'prod': PlatformConfig(filestore=FilestoreKind.s3),
'integration': PlatformConfig(filestore=FilestoreKind.s3),
'labs': PlatformConfig(filestore=FilestoreKind.s3),
'test': PlatformConfig(filestore=FilestoreKind.db),
'dev': PlatformConfig(filestore=FilestoreKind.db),
'prod': PlatformConfig(filestore=fs_kinds['s3']),
'integration': PlatformConfig(filestore=fs_kinds['s3']),
'labs': PlatformConfig(filestore=fs_kinds['s3']),
'test': PlatformConfig(filestore=fs_kinds['db']),
'dev': PlatformConfig(filestore=fs_kinds['db']),
}
return configs
@api.model
def install_exoscale(self):
self.install('exoscale')
def _check_filestore(self, environment_name):
params = self.env['ir.config_parameter'].sudo()
use_s3 = (params.get_param('ir_attachment.location') ==
S3_STORE_KIND.name)
if environment_name in ('prod', 'integration'):
# Labs instances use s3 by default, but we don't want
# to enforce it in case we want to test something with a different
# storage. At your own risks!
assert use_s3, (
"S3 must be used on production and integration instances. "
"It is activated by setting 'ir_attachment.location.' to 's3'."
" The 'install()' function sets this option "
"automatically."
)
if use_s3:
assert os.environ.get('AWS_ACCESS_KEY_ID'), (
"AWS_ACCESS_KEY_ID environment variable is required when "
"ir_attachment.location is 's3'."
)
assert os.environ.get('AWS_SECRET_ACCESS_KEY'), (
"AWS_SECRET_ACCESS_KEY environment variable is required when "
"ir_attachment.location is 's3'."
)
bucket_name = os.environ.get('AWS_BUCKETNAME', '')
if environment_name in ('prod', 'integration', 'labs'):
assert bucket_name, (
"AWS_BUCKETNAME environment variable is required when "
"ir_attachment.location is 's3'.\n"
"Normally, 's3' is activated on labs, integration "
"and production, but should not be used in dev environment"
" (or using a dedicated dev bucket, never using the "
"integration/prod bucket).\n"
"If you don't actually need a bucket, change the"
" 'ir_attachment.location' parameter."
)
# A bucket name is defined under the following format
# <client>-odoo-<env>
#
# Use AWS_BUCKETNAME_UNSTRUCTURED to by-pass check on bucket name
# structure
if os.environ.get('AWS_BUCKETNAME_UNSTRUCTURED'):
return
prod_bucket = bool(re.match(r'[a-z-0-9]+-odoo-prod', bucket_name))
if environment_name == 'prod':
assert prod_bucket, (
"AWS_BUCKETNAME should match '<client>-odoo-prod', "
"we got: '%s'" % (bucket_name,)
)
else:
# if we are using the prod bucket on another instance
# such as an integration, we must be sure to be in read only!
assert not prod_bucket, (
"AWS_BUCKETNAME should not match '<client>-odoo-prod', "
"we got: '%s'" % (bucket_name,)
)
elif environment_name == 'test':
# store in DB so we don't have files local to the host
assert params.get_param('ir_attachment.location') == 'db', (
"In test instances, files must be stored in the database with "
"'ir_attachment.location' set to 'db'. This is "
"automatically set by the function 'install()'."
)
@api.model
def install(self):
self._install('exoscale')
+7 -2
View File
@@ -2,7 +2,8 @@
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
{"name": "Cloud Platform OVH",
{
"name": "Cloud Platform OVH",
"summary": "Addons required for the Camptocamp Cloud Platform on OVH",
"version": "15.0.1.0.0",
"author": "Camptocamp,Odoo Community Association (OCA)",
@@ -11,8 +12,12 @@
"depends": [
"cloud_platform",
"attachment_swift",
"monitoring_statsd",
],
"excludes": [
"cloud_platform_exoscale",
],
"website": "https://www.camptocamp.com",
"data": [],
"installable": True,
}
}
+89 -16
View File
@@ -1,24 +1,27 @@
# Copyright 2017-2019 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
import logging
import re
import os
from odoo import api, models
_logger = logging.getLogger(__name__)
from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind
from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig
try:
from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind
from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig
except ImportError:
FilestoreKind = None
PlatformConfig = None
_logger.debug("Cannot 'import from cloud_platform'")
SWIFT_STORE_KIND = FilestoreKind('s3', 'remote')
class CloudPlatform(models.AbstractModel):
_inherit = 'cloud.platform'
@api.model
def _filestore_kinds(self):
kinds = super(CloudPlatform, self)._filestore_kinds()
kinds['swift'] = SWIFT_STORE_KIND
return kinds
@api.model
def _platform_kinds(self):
kinds = super()._platform_kinds()
@@ -27,15 +30,85 @@ class CloudPlatform(models.AbstractModel):
@api.model
def _config_by_server_env_for_ovh(self):
fs_kinds = self._filestore_kinds()
configs = {
'prod': PlatformConfig(filestore=FilestoreKind.swift),
'integration': PlatformConfig(filestore=FilestoreKind.swift),
'labs': PlatformConfig(filestore=FilestoreKind.swift),
'test': PlatformConfig(filestore=FilestoreKind.db),
'dev': PlatformConfig(filestore=FilestoreKind.db),
'prod': PlatformConfig(filestore=fs_kinds['swift']),
'integration': PlatformConfig(filestore=fs_kinds['swift']),
'labs': PlatformConfig(filestore=fs_kinds['swift']),
'test': PlatformConfig(filestore=fs_kinds['db']),
'dev': PlatformConfig(filestore=fs_kinds['db']),
}
return configs
@api.model
def install_ovh(self):
self.install('ovh')
def _check_filestore(self, environment_name):
params = self.env['ir.config_parameter'].sudo()
use_swift = (params.get_param('ir_attachment.location') ==
SWIFT_STORE_KIND.name)
if environment_name in ('prod', 'integration'):
# Labs instances use swift by default, but we don't want
# to enforce it in case we want to test something with a different
# storage. At your own risks!
assert use_swift, (
"Swift must be used on production and integration instances. "
"It is activated, setting 'ir_attachment.location.' to 'swift'"
" The 'install()' function sets this option "
"automatically."
)
if use_swift:
assert os.environ.get('SWIFT_AUTH_URL'), (
"SWIFT_AUTH_URL environment variable is required when "
"ir_attachment.location is 'swift'."
)
assert os.environ.get('SWIFT_ACCOUNT'), (
"SWIFT_ACCOUNT environment variable is required when "
"ir_attachment.location is 'swift'."
)
assert os.environ.get('SWIFT_PASSWORD'), (
"SWIFT_PASSWORD environment variable is required when "
"ir_attachment.location is 'swift'."
)
container_name = os.environ.get('SWIFT_WRITE_CONTAINER', '')
if environment_name in ('prod', 'integration', 'labs'):
assert container_name, (
"SWIFT_WRITE_CONTAINER environment variable is required when "
"ir_attachment.location is 'swift'.\n"
"Normally, 'swift' is activated on labs, integration "
"and production, but should not be used in dev environment"
" (or using a dedicated dev bucket, never using the "
"integration/prod bucket).\n"
"If you don't actually need a bucket, change the"
" 'ir_attachment.location' parameter."
)
prod_container = bool(re.match(r'[a-z0-9-]+-odoo-prod',
container_name))
# A bucket name is defined under the following format
# <client>-odoo-<env>
#
# Use SWIFT_WRITE_CONTAINER_UNSTRUCTURED to by-pass check on bucket name
# structure
if os.environ.get('SWIFT_WRITE_CONTAINER_UNSTRUCTURED'):
return
if environment_name == 'prod':
assert prod_container, (
"SWIFT_WRITE_CONTAINER should match '<client>-odoo-prod', "
"we got: '%s'" % (container_name,)
)
else:
# if we are using the prod bucket on another instance
# such as an integration, we must be sure to be in read only!
assert not prod_container, (
"SWIFT_WRITE_CONTAINER should not match "
"'<client>-odoo-prod', we got: '%s'" % (container_name,)
)
elif environment_name == 'test':
# store in DB so we don't have files local to the host
assert params.get_param('ir_attachment.location') == 'db', (
"In test instances, files must be stored in the database with "
"'ir_attachment.location' set to 'db'. This is "
"automatically set by the function 'install()'."
)
@api.model
def install(self):
self._install('ovh')