From 09459eed561d98b66eba1a1b73a1e8ce5722dba0 Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Mon, 6 May 2019 13:55:46 +0200 Subject: [PATCH 01/12] Add session_redis --- requirements.txt | 1 + session_redis/README.rst | 37 +++++++++++++ session_redis/__init__.py | 5 ++ session_redis/__openerp__.py | 19 +++++++ session_redis/http.py | 90 ++++++++++++++++++++++++++++++++ session_redis/models/__init__.py | 1 + session_redis/models/user.py | 19 +++++++ session_redis/session.py | 86 ++++++++++++++++++++++++++++++ 8 files changed, 258 insertions(+) create mode 100644 requirements.txt create mode 100644 session_redis/README.rst create mode 100644 session_redis/__init__.py create mode 100644 session_redis/__openerp__.py create mode 100644 session_redis/http.py create mode 100644 session_redis/models/__init__.py create mode 100644 session_redis/models/user.py create mode 100644 session_redis/session.py diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..2318a9f --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +redis==2.10.5 diff --git a/session_redis/README.rst b/session_redis/README.rst new file mode 100644 index 0000000..8a05b14 --- /dev/null +++ b/session_redis/README.rst @@ -0,0 +1,37 @@ +Sessions in Redis +================= + +This addon allows to store the web sessions in Redis. + +Configuration +------------- + +The storage of sessions in Redis is activated using environment variables. + +* ``ODOO_SESSION_REDIS`` has to be ``1`` or ``true`` +* ``ODOO_SESSION_REDIS_HOST`` is the redis hostname (default is ``localhost``) +* ``ODOO_SESSION_REDIS_PORT`` is the redis port (default is ``6379``) +* ``ODOO_SESSION_REDIS_PASSWORD`` is the password for the AUTH command + (optional) +* ``ODOO_SESSION_REDIS_PREFIX`` is the prefix for the session keys (optional) +* ``ODOO_SESSION_REDIS_EXPIRATION`` is the time in seconds before expiration of + the sessions (default is 7 days) + + +The keys are set to ``session:``. +When a prefix is defined, the keys are ``session::`` + +This addon must be added in the server wide addons with (``--load`` option): + +``--load=web,web_kanban,session_redis`` + +Limitations +----------- + +* The server has to be restarted in order for the sessions to be stored in + Redis. +* All the users will have to login again as their previous session will be + dropped. +* The addon monkey-patch ``openerp.http.Root.session_store`` with a custom + method when the Redis mode is active, so incompatibilities with other addons + is possible if they do the same. diff --git a/session_redis/__init__.py b/session_redis/__init__.py new file mode 100644 index 0000000..d3b5bfe --- /dev/null +++ b/session_redis/__init__.py @@ -0,0 +1,5 @@ +# -*- coding: utf-8 -*- + +from . import http +from . import session +from . import models diff --git a/session_redis/__openerp__.py b/session_redis/__openerp__.py new file mode 100644 index 0000000..eb040dc --- /dev/null +++ b/session_redis/__openerp__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{'name': 'Sessions in Redis', + 'summary': 'Store web sessions in Redis', + 'version': '8.0.1.0.0', + 'author': 'Camptocamp,Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'Extra Tools', + 'depends': ['base'], + 'external_dependencies': { + 'python': ['redis'], + }, + 'website': 'http://www.camptocamp.com', + 'data': [], + 'installable': True, + } diff --git a/session_redis/http.py b/session_redis/http.py new file mode 100644 index 0000000..912fb92 --- /dev/null +++ b/session_redis/http.py @@ -0,0 +1,90 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import logging +import os + +from distutils.util import strtobool + +import openerp +from openerp import http +from openerp.tools.func import lazy_property + +from .session import RedisSessionStore + +_logger = logging.getLogger(__name__) + +try: + import redis + from redis.sentinel import Sentinel +except ImportError: + redis = None # noqa + _logger.debug("Cannot 'import redis'.") + + +def is_true(strval): + return bool(strtobool(strval or '0'.lower())) + + +sentinel_host = os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST') +sentinel_master_name = os.environ.get( + 'ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME' +) +if sentinel_host and not sentinel_master_name: + raise Exception( + "ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME must be defined " + "when using session_redis" + ) +sentinel_port = int(os.environ.get('ODOO_SESSION_REDIS_SENTINEL_PORT', 26379)) +host = os.environ.get('ODOO_SESSION_REDIS_HOST', 'localhost') +port = int(os.environ.get('ODOO_SESSION_REDIS_PORT', 6379)) +prefix = os.environ.get('ODOO_SESSION_REDIS_PREFIX') +password = os.environ.get('ODOO_SESSION_REDIS_PASSWORD') +expiration = os.environ.get('ODOO_SESSION_REDIS_EXPIRATION') + + +@lazy_property +def session_store(self): + if sentinel_host: + sentinel = Sentinel([(sentinel_host, sentinel_port)], + password=password) + redis_client = sentinel.master_for(sentinel_master_name) + else: + redis_client = redis.Redis(host=host, port=port, password=password) + return RedisSessionStore(redis=redis_client, prefix=prefix, + expiration=expiration, + session_class=http.OpenERPSession) + + +def session_gc(session_store): + """ Do not garbage collect the sessions + + Redis keys are automatically cleaned at the end of their + expiration. + """ + return + + +def purge_fs_sessions(path): + for fname in os.listdir(path): + path = os.path.join(path, fname) + try: + os.unlink(path) + except OSError: + pass + + +if is_true(os.environ.get('ODOO_SESSION_REDIS')): + if sentinel_host: + _logger.debug("HTTP sessions stored in Redis with prefix '%s'. " + "Using Sentinel on %s:%s", + sentinel_host, sentinel_port, prefix or '') + else: + _logger.debug("HTTP sessions stored in Redis with prefix '%s' on " + "%s:%s", host, port, prefix or '') + + http.Root.session_store = session_store + http.session_gc = session_gc + # clean the existing sessions on the file system + purge_fs_sessions(openerp.tools.config.session_dir) diff --git a/session_redis/models/__init__.py b/session_redis/models/__init__.py new file mode 100644 index 0000000..f9b61db --- /dev/null +++ b/session_redis/models/__init__.py @@ -0,0 +1 @@ +from . import user diff --git a/session_redis/models/user.py b/session_redis/models/user.py new file mode 100644 index 0000000..e652fd7 --- /dev/null +++ b/session_redis/models/user.py @@ -0,0 +1,19 @@ +from openerp import models, tools + + +class User(models.Model): + _inherit = 'res.users' + + @tools.ormcache('sid') + def _compute_session_token(self, sid): + """Make sure to return an unicode string. + + Odoo creates a session token using hexdigest Session which is str + but with redis we set the token from a dictionary of values passing + it in json format. When dumping values from json, we always get unicode + thus both are incompatible. + + The shortest path is to fix the output of the computed session by Odoo. + + """ + return unicode(super(User, self)._compute_session_token(sid)) diff --git a/session_redis/session.py b/session_redis/session.py new file mode 100644 index 0000000..dce8189 --- /dev/null +++ b/session_redis/session.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import json +import logging + +from werkzeug.contrib.sessions import SessionStore + +# this is equal to the duration of the session garbage collector in +# openerp.http.session_gc() +DEFAULT_SESSION_TIMEOUT = 60 * 60 * 24 * 7 # 7 days in seconds + +_logger = logging.getLogger(__name__) + + +class RedisSessionStore(SessionStore): + """ SessionStore that saves session to redis """ + + def __init__(self, redis, session_class=None, + prefix='', expiration=None): + super(RedisSessionStore, self).__init__(session_class=session_class) + self.redis = redis + if expiration is None: + self.expiration = DEFAULT_SESSION_TIMEOUT + else: + self.expiration = expiration + self.prefix = u'session:' + if prefix: + self.prefix = u'%s:%s:' % ( + self.prefix, prefix + ) + + def build_key(self, sid): + if isinstance(sid, unicode): + sid = sid.encode('utf-8') + return '%s%s' % (self.prefix, sid) + + def save(self, session): + key = self.build_key(session.sid) + + # allow to set a custom expiration for a session + # such as a very short one for monitoring requests + expiration = session.expiration or self.expiration + if _logger.isEnabledFor(logging.DEBUG): + if session.uid: + user_msg = "user '%s' (id: %s)" % ( + session.login, session.uid) + else: + user_msg = "anonymous user" + _logger.debug("saving session with key '%s' and " + "expiration of %s seconds for %s", + key, expiration, user_msg) + + if self.redis.set(key, json.dumps(dict(session))): + return self.redis.expire(key, expiration) + + def delete(self, session): + key = self.build_key(session.sid) + _logger.debug('deleting session with key %s', key) + return self.redis.delete(key) + + def get(self, sid): + if not self.is_valid_key(sid): + _logger.debug("session with invalid sid '%s' has been asked, " + "returning a new one", sid) + return self.new() + + key = self.build_key(sid) + saved = self.redis.get(key) + if not saved: + _logger.debug("session with non-existent key '%s' has been asked, " + "returning a new one", key) + return self.new() + try: + data = json.loads(saved) + except ValueError: + _logger.debug("session for key '%s' has been asked but its json " + "content could not be read, it has been reset", key) + data = {} + return self.session_class(data, sid, False) + + def list(self): + keys = self.redis.keys('%s*' % self.prefix) + _logger.debug("a listing redis keys has been called") + return [key[len(self.prefix):] for key in keys] From 4dfc54d4c28c3d8f320641fb2c05ea43afacf820 Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Mon, 6 May 2019 14:00:07 +0200 Subject: [PATCH 02/12] Add logging_json --- logging_json/README.rst | 14 ++++++++++++++ logging_json/__init__.py | 3 +++ logging_json/__openerp__.py | 18 ++++++++++++++++++ logging_json/json_log.py | 35 +++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 5 files changed, 71 insertions(+) create mode 100644 logging_json/README.rst create mode 100644 logging_json/__init__.py create mode 100644 logging_json/__openerp__.py create mode 100644 logging_json/json_log.py diff --git a/logging_json/README.rst b/logging_json/README.rst new file mode 100644 index 0000000..4d60884 --- /dev/null +++ b/logging_json/README.rst @@ -0,0 +1,14 @@ +JSON Logging +============ + +This addon allows to output the Odoo logs in JSON. + +Configuration +------------- + +The json logging is activated with the environment variable +``ODOO_LOGGING_JSON`` set to ``1``. + +In order to have the logs from the start of the server, you should add +``logging_json`` in the ``--load`` flag or in the ``server_wide_modules`` +option in the configuration file. diff --git a/logging_json/__init__.py b/logging_json/__init__.py new file mode 100644 index 0000000..a50f468 --- /dev/null +++ b/logging_json/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from . import json_log diff --git a/logging_json/__openerp__.py b/logging_json/__openerp__.py new file mode 100644 index 0000000..5ad22c5 --- /dev/null +++ b/logging_json/__openerp__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +{'name': 'JSON Logging', + 'version': '8.0.1.0.0', + 'author': 'Camptocamp,Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'Extra Tools', + 'depends': ['base', + ], + 'external_dependencies': { + 'python': ['pythonjsonlogger'], + }, + 'website': 'http://www.camptocamp.com', + 'data': [], + 'installable': True, + } diff --git a/logging_json/json_log.py b/logging_json/json_log.py new file mode 100644 index 0000000..3fcc796 --- /dev/null +++ b/logging_json/json_log.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + +import logging +import os +import threading + +from distutils.util import strtobool + +_logger = logging.getLogger(__name__) + +try: + from pythonjsonlogger import jsonlogger +except ImportError: + jsonlogger = None # noqa + _logger.debug("Cannot 'import pythonjsonlogger'.") + + +def is_true(strval): + return bool(strtobool(strval or '0'.lower())) + + +class OdooJsonFormatter(jsonlogger.JsonFormatter): + + def add_fields(self, log_record, record, message_dict): + record.pid = os.getpid() + record.dbname = getattr(threading.currentThread(), 'dbname', '?') + _super = super(OdooJsonFormatter, self) + return _super.add_fields(log_record, record, message_dict) + + +if is_true(os.environ.get('ODOO_LOGGING_JSON')): + format = ('%(asctime)s %(pid)s %(levelname)s' + '%(dbname)s %(name)s: %(message)s') + formatter = OdooJsonFormatter(format) + logging.getLogger().handlers[0].formatter = formatter diff --git a/requirements.txt b/requirements.txt index 2318a9f..204d7d9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ redis==2.10.5 +python-json-logger==0.1.5 From 5492ca35514b2f412f2cb0ca7771b127825e1b9a Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Mon, 6 May 2019 14:26:07 +0200 Subject: [PATCH 03/12] Add monitoring_status --- monitoring_status/README.rst | 9 +++++++ monitoring_status/__init__.py | 2 ++ monitoring_status/__openerp__.py | 15 +++++++++++ monitoring_status/controllers/__init__.py | 1 + monitoring_status/controllers/main.py | 31 +++++++++++++++++++++++ 5 files changed, 58 insertions(+) create mode 100644 monitoring_status/README.rst create mode 100644 monitoring_status/__init__.py create mode 100644 monitoring_status/__openerp__.py create mode 100644 monitoring_status/controllers/__init__.py create mode 100644 monitoring_status/controllers/main.py diff --git a/monitoring_status/README.rst b/monitoring_status/README.rst new file mode 100644 index 0000000..b939c7a --- /dev/null +++ b/monitoring_status/README.rst @@ -0,0 +1,9 @@ +.. image:: https://img.shields.io/badge/licence-AGPL--3-blue.svg + :alt: License + +Monitoring: Status +================== + +Provides a HTTP route that returns health status of the instance. + +The url to call is ``http://server/monitoring/status`` diff --git a/monitoring_status/__init__.py b/monitoring_status/__init__.py new file mode 100644 index 0000000..153a9e3 --- /dev/null +++ b/monitoring_status/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +from . import controllers diff --git a/monitoring_status/__openerp__.py b/monitoring_status/__openerp__.py new file mode 100644 index 0000000..392abb6 --- /dev/null +++ b/monitoring_status/__openerp__.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{'name': 'Monitoring: Status', + 'version': '8.0.1.0.0', + 'author': 'Camptocamp,Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'category', + 'depends': ['base', 'web'], + 'website': 'http://www.camptocamp.com', + 'data': [], + 'installable': True, + } diff --git a/monitoring_status/controllers/__init__.py b/monitoring_status/controllers/__init__.py new file mode 100644 index 0000000..12a7e52 --- /dev/null +++ b/monitoring_status/controllers/__init__.py @@ -0,0 +1 @@ +from . import main diff --git a/monitoring_status/controllers/main.py b/monitoring_status/controllers/main.py new file mode 100644 index 0000000..da66f95 --- /dev/null +++ b/monitoring_status/controllers/main.py @@ -0,0 +1,31 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import json + +import werkzeug + +from openerp import http +from openerp.addons.web.controllers.main import ensure_db + + +class Monitoring(http.Controller): + + @http.route('/monitoring/status', type='http', auth='none') + def status(self): + ensure_db() + # TODO: add 'sub-systems' status and infos: + # queue job, cron, database, ... + headers = {'Content-Type': 'application/json'} + info = {'status': 1} + session = http.request.session + # We set a custom expiration of 1 second for this request, as we do a + # lot of health checks, we don't want those anonymous sessions to be + # kept. Beware, it works only when session_redis is used. + # Alternatively, we could set 'session.should_save = False', which is + # tested in odoo source code, but we wouldn't check the health of + # Redis. + if not session.uid: + session.expiration = 1 + return werkzeug.wrappers.Response(json.dumps(info), headers=headers) From bf344613f20234dc607543ca24ab8f951a24d611 Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Tue, 7 May 2019 16:42:17 +0200 Subject: [PATCH 04/12] Add base_attachment_object_storage --- base_attachment_object_storage/README.rst | 7 + base_attachment_object_storage/__init__.py | 1 + base_attachment_object_storage/__openerp__.py | 18 ++ .../models/__init__.py | 1 + .../models/ir_attachment.py | 270 ++++++++++++++++++ 5 files changed, 297 insertions(+) create mode 100644 base_attachment_object_storage/README.rst create mode 100644 base_attachment_object_storage/__init__.py create mode 100644 base_attachment_object_storage/__openerp__.py create mode 100644 base_attachment_object_storage/models/__init__.py create mode 100644 base_attachment_object_storage/models/ir_attachment.py diff --git a/base_attachment_object_storage/README.rst b/base_attachment_object_storage/README.rst new file mode 100644 index 0000000..c802faf --- /dev/null +++ b/base_attachment_object_storage/README.rst @@ -0,0 +1,7 @@ +Base class for attachments on external object store +=================================================== + +This is a base addon that regroup common code used by addons targeting specific object store + + + diff --git a/base_attachment_object_storage/__init__.py b/base_attachment_object_storage/__init__.py new file mode 100644 index 0000000..0650744 --- /dev/null +++ b/base_attachment_object_storage/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/base_attachment_object_storage/__openerp__.py b/base_attachment_object_storage/__openerp__.py new file mode 100644 index 0000000..d081f8d --- /dev/null +++ b/base_attachment_object_storage/__openerp__.py @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{ + 'name': 'Base Attachment Object Store', + 'summary': 'Base module for the implementation of external object store.', + 'version': '8.0.1.0.0', + 'author': 'Camptocamp,Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'Knowledge Management', + 'depends': ['base'], + 'website': 'http://www.camptocamp.com', + 'data': [], + 'installable': True, + 'auto_install': True, +} diff --git a/base_attachment_object_storage/models/__init__.py b/base_attachment_object_storage/models/__init__.py new file mode 100644 index 0000000..aaf38a1 --- /dev/null +++ b/base_attachment_object_storage/models/__init__.py @@ -0,0 +1 @@ +from . import ir_attachment diff --git a/base_attachment_object_storage/models/ir_attachment.py b/base_attachment_object_storage/models/ir_attachment.py new file mode 100644 index 0000000..bb0566b --- /dev/null +++ b/base_attachment_object_storage/models/ir_attachment.py @@ -0,0 +1,270 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import hashlib +import inspect +import logging +import os +import psycopg2 + +from openerp import _ +from openerp.osv import osv +from openerp.osv.orm import except_orm +from openerp import SUPERUSER_ID + + +_logger = logging.getLogger(__name__) + + +def clean_fs(files): + _logger.info('cleaning old files from filestore') + for full_path in files: + if os.path.exists(full_path): + try: + os.unlink(full_path) + except OSError: + _logger.info( + "_file_delete could not unlink %s", + full_path, exc_info=True + ) + except IOError: + # Harmless and needed for race conditions + _logger.info( + "_file_delete could not unlink %s", + full_path, exc_info=True + ) + + +class IrAttachment(osv.osv): + _inherit = 'ir.attachment' + + @staticmethod + def _compute_checksum(bin_data): + """ compute the checksum for the given datas + :param bin_data : datas in its binary form + """ + # an empty file has a checksum too (for caching) + return hashlib.sha1(bin_data or '').hexdigest() + + def _is_user_admin(self, cr, uid): + if uid == SUPERUSER_ID: + return True + else: + return self.pool.get('res.users').has_group( + cr, uid, 'base.group_erp_manager' + ) + + def _register_hook(self, cr): + super(IrAttachment, self)._register_hook(cr) + # ignore if we are not using an object storage + # Use directly SUPERUSER_ID + # because the uid parameter is required + # in function _storage and + # the SUPERUSER_ID is used directly instead of use the uid parameter. + if self._storage(cr, SUPERUSER_ID) not in self._get_stores(): + return + curframe = inspect.currentframe() + calframe = inspect.getouterframes(curframe, 2) + # the caller of _register_hook is 'load_modules' in + # odoo/modules/loading.py + # We have to go up 2 stacks because of the old api wrapper + load_modules_frame = calframe[2][0] + # 'update_module' is an argument that 'load_modules' receives with a + # True-ish value meaning that an install or upgrade of addon has been + # done during the initialization. We need to move the attachments that + # could have been created or updated in other addons before this addon + # was loaded + update_module = load_modules_frame.f_locals.get('update_module') + + # We need to call the migration on the loading of the model because + # when we are upgrading addons, some of them might add attachments. + # To be sure they are migrated to the storage we need to call the + # migration here. + # Typical example is images of ir.ui.menu which are updated in + # ir.attachment at every upgrade of the addons + if update_module: + self.pool.get('ir.attachment')._force_storage_to_object_storage( + cr, SUPERUSER_ID + ) + + def _save_in_db_anyway(self, cr, uid, ids, context=None): + """ Return whether an attachment must be stored in db + + When we are using an Object Store. This is sometimes required + because the object storage is slower than the database/filesystem. + + We store image_small and image_medium from 'Binary' fields + because they should be fast to read as they are often displayed + in kanbans / lists. The same for web_icon_data. + + We store the assets locally as well. Not only for performance, + but also because it improves the portability of the database: + when assets are invalidated, they are deleted so we don't have + an old database with attachments pointing to deleted assets. + + """ + assert (isinstance(ids, int) or + len(ids) == 1), 'Expecting only one record' + rec = self.browse(cr, uid, ids, context=context) + + # assets + if rec.res_model == 'ir.ui.view': + # assets are stored in 'ir.ui.view' + return True + + return False + + def _data_set(self, cr, uid, id, name, value, arg, context=None): + # override in order to store files that need fast access, + # we keep them in the database instead of the object storage + location = self._storage(cr, uid) + for attach in self.browse(cr, uid, id, context): + if (location in self._get_stores() and + self._save_in_db_anyway(cr, uid, [id], context)): + # compute the fields that depend on datas + bin_data = value and value.decode('base64') or '' + vals = { + 'file_size': len(bin_data), + 'checksum': self._compute_checksum(bin_data), + 'db_datas': value, + # we seriously don't need index content on those fields + 'index_content': False, + 'store_fname': False, + } + fname = attach.store_fname + # write as superuser, as user probably does not + # have write access + super(IrAttachment, self).write( + cr, SUPERUSER_ID, id, vals, context + ) + if fname: + self._file_delete(cr, uid, fname) + continue + self._data_set(cr, uid, id, 'datas', value, None, context) + + def _store_file_read(self, fname, bin_size=False): + storage = fname.partition('://')[0] + raise NotImplementedError( + 'No implementation for %s' % (storage,) + ) + + def _store_file_write(self, storage, key, bin_data): + raise NotImplementedError( + 'No implementation for %s' % (storage,) + ) + + def _store_file_delete(self, fname): + storage = fname.partition('://')[0] + raise NotImplementedError( + 'No implementation for %s' % (storage,) + ) + + def _file_read(self, cr, uid, fname, bin_size=False): + if self._is_file_from_a_store(fname): + return self._store_file_read(fname, bin_size=bin_size) + else: + _super = super(IrAttachment, self) + return _super._file_read(cr, uid, fname, bin_size=bin_size) + + def _file_write(self, cr, uid, value): + storage = self._storage(cr, uid) + if storage in self._get_stores(): + bin_data = value.decode('base64') + key = self._compute_checksum(bin_data) + filename = self._store_file_write(storage, key, bin_data) + else: + _super = super(IrAttachment, self) + filename = _super._file_write(cr, uid, value) + return filename + + def _file_delete(self, cr, uid, fname): + if self._is_file_from_a_store(fname): + # using SQL to include files hidden through unlink or due to record + # rules + cr.execute("SELECT COUNT(*) FROM ir_attachment " + "WHERE store_fname = %s", (fname,)) + count = cr.fetchone()[0] + if not count: + self._store_file_delete(fname) + else: + super(IrAttachment, self)._file_delete(cr, uid, fname) + + def _is_file_from_a_store(self, fname): + for store_name in self._get_stores(): + uri = '{}://'.format(store_name) + if fname.startswith(uri): + return True + return False + + def _move_attachment_to_store(self, cr, uid, ids, context=None): + assert (isinstance(ids, int) or + len(ids) == 1), 'Expecting only one record' + rec = self.browse(cr, uid, ids, context) + _logger.info('inspecting attachment %s (%d)', rec.name, rec.id) + fname = rec.store_fname + if fname: + # migrating from filesystem filestore + # or from the old 'store_fname' without the bucket name + _logger.info('moving %s on the object storage', fname) + self.write(cr, uid, ids, {'datas': rec.datas}, context) + _logger.info('moved %s on the object storage', fname) + return self._full_path(cr, uid, fname) + elif rec.db_datas: + _logger.info('moving on the object storage from database') + self.write(cr, uid, ids, {'datas': rec.datas}, context) + + def force_storage(self, cr, uid, context=None): + if not self._is_user_admin(cr, uid): + raise except_orm( + _('Error'), + _('Only administrators can execute this action.') + ) + storage = self._storage(cr, uid) + if storage not in self._get_stores(): + return super(IrAttachment, self).force_storage(cr, uid, context) + self._force_storage_to_object_storage(cr, uid, context) + + def _force_storage_to_object_storage(self, cr, uid, context=None): + _logger.info('migrating files to the object storage') + storage = self._storage(cr, uid) + + domain = [('store_fname', 'not like', '{}://%'.format(storage))] + + ids = self.search(cr, uid, domain, context=context) + files_to_clean = [] + for attachment_id in ids: + try: + with cr.savepoint(): + # check that no other transaction has + # locked the row, don't send a file to storage + # in that case + cr.execute( + "SELECT id " + "FROM ir_attachment " + "WHERE id = %s " + "FOR UPDATE NOWAIT", + (attachment_id,), + log_exceptions=False + ) + + path = self._move_attachment_to_store( + cr, uid, attachment_id, context + ) + if path: + files_to_clean.append(path) + except psycopg2.OperationalError: + _logger.error('Could not migrate attachment %s to %s' % + (attachment_id, storage)) + + def clean(): + clean_fs(files_to_clean) + + # delete the files from the filesystem once we know the changes + # have been committed in ir.attachment + if files_to_clean: + cr.after('commit', clean) + + def _get_stores(self): + """ To get the list of stores activated in the system """ + return [] From bb64595e4b31afab02f9deeb21f0ffa616492050 Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Tue, 7 May 2019 16:43:02 +0200 Subject: [PATCH 05/12] Add attachment_s3 --- attachment_s3/README.rst | 41 +++++++ attachment_s3/__init__.py | 3 + attachment_s3/__openerp__.py | 19 +++ attachment_s3/models/__init__.py | 2 + attachment_s3/models/ir_attachment.py | 164 ++++++++++++++++++++++++++ attachment_s3/s3uri.py | 22 ++++ requirements.txt | 1 + 7 files changed, 252 insertions(+) create mode 100644 attachment_s3/README.rst create mode 100644 attachment_s3/__init__.py create mode 100644 attachment_s3/__openerp__.py create mode 100644 attachment_s3/models/__init__.py create mode 100644 attachment_s3/models/ir_attachment.py create mode 100644 attachment_s3/s3uri.py diff --git a/attachment_s3/README.rst b/attachment_s3/README.rst new file mode 100644 index 0000000..9a887f2 --- /dev/null +++ b/attachment_s3/README.rst @@ -0,0 +1,41 @@ +Attachments on S3 storage +========================= + +This addon allows to store the attachments (documents and assets) on S3 or any +other S3-compatible Object Storage. + +Configuration +------------- + +Activate S3 storage: + +* Create or set the system parameter with the key ``ir_attachment.location`` + and the value in the form ``s3``. + +Configure accesses with environment variables: + +* ``AWS_HOST`` (not required if using AWS services) +* ``AWS_ACCESS_KEY_ID`` +* ``AWS_SECRET_ACCESS_KEY`` +* ``AWS_BUCKETNAME`` + +Read-only mode: + +The bucket and the file key are stored in the attachment. So if you change the +``AWS_BUCKETNAME`` or the ``ir_attachment.location``, the existing attachments +will still be read on their former bucket. But as soon as they are written over +or new attachments are created, they will be created on the new bucket or on +the other location (db or filesystem). This is a convenient way to be able to +read the production attachments on a replication (since you have the +credentials) without any risk to alter the production data. + +This addon must be added in the server wide addons with (``--load`` option): + +``--load=web,web_kanban,attachment_s3`` + +Limitations +----------- + +* You need to call ``env['ir.attachment'].force_storage()`` after + having changed the ``ir_attachment.location`` configuration in order to + migrate the existing attachments to S3. diff --git a/attachment_s3/__init__.py b/attachment_s3/__init__.py new file mode 100644 index 0000000..cde864b --- /dev/null +++ b/attachment_s3/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from . import models diff --git a/attachment_s3/__openerp__.py b/attachment_s3/__openerp__.py new file mode 100644 index 0000000..bccf74e --- /dev/null +++ b/attachment_s3/__openerp__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{'name': 'Attachments on S3 storage', + 'summary': 'Store assets and attachments on a S3 compatible object storage', + 'version': '8.0.1.0.0', + 'author': 'Camptocamp,Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'Knowledge Management', + 'depends': ['base_attachment_object_storage'], + 'external_dependencies': { + 'python': ['boto'], + }, + 'website': 'http://www.camptocamp.com', + 'data': [], + 'installable': True, + } diff --git a/attachment_s3/models/__init__.py b/attachment_s3/models/__init__.py new file mode 100644 index 0000000..d44b7af --- /dev/null +++ b/attachment_s3/models/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +from . import ir_attachment diff --git a/attachment_s3/models/ir_attachment.py b/attachment_s3/models/ir_attachment.py new file mode 100644 index 0000000..74afd61 --- /dev/null +++ b/attachment_s3/models/ir_attachment.py @@ -0,0 +1,164 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +import base64 +import logging +import os +import xml.dom.minidom +from functools import partial + +from openerp import _ +from openerp.osv import osv +from openerp.osv.orm import except_orm +from ..s3uri import S3Uri + +_logger = logging.getLogger(__name__) + +try: + import boto + from boto.exception import S3ResponseError +except ImportError: + boto = None # noqa + S3ResponseError = None # noqa + _logger.debug("Cannot 'import boto'.") + + +class IrAttachment(osv.osv): + _inherit = "ir.attachment" + + def _get_stores(self): + return ['s3'] + super(IrAttachment, self)._get_stores() + + def _get_s3_bucket(self, name=None): + """Connect to S3 and return the bucket + + The following environment variables can be set: + * ``AWS_HOST`` + * ``AWS_ACCESS_KEY_ID`` + * ``AWS_SECRET_ACCESS_KEY`` + * ``AWS_BUCKETNAME`` + + If a name is provided, we'll read this bucket, otherwise, the bucket + from the environment variable ``AWS_BUCKETNAME`` will be read. + + """ + host = os.environ.get('AWS_HOST') + if host: + connect_s3 = partial(boto.connect_s3, host=host) + else: + connect_s3 = boto.connect_s3 + + access_key = os.environ.get('AWS_ACCESS_KEY_ID') + secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') + if name: + bucket_name = name + else: + bucket_name = os.environ.get('AWS_BUCKETNAME') + if not (access_key and secret_key and bucket_name): + msg = _('If you want to read from the %s S3 bucket, the following ' + 'environment variables must be set:\n' + '* AWS_ACCESS_KEY_ID\n' + '* AWS_SECRET_ACCESS_KEY\n' + 'If you want to write in the %s S3 bucket, this variable ' + 'must be set as well:\n' + '* AWS_BUCKETNAME\n' + 'Optionally, the S3 host can be changed with:\n' + '* AWS_HOST\n' + ) % (bucket_name, bucket_name) + + raise except_orm(_('Configuration Error'), msg) + + try: + conn = connect_s3(aws_access_key_id=access_key, + aws_secret_access_key=secret_key) + + except S3ResponseError as error: + # log verbose error from s3, return short message for user + _logger.exception('Error during connection on S3') + raise except_orm(_('S3 Error'), self._parse_s3_error(error)) + + bucket = conn.lookup(bucket_name) + if not bucket: + bucket = conn.create_bucket(bucket_name) + return bucket + + @staticmethod + def _parse_s3_error(s3error): + msg = s3error.reason + # S3 error message is a XML message... + doc = xml.dom.minidom.parseString(s3error.body) + msg_node = doc.getElementsByTagName('Message') + if msg_node: + msg = '%s: %s' % (msg, msg_node[0].childNodes[0].data) + return msg + + def _store_file_read(self, fname, bin_size=False): + if fname.startswith('s3://'): + s3uri = S3Uri(fname) + try: + bucket = self._get_s3_bucket(name=s3uri.bucket()) + except except_orm: + _logger.exception( + "error reading attachment '%s' from object storage", fname + ) + return '' + filekey = bucket.get_key(s3uri.item()) + if filekey: + read = base64.b64encode(filekey.get_contents_as_string()) + else: + read = '' + _logger.info( + "attachment '%s' missing on object storage", fname + ) + return read + else: + return super(IrAttachment, self)._store_file_read(fname, bin_size) + + def _store_file_write(self, storage, key, bin_data): + if storage == 's3': + bucket = self._get_s3_bucket() + filekey = bucket.get_key(key) or bucket.new_key(key) + filename = 's3://%s/%s' % (bucket.name, key) + try: + filekey.set_contents_from_string(bin_data) + except S3ResponseError as error: + # log verbose error from s3, return short message for user + _logger.exception( + 'Error during storage of the file %s' % filename + ) + raise except_orm( + _('S3 Error'), + _('The file could not be stored: %s') % + (self._parse_s3_error(error),) + ) + else: + _super = super(IrAttachment, self) + filename = _super._store_file_write(key, bin_data) + return filename + + def _store_file_delete(self, fname): + if fname.startswith('s3://'): + s3uri = S3Uri(fname) + bucket_name = s3uri.bucket() + item_name = s3uri.item() + # delete the file only if it is on the current configured bucket + # otherwise, we might delete files used on a different environment + if bucket_name == os.environ.get('AWS_BUCKETNAME'): + bucket = self._get_s3_bucket() + filekey = bucket.get_key(item_name) + if filekey: + try: + filekey.delete() + _logger.info( + 'file %s deleted on the object storage' % (fname,) + ) + except S3ResponseError: + # log verbose error from s3, return short message for + # user + _logger.exception( + 'Error during deletion of the file %s' % fname + ) + else: + super(IrAttachment, self)._file_delete_from_store(fname) diff --git a/attachment_s3/s3uri.py b/attachment_s3/s3uri.py new file mode 100644 index 0000000..f94df79 --- /dev/null +++ b/attachment_s3/s3uri.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import re + + +class S3Uri(object): + + _url_re = re.compile("^s3:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE) + + def __init__(self, uri): + match = self._url_re.match(uri) + if not match: + raise ValueError("%s: is not a valid S3 URI" % (uri,)) + self._bucket, self._item = match.groups() + + def bucket(self): + return self._bucket + + def item(self): + return self._item diff --git a/requirements.txt b/requirements.txt index 204d7d9..3d3be20 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ +boto==2.42.0 redis==2.10.5 python-json-logger==0.1.5 From 2dc5842b3dd6b73753f8c7564cc32bb9d9f01794 Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Tue, 7 May 2019 17:00:59 +0200 Subject: [PATCH 06/12] Add attachment_swift --- attachment_swift/README.rst | 51 ++++++++++ attachment_swift/__init__.py | 1 + attachment_swift/__openerp__.py | 21 ++++ attachment_swift/models/__init__.py | 1 + attachment_swift/models/ir_attachment.py | 120 +++++++++++++++++++++++ attachment_swift/swift_uri.py | 23 +++++ requirements.txt | 2 + 7 files changed, 219 insertions(+) create mode 100644 attachment_swift/README.rst create mode 100644 attachment_swift/__init__.py create mode 100644 attachment_swift/__openerp__.py create mode 100644 attachment_swift/models/__init__.py create mode 100644 attachment_swift/models/ir_attachment.py create mode 100644 attachment_swift/swift_uri.py diff --git a/attachment_swift/README.rst b/attachment_swift/README.rst new file mode 100644 index 0000000..8999290 --- /dev/null +++ b/attachment_swift/README.rst @@ -0,0 +1,51 @@ +Attachments on Swift storage +============================ + +This addon enable storing attachments (documents and assets) on OpenStack Object Storage (Swift) + +Configuration +------------- + +Activate Swift storage: + +* Create or set the system parameter with the key ``ir_attachment.location`` with the following value ``swift``. + +Configure accesses with environment variables: + +* ``SWIFT_AUTH_URL`` : URL of the Swift server +* ``SWIFT_TENANT_NAME`` +* ``SWIFT_ACCOUNT`` +* ``SWIFT_PASSWORD`` +* ``SWIFT_WRITE_CONTAINER`` : Name of the container to use in the store (created if not existing) + +Read-only mode: + +The container name and the key are stored in the attachment. So if you change the +``SWIFT_WRITE_CONTAINER`` or the ``ir_attachment.location``, the existing attachments +will still be read on their former container. But as soon as they are written over +or new attachments are created, they will be created on the new container or on +the other location (db or filesystem). This is a convenient way to be able to +read the production attachments on a replication (since you have the +credentials) without any risk to alter the production data. + +This addon must be added in the server wide addons with (``--load`` option): + +``--load=web,web_kanban,attachment_swift`` + +Python Dependencies +------------------- + +This module needs the python-swiftclient and the python-keystoneclient (For auth v2.0) to work. +The python-keystoneclient needs the linux package build-essential and python-dev to install properly. + +The python-swiftclient can be used from the command line, useful to test: + + export AUTH_VERSION=2.0 + export OS_USERNAME={SWIFT_ACCOUNT} + export OS_PASSWORD={SWIFT_PASSWORD} + export OS_TENANT_NAME={SWIFT_TENANT_NAME} + export OS_AUTH_URL=https://auth.cloud.ovh.net/v2.0 + swift stat + +More information at +https://docs.openstack.org/python-swiftclient/latest/cli/index.html#swift-usage diff --git a/attachment_swift/__init__.py b/attachment_swift/__init__.py new file mode 100644 index 0000000..0650744 --- /dev/null +++ b/attachment_swift/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/attachment_swift/__openerp__.py b/attachment_swift/__openerp__.py new file mode 100644 index 0000000..55e825b --- /dev/null +++ b/attachment_swift/__openerp__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{'name': 'Attachments on Swift storage', + 'summary': 'Store assets and attachments on a Swift compatible object store', + 'version': '8.0.1.0.0', + 'author': 'Camptocamp,Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'Knowledge Management', + 'depends': ['base_attachment_object_storage'], + 'external_dependencies': { + 'python': ['swiftclient', + 'keystoneclient', + ], + }, + 'website': 'https://www.camptocamp.com', + 'data': [], + 'installable': True, + } diff --git a/attachment_swift/models/__init__.py b/attachment_swift/models/__init__.py new file mode 100644 index 0000000..aaf38a1 --- /dev/null +++ b/attachment_swift/models/__init__.py @@ -0,0 +1 @@ +from . import ir_attachment diff --git a/attachment_swift/models/ir_attachment.py b/attachment_swift/models/ir_attachment.py new file mode 100644 index 0000000..62fc203 --- /dev/null +++ b/attachment_swift/models/ir_attachment.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +import base64 +import logging +import os +from ..swift_uri import SwiftUri + +from openerp import _ +from openerp.osv import osv +from openerp.osv.orm import except_orm + +_logger = logging.getLogger(__name__) + +try: + import swiftclient + from swiftclient.exceptions import ClientException +except ImportError: + swiftclient = None + ClientException = None + _logger.debug("Cannot 'import swiftclient'.") + + +class IrAttachment(osv.osv): + _inherit = 'ir.attachment' + + def _get_stores(self): + return ['swift'] + super(IrAttachment, self)._get_stores() + + def _get_swift_connection(self): + """ Returns a connection object for the Swift object store """ + host = os.environ.get('SWIFT_AUTH_URL') + account = os.environ.get('SWIFT_ACCOUNT') + password = os.environ.get('SWIFT_PASSWORD') + tenant_name = os.environ.get('SWIFT_TENANT_NAME') + region = os.environ.get('SWIFT_REGION_NAME') + os_options = {} + if region: + os_options['region_name'] = region + if not (host and account and password and tenant_name): + raise except_orm( + _("Error"), + _("Problem connecting to Swift store, are the env variables " + "(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, " + "SWIFT_TENANT_NAME) properly set?") + ) + try: + conn = swiftclient.client.Connection(authurl=host, + user=account, + key=password, + tenant_name=tenant_name, + auth_version='2.0', + os_options=os_options, + ) + except ClientException: + _logger.exception('Error connecting to Swift object store') + raise except_orm( + _("Error"), + _('Error on Swift connection')) + return conn + + def _store_file_read(self, fname, bin_size=False): + if fname.startswith('swift://'): + swifturi = SwiftUri(fname) + try: + conn = self._get_swift_connection() + except except_orm: + _logger.exception( + "error reading attachment '%s' from object storage", fname + ) + return '' + try: + resp, obj_content = conn.get_object(swifturi.container(), + swifturi.item()) + read = base64.b64encode(obj_content) + except ClientException: + read = '' + _logger.exception( + 'Error reading object from Swift object store') + return read + else: + return super(IrAttachment, self)._store_file_read(fname, bin_size) + + def _store_file_write(self, storage, key, bin_data): + if storage == 'swift': + container = os.environ.get('SWIFT_WRITE_CONTAINER') + conn = self._get_swift_connection() + conn.put_container(container) + filename = 'swift://{}/{}'.format(container, key) + try: + conn.put_object(container, key, bin_data) + except ClientException: + _logger.exception('Error writing to Swift object store') + raise except_orm( + _("Error"), + _('Error writing to Swift')) + else: + _super = super(IrAttachment, self) + filename = _super._store_file_write(key, bin_data) + return filename + + def _store_file_delete(self, fname): + if fname.startswith('swift://'): + swifturi = SwiftUri(fname) + container = swifturi.container() + # delete the file only if it is on the current configured bucket + # otherwise, we might delete files used on a different environment + if container == os.environ.get('SWIFT_WRITE_CONTAINER'): + conn = self._get_swift_connection() + try: + conn.delete_object(container, swifturi.item()) + except ClientException: + _logger.exception( + _('Error deleting an object on the Swift store')) + # we ignore the error, file will stay on the object + # storage but won't disrupt the process + else: + super(IrAttachment, self)._store_file_delete(fname) diff --git a/attachment_swift/swift_uri.py b/attachment_swift/swift_uri.py new file mode 100644 index 0000000..268eeec --- /dev/null +++ b/attachment_swift/swift_uri.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import re + + +class SwiftUri(object): + + _url_re = re.compile("^swift:///*([^/]*)/?(.*)", + re.IGNORECASE | re.UNICODE) + + def __init__(self, uri): + match = self._url_re.match(uri) + if not match: + raise ValueError("%s: is not a valid Swift URI" % (uri,)) + self._container, self._item = match.groups() + + def container(self): + return self._container + + def item(self): + return self._item diff --git a/requirements.txt b/requirements.txt index 3d3be20..b62181e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,5 @@ boto==2.42.0 redis==2.10.5 python-json-logger==0.1.5 +python-swiftclient==3.4.0 +python-keystoneclient==3.13.0 From d85722fdf15d3b1080665e0554368d4ba91b6292 Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Wed, 8 May 2019 11:43:43 +0200 Subject: [PATCH 07/12] Add cloud_platform --- cloud_platform/README.rst | 11 ++ cloud_platform/__init__.py | 3 + cloud_platform/__openerp__.py | 22 +++ cloud_platform/models/__init__.py | 2 + cloud_platform/models/cloud_platform.py | 248 ++++++++++++++++++++++++ oca_dependencies.txt | 1 + 6 files changed, 287 insertions(+) create mode 100644 cloud_platform/README.rst create mode 100644 cloud_platform/__init__.py create mode 100644 cloud_platform/__openerp__.py create mode 100644 cloud_platform/models/__init__.py create mode 100644 cloud_platform/models/cloud_platform.py create mode 100644 oca_dependencies.txt diff --git a/cloud_platform/README.rst b/cloud_platform/README.rst new file mode 100644 index 0000000..5e1e75c --- /dev/null +++ b/cloud_platform/README.rst @@ -0,0 +1,11 @@ +Cloud Platform +============== + +Install addons required for the Camptocamp Cloud platform. + +* Provide a quick install that we can call at the setup / migration + of a database +* Check if the environment variables are configured correctly according + to the instance's environment (prod, integration, test or dev) to prevent + data corruption between the environments (such as the integration server + writing on the production object storage). diff --git a/cloud_platform/__init__.py b/cloud_platform/__init__.py new file mode 100644 index 0000000..cde864b --- /dev/null +++ b/cloud_platform/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from . import models diff --git a/cloud_platform/__openerp__.py b/cloud_platform/__openerp__.py new file mode 100644 index 0000000..b9ebbfc --- /dev/null +++ b/cloud_platform/__openerp__.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{'name': 'Cloud Platform', + 'summary': 'Addons required for the Camptocamp Cloud Platform', + 'version': '8.0.1.0.0', + 'author': 'Camptocamp,Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'Extra Tools', + 'depends': [ + 'base_attachment_object_storage', + 'session_redis', + 'monitoring_status', + 'logging_json', + 'server_environment', # OCA/server-tools + ], + 'website': 'http://www.camptocamp.com', + 'data': [], + 'installable': True, + } diff --git a/cloud_platform/models/__init__.py b/cloud_platform/models/__init__.py new file mode 100644 index 0000000..5f2c99d --- /dev/null +++ b/cloud_platform/models/__init__.py @@ -0,0 +1,2 @@ +# -*- coding: utf-8 -*- +from . import cloud_platform diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py new file mode 100644 index 0000000..f038f3d --- /dev/null +++ b/cloud_platform/models/cloud_platform.py @@ -0,0 +1,248 @@ +# -*- coding: utf-8 -*- +# Copyright 2016 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import logging +import os +import re + +from collections import namedtuple + +from distutils.util import strtobool + +from openerp import SUPERUSER_ID +from openerp.osv import osv +from openerp.tools.config import config + + +_logger = logging.getLogger(__name__) + + +def is_true(strval): + return bool(strtobool(strval or '0'.lower())) + + +PlatformConfig = namedtuple( + 'PlatformConfig', + 'filestore' +) + + +class FilestoreKind(object): + db = 'db' + s3 = 's3' # or compatible s3 object storage + swift = 'swift' + file = 'file' + + +class CloudPlatform(osv.osv_abstract): + _name = 'cloud.platform' + + def _platform_kinds(self): + # XXX for backward compatibility, we need this one here, move + # it in cloud_platform_exoscale in V11 + return ['exoscale'] + + # XXX for backward compatibility, we need this one here, move + # it in cloud_platform_exoscale in V11 + def _config_by_server_env_for_exoscale(self): + configs = { + 'prod': PlatformConfig(filestore=FilestoreKind.s3), + 'integration': PlatformConfig(filestore=FilestoreKind.s3), + 'test': PlatformConfig(filestore=FilestoreKind.db), + 'dev': PlatformConfig(filestore=FilestoreKind.db), + } + return configs + + def _config_by_server_env(self, platform_kind, environment): + configs_getter = getattr( + self, + '_config_by_server_env_for_%s' % platform_kind, + None + ) + configs = configs_getter() if configs_getter else {} + return configs.get(environment) or FilestoreKind.db + + # Due to the addition of the ovh cloud platform + # This will be moved to cloud_platform_exoscale on v11 + def install_exoscale(self, cr, uid, context=None): + self.install(cr, uid, 'exoscale', context) + + def install(self, cr, uid, platform_kind, context=None): + assert platform_kind in self._platform_kinds() + params = self.pool.get('ir.config_parameter') + params.set_param( + cr, SUPERUSER_ID, + 'cloud.platform.kind', platform_kind, + context=context + ) + environment = config['running_env'] + configs = self._config_by_server_env(platform_kind, environment) + params.set_param( + cr, SUPERUSER_ID, + 'ir_attachment.location', configs.filestore, + context=context + ) + self.check(cr, uid, context) + if configs.filestore in [FilestoreKind.swift, FilestoreKind.s3]: + self.pool.get('ir.attachment').force_storage( + cr, SUPERUSER_ID, context=context + ) + _logger.info('cloud platform configured for {}'.format(platform_kind)) + + def _check_swift(self, cr, uid, environment_name, context=None): + params = self.pool.get('ir.config_parameter') + use_swift = ( + params.get_param( + cr, SUPERUSER_ID, 'ir_attachment.location', context=context + ) == FilestoreKind.swift + ) + if environment_name in ('prod', 'integration'): + assert use_swift, ( + "Swift must be used on production and integration instances. " + "It is activated, setting 'ir_attachment.location.' to 'swift'" + " The 'install_exoscale()' function sets this option " + "automatically." + ) + if use_swift: + assert os.environ.get('SWIFT_AUTH_URL'), ( + "SWIFT_AUTH_URL environment variable is required when " + "ir_attachment.location is 'swift'." + ) + assert os.environ.get('SWIFT_ACCOUNT'), ( + "SWIFT_ACCOUNT environment variable is required when " + "ir_attachment.location is 'swift'." + ) + assert os.environ.get('SWIFT_PASSWORD'), ( + "SWIFT_PASSWORD environment variable is required when " + "ir_attachment.location is 'swift'." + ) + container_name = os.environ['SWIFT_WRITE_CONTAINER'] + if environment_name in ('integration', 'prod'): + assert container_name, ( + "SWIFT_WRITE_CONTAINER must not be empty for prod " + "and integration" + ) + prod_container = bool(re.match(r'[a-z0-9_-]+-odoo-prod', + container_name)) + if environment_name == 'prod': + assert prod_container, ( + "SWIFT_WRITE_CONTAINER should match '-odoo-prod', " + "we got: '%s'" % (container_name,) + ) + else: + # if we are using the prod bucket on another instance + # such as an integration, we must be sure to be in read only! + assert not prod_container, ( + "SWIFT_WRITE_CONTAINER should not match " + "'-odoo-prod', we got: '%s'" % (container_name,) + ) + elif environment_name == 'test': + # store in DB so we don't have files local to the host + assert params.get_param(cr, SUPERUSER_ID, 'ir_attachment.location', + context=context) == 'db', ( + "In test instances, files must be stored in the database with " + "'ir_attachment.location' set to 'db'. This is " + "automatically set by the function 'install_ovh()'." + ) + + def _check_s3(self, cr, uid, environment_name, context=None): + params = self.pool.get('ir.config_parameter') + use_s3 = params.get_param( + cr, SUPERUSER_ID, 'ir_attachment.location', context=context + ) == FilestoreKind.s3 + if environment_name in ('prod', 'integration'): + assert use_s3, ( + "S3 must be used on production and integration instances. " + "It is activated by setting 'ir_attachment.location.' to 's3'." + " The 'install_exoscale()' function sets this option " + "automatically." + ) + if use_s3: + assert os.environ.get('AWS_ACCESS_KEY_ID'), ( + "AWS_ACCESS_KEY_ID environment variable is required when " + "ir_attachment.location is 's3'." + ) + assert os.environ.get('AWS_SECRET_ACCESS_KEY'), ( + "AWS_SECRET_ACCESS_KEY environment variable is required when " + "ir_attachment.location is 's3'." + ) + assert os.environ.get('AWS_BUCKETNAME'), ( + "AWS_BUCKETNAME environment variable is required when " + "ir_attachment.location is 's3'.\n" + "Normally, 's3' is activated on integration and production, " + "but should not be used in dev environment (or at least " + "not with a dev bucket, but never the " + "integration/prod bucket)." + ) + bucket_name = os.environ['AWS_BUCKETNAME'] + prod_bucket = bool(re.match(r'[a-z0-9_-]+-odoo-prod', bucket_name)) + if environment_name == 'prod': + assert prod_bucket, ( + "AWS_BUCKETNAME should match '-odoo-prod', " + "we got: '%s'" % (bucket_name,) + ) + else: + # if we are using the prod bucket on another instance + # such as an integration, we must be sure to be in read only! + assert not prod_bucket, ( + "AWS_BUCKETNAME should not match '-odoo-prod', " + "we got: '%s'" % (bucket_name,) + ) + + elif environment_name == 'test': + # store in DB so we don't have files local to the host + assert params.get_param(cr, SUPERUSER_ID, 'ir_attachment.location', + context=context) == 'db', ( + "In test instances, files must be stored in the database with " + "'ir_attachment.location' set to 'db'. This is " + "automatically set by the function 'install_exoscale()'." + ) + + def _check_redis(self, cr, uid, environment_name, context=None): + if environment_name in ('prod', 'integration', 'test'): + assert is_true(os.environ.get('ODOO_SESSION_REDIS')), ( + "Redis must be activated on prod, integration, test instances." + "This is done by setting ODOO_SESSION_REDIS=1." + ) + assert (os.environ.get('ODOO_SESSION_REDIS_HOST') or + os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST')), ( + "ODOO_SESSION_REDIS_HOST or ODOO_SESSION_REDIS_SENTINEL_HOST " + "environment variable is required to connect on Redis" + ) + assert os.environ.get('ODOO_SESSION_REDIS_PREFIX'), ( + "ODOO_SESSION_REDIS_PREFIX environment variable is required " + "to store sessions on Redis" + ) + + prefix = os.environ['ODOO_SESSION_REDIS_PREFIX'] + assert re.match(r'[a-z0-9_-]+-odoo-[a-z]+', prefix), ( + "ODOO_SESSION_REDIS_PREFIX must match '-odoo-'" + ", we got: '%s'" % (prefix,) + ) + + def check(self, cr, uid, context=None): + if is_true(os.environ.get('ODOO_CLOUD_PLATFORM_UNSAFE')): + _logger.warning( + "cloud platform checks disabled, this is not safe" + ) + return + params = self.pool.get('ir.config_parameter') + kind = params.get_param(cr, SUPERUSER_ID, + 'cloud.platform.kind', context=None) + if not kind: + _logger.warning( + "cloud platform not configured, you should " + "probably run 'env['cloud.platform'].install_exoscale()'" + ) + return + environment_name = config['running_env'] + if kind == 'exoscale': + self._check_s3(cr, uid, environment_name, context) + elif kind == 'ovh': + self._check_swift(cr, uid, environment_name, context) + self._check_redis(cr, uid, environment_name, context) + + def _register_hook(self, cr): + super(CloudPlatform, self)._register_hook(cr) + self.pool.get('cloud.platform').check(cr, SUPERUSER_ID) diff --git a/oca_dependencies.txt b/oca_dependencies.txt new file mode 100644 index 0000000..9c8c917 --- /dev/null +++ b/oca_dependencies.txt @@ -0,0 +1 @@ +server-tools From a6bc06d9a99ee7ed1d9a255929f9f2738e01130b Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Wed, 8 May 2019 12:00:32 +0200 Subject: [PATCH 08/12] Add cloud_platform_ovh --- cloud_platform_ovh/README.md | 7 ++++ cloud_platform_ovh/__init__.py | 1 + cloud_platform_ovh/__openerp__.py | 19 ++++++++++ cloud_platform_ovh/models/__init__.py | 1 + cloud_platform_ovh/models/cloud_platform.py | 40 +++++++++++++++++++++ 5 files changed, 68 insertions(+) create mode 100644 cloud_platform_ovh/README.md create mode 100644 cloud_platform_ovh/__init__.py create mode 100644 cloud_platform_ovh/__openerp__.py create mode 100644 cloud_platform_ovh/models/__init__.py create mode 100644 cloud_platform_ovh/models/cloud_platform.py diff --git a/cloud_platform_ovh/README.md b/cloud_platform_ovh/README.md new file mode 100644 index 0000000..c350eba --- /dev/null +++ b/cloud_platform_ovh/README.md @@ -0,0 +1,7 @@ +Cloud Platform OVH +================== + +Install addons specific to the OVH setup. + + * The object storage is Swift + diff --git a/cloud_platform_ovh/__init__.py b/cloud_platform_ovh/__init__.py new file mode 100644 index 0000000..0650744 --- /dev/null +++ b/cloud_platform_ovh/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/cloud_platform_ovh/__openerp__.py b/cloud_platform_ovh/__openerp__.py new file mode 100644 index 0000000..2c6369e --- /dev/null +++ b/cloud_platform_ovh/__openerp__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{'name': 'Cloud Platform OVH', + 'summary': 'Addons required for the Camptocamp Cloud Platform on OVH', + 'version': '8.0.1.0.0', + 'author': 'Camptocamp,Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'Extra Tools', + 'depends': [ + 'cloud_platform', + 'attachment_swift', + ], + 'website': 'https://www.camptocamp.com', + 'data': [], + 'installable': True, + } diff --git a/cloud_platform_ovh/models/__init__.py b/cloud_platform_ovh/models/__init__.py new file mode 100644 index 0000000..5d08f36 --- /dev/null +++ b/cloud_platform_ovh/models/__init__.py @@ -0,0 +1 @@ +from . import cloud_platform diff --git a/cloud_platform_ovh/models/cloud_platform.py b/cloud_platform_ovh/models/cloud_platform.py new file mode 100644 index 0000000..fed682d --- /dev/null +++ b/cloud_platform_ovh/models/cloud_platform.py @@ -0,0 +1,40 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import logging + +from openerp.osv import osv + +_logger = logging.getLogger(__name__) + +try: + from openerp.addons.cloud_platform.models.cloud_platform \ + import FilestoreKind + from openerp.addons.cloud_platform.models.cloud_platform \ + import PlatformConfig +except ImportError: + FilestoreKind = None + PlatformConfig = None + _logger.debug("Cannot 'import from cloud_platform'") + + +class CloudPlatform(osv.osv_abstract): + _inherit = 'cloud.platform' + + def _platform_kinds(self): + kinds = super(CloudPlatform, self)._platform_kinds() + kinds.append('ovh') + return kinds + + def _config_by_server_env_for_ovh(self): + configs = { + 'prod': PlatformConfig(filestore=FilestoreKind.swift), + 'integration': PlatformConfig(filestore=FilestoreKind.swift), + 'test': PlatformConfig(filestore=FilestoreKind.db), + 'dev': PlatformConfig(filestore=FilestoreKind.db), + } + return configs + + def install_ovh(self, cr, uid, context=None): + self.install(cr, uid, 'ovh', context) From 2259581ef861c07e8bdcb37e5d10bdd5beb73f36 Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Wed, 8 May 2019 12:17:55 +0200 Subject: [PATCH 09/12] Add README.md --- README.md | 134 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 134 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..3b4940c --- /dev/null +++ b/README.md @@ -0,0 +1,134 @@ +[![Build Status](https://travis-ci.com/camptocamp/odoo-cloud-platform.svg?token=Lpp9PcS5on9AGbp76WKB&branch=8.0)](https://travis-ci.com/camptocamp/odoo-cloud-platform) + +# Odoo Cloud Addons + +Camptocamp odoo addons used on our Cloud Platform. + +## Introduction + +On the platform we want to achieve having: + +* No data stored on the local filesystem so we can move an instance + between hosts and even have several running front-ends +* Logs sent to ElasticSearch-Kibana structured as JSON for better searching + +For the storage, we store all the attachments on an object storage such as S3 or +Swift, and we store the werkzeug sessions on Redis. + +## Setup + +### Python dependencies + +Libraries that must be added in ``requirements.txt``: + +``` +redis==2.10.5 +python-json-logger==0.1.5 + +# For S3 object storage (Exoscale, AWS) +boto==2.42.0 + +# For Swift object storage (Openstack, OVH) +python-swiftclient==3.4.0 +python-keystoneclient==3.13.0 +``` + +### Odoo Startup + +The `--load` option of Odoo must contains the following addons: + +* `session_redis` +* `logging_json` + +Example: + +`--load=web,web_kanban,session_redis,logging_json` + +### Server Environment + +The addon `cloud_platform` is an addon that we use for 2 things: + +* validate that we setup the required environment variables depending on the running environment +* install and configure the cloud addons + +For this purpose, we use the `server_environment` with the following envs: + +* `prod` +* `integration` +* `test` +* `dev` + +The exact naming is important, because the `cloud_platform` addon rely on these keys to know and check the running environment. + + +### Attachments in the Object Storage + +* prod: stored RW in the object storage + * `AWS_HOST`: depends of the platform + * `AWS_ACCESS_KEY_ID`: depends of the platform + * `AWS_SECRET_ACCESS_KEY`: depends of the platform + * `AWS_BUCKETNAME`: `-odoo-prod` +* integration: + * `AWS_HOST`: depends of the platform + * `AWS_ACCESS_KEY_ID`: depends of the platform + * `AWS_SECRET_ACCESS_KEY`: depends of the platform + * `AWS_BUCKETNAME`: `-odoo-integration` +* test: attachments are stored in database + +Besides, the attachment location should be set to `s3` (this is +automatically done by the `install` methods of the `cloud_platform` module). + * `ir.config_parameter` `ir_attachment.location`: `s3` + + +### Attachments in the Object Storage Swift + +* prod: stored RW in the object storage + * `SWIFT_AUTH_URL`: depends of the platform + * `SWIFT_ACCOUNT`: depends of the platform + * `SWIFT_PASSWORD`: depends of the platform + * `SWIFT_WRITE_CONTAINER`: `-odoo-prod` +* integration: + * `SWIFT_AUTH_URL`: depends of the platform + * `SWIFT_ACCOUNT`: depends of the platform + * `SWIFT_PASSWORD`: depends of the platform + * `SWIFT_WRITE_CONTAINER`: `-odoo-integration` +* test: attachments are stored in database + +Besides, the attachment location should be set to `swift` (this is +automatically done by the `install` methods of the `cloud_platform` module). + * `ir.config_parameter` `ir_attachment.location`: `swift` + +### Sessions in Redis + +* prod: + * `ODOO_SESSION_REDIS`: 1 + * `ODOO_SESSION_REDIS_HOST`: depends of the platform + * `ODOO_SESSION_REDIS_PASSWORD`: depends of the platform + * `ODOO_SESSION_REDIS_PREFIX`: `-odoo-prod` +* integration: + * `ODOO_SESSION_REDIS`: 1 + * `ODOO_SESSION_REDIS_HOST`: depends of the platform + * `ODOO_SESSION_REDIS_PASSWORD`: depends of the platform + * `ODOO_SESSION_REDIS_PREFIX`: `-odoo-integration` +* test: + * `ODOO_SESSION_REDIS`: 1 + * `ODOO_SESSION_REDIS_HOST`: depends of the platform + * `ODOO_SESSION_REDIS_PASSWORD`: depends of the platform + * `ODOO_SESSION_REDIS_PREFIX`: `-odoo-test` + * `ODOO_SESSION_REDIS_EXPIRATION`: `86400` (1 day) + +### JSON Logging + +At least on production and integration, activate: +* `ODOO_LOGGING_JSON`: 1 +* Add ``logging_json`` in the ``server_wide_modules`` option in the + configuration file + +### Startup checks + +At loading of the database, the addon will check if the environment variables +for Redis and the object storage are set as expected for the loaded +environment. It will refuse to start if anything is badly configured. + +The checks can be bypassed with the environment variable +`ODOO_CLOUD_PLATFORM_UNSAFE` set to `1`. From fa512a6b746795556a5ab39e31aa7952cf28b05e Mon Sep 17 00:00:00 2001 From: Patrick Tombez <35060345+p-tombez@users.noreply.github.com> Date: Mon, 24 Jun 2019 10:28:00 +0200 Subject: [PATCH 10/12] Update base_attachment_object_storage/README.rst Co-Authored-By: Alex Saunier --- base_attachment_object_storage/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/base_attachment_object_storage/README.rst b/base_attachment_object_storage/README.rst index c802faf..08e97ad 100644 --- a/base_attachment_object_storage/README.rst +++ b/base_attachment_object_storage/README.rst @@ -1,7 +1,7 @@ Base class for attachments on external object store =================================================== -This is a base addon that regroup common code used by addons targeting specific object store +This is a base addon that regroups common code used by addons targeting specific object store. From 8100d7d11c7c409b1caaad3f8f591c5364cb17ea Mon Sep 17 00:00:00 2001 From: Patrick Tombez <35060345+p-tombez@users.noreply.github.com> Date: Mon, 24 Jun 2019 10:28:37 +0200 Subject: [PATCH 11/12] Update attachment_swift/README.rst Co-Authored-By: Alex Saunier --- attachment_swift/README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/attachment_swift/README.rst b/attachment_swift/README.rst index 8999290..a5bcc19 100644 --- a/attachment_swift/README.rst +++ b/attachment_swift/README.rst @@ -1,7 +1,7 @@ Attachments on Swift storage ============================ -This addon enable storing attachments (documents and assets) on OpenStack Object Storage (Swift) +This addon enables storing attachments (documents and assets) on OpenStack Object Storage (Swift). Configuration ------------- From 1af4f271df08844245f1565c4fb5904ccc18a6a1 Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Wed, 26 Jun 2019 14:42:50 +0200 Subject: [PATCH 12/12] Fix copyright year --- attachment_s3/__openerp__.py | 2 +- cloud_platform/__openerp__.py | 2 +- logging_json/__openerp__.py | 2 +- monitoring_status/__openerp__.py | 2 +- session_redis/__openerp__.py | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/attachment_s3/__openerp__.py b/attachment_s3/__openerp__.py index bccf74e..dd072dd 100644 --- a/attachment_s3/__openerp__.py +++ b/attachment_s3/__openerp__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2016 Camptocamp SA +# Copyright 2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) diff --git a/cloud_platform/__openerp__.py b/cloud_platform/__openerp__.py index b9ebbfc..8119376 100644 --- a/cloud_platform/__openerp__.py +++ b/cloud_platform/__openerp__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2016 Camptocamp SA +# Copyright 2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) diff --git a/logging_json/__openerp__.py b/logging_json/__openerp__.py index 5ad22c5..a6fb581 100644 --- a/logging_json/__openerp__.py +++ b/logging_json/__openerp__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2016 Camptocamp SA +# Copyright 2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) {'name': 'JSON Logging', diff --git a/monitoring_status/__openerp__.py b/monitoring_status/__openerp__.py index 392abb6..7f6f5d1 100644 --- a/monitoring_status/__openerp__.py +++ b/monitoring_status/__openerp__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2016 Camptocamp SA +# Copyright 2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) diff --git a/session_redis/__openerp__.py b/session_redis/__openerp__.py index eb040dc..1231894 100644 --- a/session_redis/__openerp__.py +++ b/session_redis/__openerp__.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -# Copyright 2016 Camptocamp SA +# Copyright 2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)