Merge pull request #40 from camptocamp/add_ovh_in_v9

[9.0] Backport OVH modules from 10.0

attachment_swift/README.rst

@@ -0,0 +1,51 @@
+Attachments on Swift storage
+============================
+
+This addon enable storing attachments (documents and assets) on OpenStack Object Storage (Swift)
+
+Configuration
+-------------
+
+Activate Swift storage:
+
+* Create or set the system parameter with the key ``ir_attachment.location`` with the following value ``swift``.
+
+Configure accesses with environment variables:
+
+* ``SWIFT_AUTH_URL``            : URL of the Swift server
+* ``SWIFT_TENANT_NAME``
+* ``SWIFT_ACCOUNT``
+* ``SWIFT_PASSWORD``
+* ``SWIFT_WRITE_CONTAINER``     : Name of the container to use in the store (created if not existing)
+
+Read-only mode:
+
+The container name and the key are stored in the attachment. So if you change the
+``SWIFT_WRITE_CONTAINER`` or the ``ir_attachment.location``, the existing attachments
+will still be read on their former container. But as soon as they are written over
+or new attachments are created, they will be created on the new container or on
+the other location (db or filesystem). This is a convenient way to be able to
+read the production attachments on a replication (since you have the
+credentials) without any risk to alter the production data.
+
+This addon must be added in the server wide addons with (``--load`` option):
+
+``--load=web,web_kanban,attachment_swift``
+
+Python Dependencies
+-------------------
+
+This module needs the python-swiftclient and the python-keystoneclient (For auth v2.0) to work.
+The python-keystoneclient needs the linux package build-essential and python-dev to install properly.
+
+The python-swiftclient can be used from the command line, useful to test:
+
+    export AUTH_VERSION=2.0
+    export OS_USERNAME={SWIFT_ACCOUNT}
+    export OS_PASSWORD={SWIFT_PASSWORD}
+    export OS_TENANT_NAME={SWIFT_TENANT_NAME}
+    export OS_AUTH_URL=https://auth.cloud.ovh.net/v2.0
+    swift stat
+
+More information at
+https://docs.openstack.org/python-swiftclient/latest/cli/index.html#swift-usage

attachment_swift/__init__.py

@@ -0,0 +1 @@
+from . import models

attachment_swift/__openerp__.py

@@ -0,0 +1,21 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{'name': 'Attachments on Swift storage',
+ 'summary': 'Store assets and attachments on a Swift compatible object store',
+ 'version': '9.0.1.1.0',
+ 'author': 'Camptocamp,Odoo Community Association (OCA)',
+ 'license': 'AGPL-3',
+ 'category': 'Knowledge Management',
+ 'depends': ['base_attachment_object_storage'],
+ 'external_dependencies': {
+     'python': ['swiftclient',
+                'keystoneclient',
+                ],
+ },
+ 'website': 'https://www.camptocamp.com',
+ 'data': [],
+ 'installable': True,
+ }

attachment_swift/models/__init__.py

@@ -0,0 +1 @@
+from . import ir_attachment

attachment_swift/models/ir_attachment.py

@@ -0,0 +1,118 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+import base64
+import logging
+import os
+from ..swift_uri import SwiftUri
+
+from openerp import api, exceptions, models, _
+
+_logger = logging.getLogger(__name__)
+
+try:
+    import swiftclient
+    from swiftclient.exceptions import ClientException
+except ImportError:
+    swiftclient = None
+    ClientException = None
+    _logger.debug("Cannot 'import swiftclient'.")
+
+
+class IrAttachment(models.Model):
+    _inherit = 'ir.attachment'
+
+    def _get_stores(self):
+        l = ['swift']
+        l += super(IrAttachment, self)._get_stores()
+        return l
+
+    @api.model
+    def _get_swift_connection(self):
+        """ Returns a connection object for the Swift object store """
+        host = os.environ.get('SWIFT_AUTH_URL')
+        account = os.environ.get('SWIFT_ACCOUNT')
+        password = os.environ.get('SWIFT_PASSWORD')
+        tenant_name = os.environ.get('SWIFT_TENANT_NAME')
+        region = os.environ.get('SWIFT_REGION_NAME')
+        os_options = {}
+        if region:
+            os_options['region_name'] = region
+        if not (host and account and password and tenant_name):
+            raise exceptions.UserError(_(
+                "Problem connecting to Swift store, are the env variables "
+                "(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, "
+                "SWIFT_TENANT_NAME) properly set?"
+                ))
+        try:
+            conn = swiftclient.client.Connection(authurl=host,
+                                                 user=account,
+                                                 key=password,
+                                                 tenant_name=tenant_name,
+                                                 auth_version='2.0',
+                                                 os_options=os_options,
+                                                 )
+        except ClientException:
+            _logger.exception('Error connecting to Swift object store')
+            raise exceptions.UserError(_('Error on Swift connection'))
+        return conn
+
+    @api.model
+    def _store_file_read(self, fname, bin_size=False):
+        if fname.startswith('swift://'):
+            swifturi = SwiftUri(fname)
+            try:
+                conn = self._get_swift_connection()
+            except exceptions.UserError:
+                _logger.exception(
+                    "error reading attachment '%s' from object storage", fname
+                )
+                return ''
+            try:
+                resp, obj_content = conn.get_object(swifturi.container(),
+                                                    swifturi.item())
+                read = base64.b64encode(obj_content)
+            except ClientException:
+                read = ''
+                _logger.exception(
+                    'Error reading object from Swift object store')
+            return read
+        else:
+            return super(IrAttachment, self)._store_file_read(fname, bin_size)
+
+    def _store_file_write(self, key, bin_data):
+        if self._storage() == 'swift':
+            container = os.environ.get('SWIFT_WRITE_CONTAINER')
+            conn = self._get_swift_connection()
+            conn.put_container(container)
+            filename = 'swift://{}/{}'.format(container, key)
+            try:
+                conn.put_object(container, key, bin_data)
+            except ClientException:
+                _logger.exception('Error writing to Swift object store')
+                raise exceptions.UserError(_('Error writing to Swift'))
+        else:
+            _super = super(IrAttachment, self)
+            filename = _super._store_file_write(key, bin_data)
+        return filename
+
+    @api.model
+    def _store_file_delete(self, fname):
+        if fname.startswith('swift://'):
+            swifturi = SwiftUri(fname)
+            container = swifturi.container()
+            # delete the file only if it is on the current configured bucket
+            # otherwise, we might delete files used on a different environment
+            if container == os.environ.get('SWIFT_WRITE_CONTAINER'):
+                conn = self._get_swift_connection()
+                try:
+                    conn.delete_object(container, swifturi.item())
+                except ClientException:
+                    _logger.exception(
+                        _('Error deleting an object on the Swift store'))
+                    # we ignore the error, file will stay on the object
+                    # storage but won't disrupt the process
+        else:
+            super(IrAttachment, self)._file_delete_from_store(fname)

attachment_swift/swift_uri.py

@@ -0,0 +1,23 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import re
+
+
+class SwiftUri(object):
+
+    _url_re = re.compile("^swift:///*([^/]*)/?(.*)",
+                         re.IGNORECASE | re.UNICODE)
+
+    def __init__(self, uri):
+        match = self._url_re.match(uri)
+        if not match:
+            raise ValueError("%s: is not a valid Swift URI" % (uri,))
+        self._container, self._item = match.groups()
+
+    def container(self):
+        return self._container
+
+    def item(self):
+        return self._item

attachment_swift/tests/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+from . import test_mock_swift_api

attachment_swift/tests/test_mock_swift_api.py

@@ -0,0 +1,79 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import os
+
+from mock import patch
+from openerp.addons.base.tests.test_ir_attachment import TestIrAttachment
+from ..swift_uri import SwiftUri
+
+
+class TestAttachmentSwift(TestIrAttachment):
+
+    def setup(self):
+        super(TestAttachmentSwift, self).setUp()
+        self.env['ir.config_parameter'].set_param('ir_attachment.location',
+                                                  'swift')
+
+    @patch('swiftclient.client')
+    def test_connection(self, mock_swift_client):
+        """ Test the connection to the store"""
+        os.environ['SWIFT_AUTH_URL'] = 'auth_url'
+        os.environ['SWIFT_ACCOUNT'] = 'account'
+        os.environ['SWIFT_PASSWORD'] = 'password'
+        os.environ['SWIFT_TENANT_NAME'] = 'tenant_name'
+        os.environ['SWIFT_REGION_NAME'] = 'NOWHERE'
+        attachment = self.Attachment
+        attachment._get_swift_connection()
+        mock_swift_client.Connection.assert_called_once_with(
+            authurl=os.environ.get('SWIFT_AUTH_URL'),
+            user=os.environ.get('SWIFT_ACCOUNT'),
+            key=os.environ.get('SWIFT_PASSWORD'),
+            tenant_name=os.environ.get('SWIFT_TENANT_NAME'),
+            auth_version='2.0',
+            os_options={'region_name': os.environ.get('SWIFT_REGION_NAME')},
+        )
+
+    def test_store_file_on_swift(self):
+        """
+            Test writing a file
+        """
+        (self.env['ir.config_parameter'].
+            set_param('ir_attachment.location', 'swift'))
+        os.environ['SWIFT_AUTH_URL'] = 'auth_url'
+        os.environ['SWIFT_ACCOUNT'] = 'account'
+        os.environ['SWIFT_PASSWORD'] = 'password'
+        os.environ['SWIFT_TENANT_NAME'] = 'tenant_name'
+        os.environ['SWIFT_WRITE_CONTAINER'] = 'my_container'
+        container = os.environ.get('SWIFT_WRITE_CONTAINER')
+        attachment = self.Attachment
+        bin_data = self.blob1_b64.decode('base64')
+        with patch('swiftclient.client.Connection') as MockConnection:
+            conn = MockConnection.return_value
+            attachment.create({'name': 'a5', 'datas': self.blob1_b64})
+            conn.put_object.assert_called_with(
+                container,
+                attachment._compute_checksum(bin_data),
+                bin_data)
+
+    def test_delete_file_on_swift(self):
+        """
+            Test deleting a file
+        """
+        (self.env['ir.config_parameter'].
+            set_param('ir_attachment.location', 'swift'))
+        os.environ['SWIFT_AUTH_URL'] = 'auth_url'
+        os.environ['SWIFT_ACCOUNT'] = 'account'
+        os.environ['SWIFT_PASSWORD'] = 'password'
+        os.environ['SWIFT_TENANT_NAME'] = 'tenant_name'
+        os.environ['SWIFT_WRITE_CONTAINER'] = 'my_container'
+
+        attachment = self.Attachment
+        container = os.environ.get('SWIFT_WRITE_CONTAINER')
+        with patch('swiftclient.client.Connection') as MockConnection:
+            conn = MockConnection.return_value
+            a5 = attachment.create({'name': 'a5', 'datas': self.blob1_b64})
+            uri = SwiftUri(a5.store_fname)
+            a5.unlink()
+            conn.delete_object.assert_called_with(container, uri.item())

attachment_swift/tests/test_with_swift_store.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+from openerp.addons.base.tests.test_ir_attachment import TestIrAttachment
+from ..swift_uri import SwiftUri
+from swiftclient.exceptions import ClientException
+
+
+class TestAttachmentSwift(TestIrAttachment):
+    """
+    Those tests are made to be run against a real Swift store (local or remote)
+    """
+
+    def setup(self):
+        super(TestAttachmentSwift, self).setUp()
+        self.env['ir.config_parameter'].set_param('ir_attachment.location',
+                                                  'swift')
+
+    def test_connection(self):
+        """ Test the connection to the Swift object store """
+        conn = self.Attachment._get_swift_connection()
+        self.assertNotEquals(conn, False)
+
+    def test_store_file_on_swift(self):
+        """ Test writing a file and then reading it """
+        (self.env['ir.config_parameter'].
+            set_param('ir_attachment.location', 'swift'))
+        a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64})
+        a5bis = self.Attachment.browse(a5.id)[0]
+        self.assertEquals(a5.datas, a5bis.datas)
+
+    def test_delete_file_on_swift(self):
+        """ Create a file and then test the deletion """
+        (self.env['ir.config_parameter'].
+            set_param('ir_attachment.location', 'swift'))
+        a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64})
+        uri = SwiftUri(a5.store_fname)
+        con = self.Attachment._get_swift_connection()
+        con.get_object(uri.container(), uri.item())
+        a5.unlink()
+        with self.assertRaises(ClientException):
+            con.get_object(uri.container(), uri.item())

base_attachment_object_storage/README.rst

@@ -0,0 +1,7 @@
+Base class for attachments on external object store
+===================================================
+
+This is a base addon that regroup common code used by addons targeting specific object store 
+
+
+

base_attachment_object_storage/__init__.py

@@ -0,0 +1 @@
+from . import models

base_attachment_object_storage/__openerp__.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{'name': 'Base Attachment Object Store',
+ 'summary': 'Base module for the implementation of external object store.',
+ 'version': '9.0.1.2.0',
+ 'author': 'Camptocamp,Odoo Community Association (OCA)',
+ 'license': 'AGPL-3',
+ 'category': 'Knowledge Management',
+ 'depends': ['base'],
+ 'website': 'http://www.camptocamp.com',
+ 'data': [],
+ 'installable': True,
+ 'auto_install': True,
+ }

base_attachment_object_storage/models/__init__.py

@@ -0,0 +1 @@
+from . import ir_attachment

base_attachment_object_storage/models/ir_attachment.py

@@ -0,0 +1,304 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import inspect
+import logging
+import os
+import psycopg2
+import odoo
+
+from contextlib import closing, contextmanager
+from openerp import api, exceptions, models, _
+from openerp import SUPERUSER_ID
+
+
+_logger = logging.getLogger(__name__)
+
+
+def clean_fs(files):
+    _logger.info('cleaning old files from filestore')
+    for full_path in files:
+        if os.path.exists(full_path):
+            try:
+                os.unlink(full_path)
+            except OSError:
+                _logger.info(
+                    "_file_delete could not unlink %s",
+                    full_path, exc_info=True
+                )
+            except IOError:
+                # Harmless and needed for race conditions
+                _logger.info(
+                    "_file_delete could not unlink %s",
+                    full_path, exc_info=True
+                )
+
+
+class IrAttachment(models.Model):
+    _inherit = 'ir.attachment'
+
+    _local_fields = ('image_small', 'image_medium', 'web_icon_data')
+
+    @api.cr
+    def _register_hook(self, cr):
+        super(IrAttachment, self)._register_hook(cr)
+        # ignore if we are not using an object storage
+        # Use directly SUPERUSER_ID
+        # because the uid parameter is required
+        # in function _storage and
+        # the SUPERUSER_ID is used directly instead of use the uid parameter.
+        if self._storage(cr, SUPERUSER_ID) not in self._get_stores():
+            return
+        curframe = inspect.currentframe()
+        calframe = inspect.getouterframes(curframe, 2)
+        # the caller of _register_hook is 'load_modules' in
+        # odoo/modules/loading.py
+        load_modules_frame = calframe[1][0]
+        # 'update_module' is an argument that 'load_modules' receives with a
+        # True-ish value meaning that an install or upgrade of addon has been
+        # done during the initialization. We need to move the attachments that
+        # could have been created or updated in other addons before this addon
+        # was loaded
+        update_module = load_modules_frame.f_locals.get('update_module')
+
+        # We need to call the migration on the loading of the model because
+        # when we are upgrading addons, some of them might add attachments.
+        # To be sure they are migrated to the storage we need to call the
+        # migration here.
+        # Typical example is images of ir.ui.menu which are updated in
+        # ir.attachment at every upgrade of the addons
+        if update_module:
+            self.env['ir.attachment'].sudo()._force_storage_to_object_storage()
+
+    @api.multi
+    def _save_in_db_anyway(self):
+        """ Return whether an attachment must be stored in db
+
+        When we are using an Object Store. This is sometimes required
+        because the object storage is slower than the database/filesystem.
+
+        We store image_small and image_medium from 'Binary' fields
+        because they should be fast to read as they are often displayed
+        in kanbans / lists. The same for web_icon_data.
+
+        We store the assets locally as well. Not only for performance,
+        but also because it improves the portability of the database:
+        when assets are invalidated, they are deleted so we don't have
+        an old database with attachments pointing to deleted assets.
+
+        """
+        self.ensure_one()
+        # assets
+        if self.res_model == 'ir.ui.view':
+            # assets are stored in 'ir.ui.view'
+            return True
+
+        # Binary fields
+        if self.res_field:
+            # Binary fields are stored with the name of the field in
+            # 'res_field'
+            # 'image' fields can be rather large and should usually
+            # not be requests in bulk in lists
+            if self.res_field and self.res_field in self._local_fields:
+                return True
+        return False
+
+    def _inverse_datas(self):
+        # override in order to store files that need fast access,
+        # we keep them in the database instead of the object storage
+        location = self._storage()
+        for attach in self:
+            if location in self._get_stores() and attach._save_in_db_anyway():
+                # compute the fields that depend on datas
+                value = attach.datas
+                bin_data = value and value.decode('base64') or ''
+                vals = {
+                    'file_size': len(bin_data),
+                    'checksum': self._compute_checksum(bin_data),
+                    'db_datas': value,
+                    # we seriously don't need index content on those fields
+                    'index_content': False,
+                    'store_fname': False,
+                }
+                fname = attach.store_fname
+                # write as superuser, as user probably does not
+                # have write access
+                super(IrAttachment, attach.sudo()).write(vals)
+                if fname:
+                    self._file_delete(fname)
+                continue
+            super(IrAttachment, attach)._inverse_datas()
+
+    @api.model
+    def _file_read(self, fname, bin_size=False):
+        if self._is_file_from_a_store(fname):
+            return self._store_file_read(fname, bin_size=bin_size)
+        else:
+            _super = super(IrAttachment, self)
+            return _super._file_read(fname, bin_size=bin_size)
+
+    def _store_file_read(self, fname, bin_size=False):
+        storage = fname.partition('://')[0]
+        raise NotImplementedError(
+            'No implementation for %s' % (storage,)
+        )
+
+    def _store_file_write(self, key, bin_data):
+        raise NotImplementedError(
+            'No implementation for %s' % (self.storage(),)
+        )
+
+    def _store_file_delete(self, fname):
+        storage = fname.partition('://')[0]
+        raise NotImplementedError(
+            'No implementation for %s' % (storage,)
+        )
+
+    @api.model
+    def _file_write(self, value, checksum):
+        if self._storage() in self._get_stores():
+            bin_data = value.decode('base64')
+            key = self._compute_checksum(bin_data)
+            filename = self._store_file_write(key, bin_data)
+        else:
+            filename = super(IrAttachment, self)._file_write(value, checksum)
+        return filename
+
+    @api.model
+    def _file_delete(self, fname):
+        if self._is_file_from_a_store(fname):
+            cr = self.env.cr
+            # using SQL to include files hidden through unlink or due to record
+            # rules
+            cr.execute("SELECT COUNT(*) FROM ir_attachment "
+                       "WHERE store_fname = %s", (fname,))
+            count = cr.fetchone()[0]
+            if not count:
+                self._store_file_delete(fname)
+        else:
+            super(IrAttachment, self)._file_delete(fname)
+
+    @api.model
+    def _is_file_from_a_store(self, fname):
+        for store_name in self._get_stores():
+            uri = '{}://'.format(store_name)
+            if fname.startswith(uri):
+                return True
+        return False
+
+    @contextmanager
+    def do_in_new_env(self, new_cr=False):
+        """ Context manager that yields a new environment
+
+        Using a new Odoo Environment thus a new PG transaction.
+        """
+        with api.Environment.manage():
+            if new_cr:
+                registry = odoo.modules.registry.RegistryManager.get(
+                    self.env.cr.dbname
+                )
+                with closing(registry.cursor()) as cr:
+                    try:
+                        yield self.env(cr=cr)
+                    except:
+                        cr.rollback()
+                        raise
+                    else:
+                        # disable pylint error because this is a valid commit,
+                        # we are in a new env
+                        cr.commit()  # pylint: disable=invalid-commit
+            else:
+                # make a copy
+                yield self.env()
+
+    @api.multi
+    def _move_attachment_to_store(self):
+        self.ensure_one()
+        _logger.info('inspecting attachment %s (%d)', self.name, self.id)
+        fname = self.store_fname
+        if fname:
+            # migrating from filesystem filestore
+            # or from the old 'store_fname' without the bucket name
+            _logger.info('moving %s on the object storage', fname)
+            self.write({'datas': self.datas,
+                        # this is required otherwise the
+                        # mimetype gets overriden with
+                        # 'application/octet-stream'
+                        # on assets
+                        'mimetype': self.mimetype})
+            _logger.info('moved %s on the object storage', fname)
+            return self._full_path(fname)
+        elif self.db_datas:
+            _logger.info('moving on the object storage from database')
+            self.write({'datas': self.datas})
+
+    @api.model
+    def force_storage(self):
+        if not self.env['res.users'].browse(self.env.uid)._is_admin():
+            raise exceptions.AccessError(
+                _('Only administrators can execute this action.'))
+        storage = self._storage()
+        if storage not in self._get_stores():
+            return super(IrAttachment, self).force_storage()
+        self._force_storage_to_object_storage()
+
+    @api.model
+    def _force_storage_to_object_storage(self, new_cr=False):
+        _logger.info('migrating files to the object storage')
+        storage = self._storage()
+        # The weird "res_field = False OR res_field != False" domain
+        # is required! It's because of an override of _search in ir.attachment
+        # which adds ('res_field', '=', False) when the domain does not
+        # contain 'res_field'.
+        # https://github.com/odoo/odoo/blob/9032617120138848c63b3cfa5d1913c5e5ad76db/odoo/addons/base/ir/ir_attachment.py#L344-L347
+        domain = ['!', ('store_fname', '=like', '{}://%'.format(storage)),
+                  '|',
+                  ('res_field', '=', False),
+                  ('res_field', '!=', False)]
+        # We do a copy of the environment so we can workaround the cache issue
+        # below. We do not create a new cursor by default because it causes
+        # serialization issues due to concurrent updates on attachments during
+        # the installation
+        with self.do_in_new_env(new_cr=new_cr) as new_env:
+            model_env = new_env['ir.attachment']
+            ids = model_env.search(domain).ids
+            files_to_clean = []
+            for attachment_id in ids:
+                try:
+                    with new_env.cr.savepoint():
+                        # check that no other transaction has
+                        # locked the row, don't send a file to storage
+                        # in that case
+                        self.env.cr.execute("SELECT id "
+                                            "FROM ir_attachment "
+                                            "WHERE id = %s "
+                                            "FOR UPDATE NOWAIT",
+                                            (attachment_id,),
+                                            log_exceptions=False)
+
+                        # This is a trick to avoid having the 'datas'
+                        # function fields computed for every attachment on
+                        # each iteration of the loop. The former issue
+                        # being that it reads the content of the file of
+                        # ALL the attachments on each loop.
+                        new_env.clear()
+                        attachment = model_env.browse(attachment_id)
+                        path = attachment._move_attachment_to_store()
+                        if path:
+                            files_to_clean.append(path)
+                except psycopg2.OperationalError:
+                    _logger.error('Could not migrate attachment %s to S3',
+                                  attachment_id)
+
+            def clean():
+                clean_fs(files_to_clean)
+
+            # delete the files from the filesystem once we know the changes
+            # have been committed in ir.attachment
+            if files_to_clean:
+                new_env.cr.after('commit', clean)
+
+    def _get_stores(self):
+        """ To get the list of stores activated in the system  """
+        return []

cloud_platform/__openerp__.py

@@ -10,7 +10,7 @@
  'license': 'AGPL-3',
  'category': 'Extra Tools',
  'depends': [
-     'attachment_s3',
+     'base_attachment_object_storage',
      'session_redis',
      'monitoring_status',
      'logging_json',

cloud_platform/models/cloud_platform.py

@@ -30,6 +30,7 @@ PlatformConfig = namedtuple(
 class FilestoreKind(object):
     db = 'db'
     s3 = 's3'  # or compatible s3 object storage
+    swift = 'swift'
     file = 'file'
 
 
@@ -37,26 +38,104 @@ class CloudPlatform(models.AbstractModel):
     _name = 'cloud.platform'
 
     @api.model
-    def _config_by_server_env(self, environment):
+    def _platform_kinds(self):
+        # XXX for backward compatibility, we need this one here, move
+        # it in cloud_platform_exoscale in V11
+        return ['exoscale']
+
+    # XXX for backward compatibility, we need this one here, move
+    # it in cloud_platform_exoscale in V11
+    @api.model
+    def _config_by_server_env_for_exoscale(self):
         configs = {
             'prod': PlatformConfig(filestore=FilestoreKind.s3),
             'integration': PlatformConfig(filestore=FilestoreKind.s3),
             'test': PlatformConfig(filestore=FilestoreKind.db),
             'dev': PlatformConfig(filestore=FilestoreKind.db),
         }
-        return configs.get(environment) or configs['dev']
+        return configs
 
+    @api.model
+    def _config_by_server_env(self, platform_kind, environment):
+        configs_getter = getattr(
+            self,
+            '_config_by_server_env_for_%s' % platform_kind,
+            None
+        )
+        configs = configs_getter() if configs_getter else {}
+        return configs.get(environment) or FilestoreKind.db
+
+    # Due to the addition of the ovh cloud platform
+    # This will be moved to cloud_platform_exoscale on v11
     @api.model
     def install_exoscale(self):
+        self.install('exoscale')
+
+    @api.model
+    def install(self, platform_kind):
+        assert platform_kind in self._platform_kinds()
         params = self.env['ir.config_parameter'].sudo()
-        params.set_param('cloud.platform.kind', 'exoscale')
+        params.set_param('cloud.platform.kind', platform_kind)
         environment = config['running_env']
-        configs = self._config_by_server_env(environment)
+        configs = self._config_by_server_env(platform_kind, environment)
         params.set_param('ir_attachment.location', configs.filestore)
         self.check()
-        if configs.filestore == FilestoreKind.s3:
+        if configs.filestore in [FilestoreKind.swift, FilestoreKind.s3]:
             self.env['ir.attachment'].sudo().force_storage()
-        _logger.info('cloud platform configured for exoscale')
+        _logger.info('cloud platform configured for {}'.format(platform_kind))
+
+    @api.model
+    def _check_swift(self, environment_name):
+        params = self.env['ir.config_parameter'].sudo()
+        use_swift = (params.get_param('ir_attachment.location') ==
+                     FilestoreKind.swift)
+        if environment_name in ('prod', 'integration'):
+            assert use_swift, (
+                "Swift must be used on production and integration instances. "
+                "It is activated, setting 'ir_attachment.location.' to 'swift'"
+                " The 'install_exoscale()' function sets this option "
+                "automatically."
+            )
+        if use_swift:
+            assert os.environ.get('SWIFT_AUTH_URL'), (
+                "SWIFT_AUTH_URL environment variable is required when "
+                "ir_attachment.location is 'swift'."
+            )
+            assert os.environ.get('SWIFT_ACCOUNT'), (
+                "SWIFT_ACCOUNT environment variable is required when "
+                "ir_attachment.location is 'swift'."
+            )
+            assert os.environ.get('SWIFT_PASSWORD'), (
+                "SWIFT_PASSWORD environment variable is required when "
+                "ir_attachment.location is 'swift'."
+            )
+            container_name = os.environ['SWIFT_WRITE_CONTAINER']
+            if environment_name in ('integration', 'prod'):
+                assert container_name, (
+                    "SWIFT_WRITE_CONTAINER must not be empty for prod "
+                    "and integration"
+                )
+            prod_container = bool(re.match(r'[a-z]+-odoo-prod',
+                                           container_name))
+            if environment_name == 'prod':
+                assert prod_container, (
+                    "SWIFT_WRITE_CONTAINER should match '<client>-odoo-prod', "
+                    "we got: '%s'" % (container_name,)
+                )
+            else:
+                # if we are using the prod bucket on another instance
+                # such as an integration, we must be sure to be in read only!
+                assert not prod_container, (
+                    "SWIFT_WRITE_CONTAINER should not match "
+                    "'<client>-odoo-prod', we got: '%s'" % (container_name,)
+                )
+        elif environment_name == 'test':
+            # store in DB so we don't have files local to the host
+            assert params.get_param('ir_attachment.location') == 'db', (
+                "In test instances, files must be stored in the database with "
+                "'ir_attachment.location' set to 'db'. This is "
+                "automatically set by the function 'install_ovh()'."
+            )
 
     @api.model
     def _check_s3(self, environment_name):
@@ -148,7 +227,10 @@ class CloudPlatform(models.AbstractModel):
             )
             return
         environment_name = config['running_env']
+        if kind == 'exoscale':
             self._check_s3(environment_name)
+        elif kind == 'ovh':
+            self._check_swift(environment_name)
         self._check_redis(environment_name)
 
     @api.cr

cloud_platform/songs.py

@@ -3,3 +3,7 @@
 
 def install_exoscale(ctx):
     ctx.env['cloud.platform'].install_exoscale()
+
+
+def install_ovh(ctx):
+    ctx.env['cloud.platform'].install('ovh')

cloud_platform_ovh/README.md

@@ -0,0 +1,7 @@
+Cloud Platform OVH
+==================
+
+Install addons specific to the OVH setup.
+
+    * The object storage is Swift
+

cloud_platform_ovh/__init__.py

@@ -0,0 +1 @@
+from . import models

cloud_platform_ovh/__openerp__.py

@@ -0,0 +1,19 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{'name': 'Cloud Platform OVH',
+ 'summary': 'Addons required for the Camptocamp Cloud Platform on OVH',
+ 'version': '9.0.1.1.0',
+ 'author': 'Camptocamp,Odoo Community Association (OCA)',
+ 'license': 'AGPL-3',
+ 'category': 'Extra Tools',
+ 'depends': [
+     'cloud_platform',
+     'attachment_swift',
+ ],
+ 'website': 'https://www.camptocamp.com',
+ 'data': [],
+ 'installable': True,
+ }

cloud_platform_ovh/models/__init__.py

@@ -0,0 +1 @@
+from . import cloud_platform

cloud_platform_ovh/models/cloud_platform.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+# Copyright 2017 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import logging
+
+from openerp import api, models
+
+_logger = logging.getLogger(__name__)
+
+try:
+    from openerp.addons.cloud_platform.models.cloud_platform \
+        import FilestoreKind
+    from openerp.addons.cloud_platform.models.cloud_platform \
+        import PlatformConfig
+except ImportError:
+    FilestoreKind = None
+    PlatformConfig = None
+    _logger.debug("Cannot 'import from cloud_platform'")
+
+
+class CloudPlatform(models.AbstractModel):
+    _inherit = 'cloud.platform'
+
+    @api.model
+    def _platform_kinds(self):
+        kinds = super(CloudPlatform, self)._platform_kinds()
+        kinds.append('ovh')
+        return kinds
+
+    @api.model
+    def _config_by_server_env_for_ovh(self):
+        configs = {
+            'prod': PlatformConfig(filestore=FilestoreKind.swift),
+            'integration': PlatformConfig(filestore=FilestoreKind.swift),
+            'test': PlatformConfig(filestore=FilestoreKind.db),
+            'dev': PlatformConfig(filestore=FilestoreKind.db),
+        }
+        return configs
+
+    @api.model
+    def install_ovh(self):
+        self.install('ovh')

requirements.txt

@@ -2,3 +2,5 @@ boto==2.42.0
 redis==2.10.5
 python-json-logger==0.1.5
 statsd==3.2.1
+python-swiftclient==3.4.0
+python-keystoneclient==3.13.0

session_redis/__init__.py

@@ -2,3 +2,4 @@
 
 from . import http
 from . import session
+from . import models

session_redis/models/__init__.py

@@ -0,0 +1 @@
+from . import user

session_redis/models/user.py

@@ -0,0 +1,19 @@
+from openerp import models, tools
+
+
+class User(models.Model):
+    _inherit = 'res.users'
+
+    @tools.ormcache('sid')
+    def _compute_session_token(self, sid):
+        """Make sure to return an unicode string.
+
+        Odoo creates a session token using hexdigest Session which is str
+        but with redis we set the token from a dictionary of values passing
+        it in json format. When dumping values from json, we always get unicode
+        thus both are incompatible.
+
+        The shortest path is to fix the output of the computed session by Odoo.
+
+        """
+        return unicode(super(User, self)._compute_session_token(sid))