From beea07d44fbc9c76232163d9c636472c3aba0825 Mon Sep 17 00:00:00 2001 From: Thierry Ducrest Date: Mon, 28 Aug 2017 16:08:07 +0200 Subject: [PATCH] Add implementation of reading writing on the Swift object store --- attachment_swift/models/ir_attachment.py | 105 ++++++++++++++++++++++- attachment_swift/swift_uri.py | 22 +++++ attachment_swift/tests/__init__.py | 3 + attachment_swift/tests/tests.py | 33 +++++++ 4 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 attachment_swift/swift_uri.py create mode 100644 attachment_swift/tests/__init__.py create mode 100644 attachment_swift/tests/tests.py diff --git a/attachment_swift/models/ir_attachment.py b/attachment_swift/models/ir_attachment.py index f5bd7da..ec03a09 100644 --- a/attachment_swift/models/ir_attachment.py +++ b/attachment_swift/models/ir_attachment.py @@ -3,8 +3,14 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +import base64 import logging -from odoo import api, models +import os +import swiftclient +from swiftclient.exceptions import ClientException +from ..swift_uri import SwiftUri + +from odoo import api, exceptions, models, _ _logger = logging.getLogger(__name__) @@ -12,8 +18,99 @@ _logger = logging.getLogger(__name__) class IrAttachment(models.Model): _inherit = 'ir.attachment' + @api.multi + def _store_in_db_when_swift(self): + # For testing lets save everything in Swift object store + # TODO: same than in attachment_s3 + return False + @api.model - def _file_write(self, value, checksum): - _logger.debug('Writing a file :)') - filename = super(IrAttachment, self)._file_write(value, checksum) + def _get_swift_connection(self): + """ Returns a connection object for the Swift object store """ + host = os.environ.get('SWIFT_HOST') + account = os.environ.get('SWIFT_ACCOUNT') + password = os.environ.get('SWIFT_PASSWORD') + if not (host and account and password): + raise exceptions.UserError( + _('Problem connecting to Swift store, are not the env variables set ?')) + print 'Connection to host: {}, account: {}, password: {}'.format(host, account, password) + try: + conn = swiftclient.client.Connection(authurl=host, user=account, key=password) + except ClientException: + _logger.exception('Error connecting to Swift object store') + raise exceptions.UserError('Error connection to Swift') + return conn + + @api.model + def _file_read_swift(self, fname, bin_size=False): + swifturi = SwiftUri(fname) + conn = self._get_swift_connection() + print 'Swift reading on {} of {} '.format(swifturi.container(), swifturi.item()) + try: + resp_headers, obj_content = conn.get_object(swifturi.container(), swifturi.item()) + read = base64.b64encode(obj_content) + except ClientException: + _logger.exception('Error reading object from Swift object store'); + #raise exceptions.UserError('Error reading to Swift') + return '' + return read + + @api.model + def _file_read(self, fname, bin_size=False): + if fname.startswith('swift://'): + return self._file_read_swift(fname, bin_size=bin_size) + else: + _super = super(IrAttachment, self) + return _super._file_read(fname, bin_size=bin_size) + + def _file_write_swift(self, value, checksum): + container = os.environ.get('SWIFT_WRITE_CONTAINER') + conn = self._get_swift_connection() + conn.put_container(container) + bin_data = value.decode('base64') + # No keys given by the store, use checksum !? + key = self._compute_checksum(bin_data) + filename = 'swift://{}/{}'.format(container, key) + print 'Saving {}'.format(filename) + try: + conn.put_object(container, key, bin_data) + except ClientException: + _logger.exception('Error connecting to Swift object store') + raise exceptions.UserError('Error writting to Swift') return filename + + def _file_write(self, value, checksum): + storage = self._storage() + if storage == 'swift': + filename = self._file_write_swift(value, checksum) + else: + filename = super(IrAttachment, self)._file_write(value, checksum) + return filename + + @api.model + def _file_delete(self, fname): + if fname.startswith('swift://'): + swifturi = SwiftUri(fname) + container = swifturi.container() + print 'Deleting... container: {} | filename: {}'.format(container, swifturi.item()) + if container == os.environ.get('SWIFT_WRITE_CONTAINER'): + conn = self._get_swift_connection() + try: + conn.delete_object(container, swifturi.item()) + except ClientException as error: + _logger.exception('Error connecting to Swift object store'); + raise exceptions.UserError('Error deleting in Swift') + else: + super(IrAttachment, self)._file_delete(fname) + + @api.model + def _force_storage_swift(self, new_cr=False): + return + + @api.model + def force_storage(self): + storage = self._storage() + if storage == 'swift': + self._force_storage_swift() + else: + return super(IrAttachment, self).force_storage() diff --git a/attachment_swift/swift_uri.py b/attachment_swift/swift_uri.py new file mode 100644 index 0000000..59569ab --- /dev/null +++ b/attachment_swift/swift_uri.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import re + + +class SwiftUri(object): + + _url_re = re.compile("^swift:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE) + + def __init__(self, uri): + match = self._url_re.match(uri) + if not match: + raise ValueError("%s: is not a valid Swift URI" % (uri,)) + self._container, self._item = match.groups() + + def container(self): + return self._container + + def item(self): + return self._item diff --git a/attachment_swift/tests/__init__.py b/attachment_swift/tests/__init__.py new file mode 100644 index 0000000..d8d07b2 --- /dev/null +++ b/attachment_swift/tests/__init__.py @@ -0,0 +1,3 @@ +# -*- coding: utf-8 -*- + +from . import tests diff --git a/attachment_swift/tests/tests.py b/attachment_swift/tests/tests.py new file mode 100644 index 0000000..8cc73fc --- /dev/null +++ b/attachment_swift/tests/tests.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- + +import unittest +from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment +from ..models import ir_attachment +from ..swift_uri import SwiftUri +from swiftclient.exceptions import ClientException + + +class TestAttachmentSwift(TestIrAttachment): + + def setup(self): + super(TestAttachmentSwift, self).setUp() + self.env['ir.config_parameter'].set_param('ir_attachment.location', 'swift') + + def test_connection(self): + """ Test the connection to the Swift object store """ + conn = self.Attachment._get_swift_connection() + self.assertNotEquals(conn, False) + + def test_store_file_on_swift(self): + a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64}) + a5bis = self.Attachment.browse(a5.id)[0] + + def test_delete_file_on_swift(self): + self.env['ir.config_parameter'].set_param('ir_attachment.location', 'swift') + a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64}) + uri = SwiftUri(a5.store_fname) + con = self.Attachment._get_swift_connection() + data = con.get_object(uri.container(), uri.item()) + a5.unlink() + with self.assertRaises(ClientException): + con.get_object(uri.container(), uri.item())