From ab1f9595b1e0994f3af3020fcedbb11061413837 Mon Sep 17 00:00:00 2001 From: Patrick Tombez Date: Wed, 8 May 2019 17:09:44 +0200 Subject: [PATCH] Add attachment_swift --- attachment_swift/README.rst | 51 ++++++++++ attachment_swift/__init__.py | 1 + attachment_swift/__openerp__.py | 21 ++++ attachment_swift/models/__init__.py | 1 + attachment_swift/models/ir_attachment.py | 120 +++++++++++++++++++++++ attachment_swift/swift_uri.py | 23 +++++ requirements.txt | 2 + 7 files changed, 219 insertions(+) create mode 100644 attachment_swift/README.rst create mode 100644 attachment_swift/__init__.py create mode 100644 attachment_swift/__openerp__.py create mode 100644 attachment_swift/models/__init__.py create mode 100644 attachment_swift/models/ir_attachment.py create mode 100644 attachment_swift/swift_uri.py diff --git a/attachment_swift/README.rst b/attachment_swift/README.rst new file mode 100644 index 0000000..8999290 --- /dev/null +++ b/attachment_swift/README.rst @@ -0,0 +1,51 @@ +Attachments on Swift storage +============================ + +This addon enable storing attachments (documents and assets) on OpenStack Object Storage (Swift) + +Configuration +------------- + +Activate Swift storage: + +* Create or set the system parameter with the key ``ir_attachment.location`` with the following value ``swift``. + +Configure accesses with environment variables: + +* ``SWIFT_AUTH_URL`` : URL of the Swift server +* ``SWIFT_TENANT_NAME`` +* ``SWIFT_ACCOUNT`` +* ``SWIFT_PASSWORD`` +* ``SWIFT_WRITE_CONTAINER`` : Name of the container to use in the store (created if not existing) + +Read-only mode: + +The container name and the key are stored in the attachment. So if you change the +``SWIFT_WRITE_CONTAINER`` or the ``ir_attachment.location``, the existing attachments +will still be read on their former container. But as soon as they are written over +or new attachments are created, they will be created on the new container or on +the other location (db or filesystem). This is a convenient way to be able to +read the production attachments on a replication (since you have the +credentials) without any risk to alter the production data. + +This addon must be added in the server wide addons with (``--load`` option): + +``--load=web,web_kanban,attachment_swift`` + +Python Dependencies +------------------- + +This module needs the python-swiftclient and the python-keystoneclient (For auth v2.0) to work. +The python-keystoneclient needs the linux package build-essential and python-dev to install properly. + +The python-swiftclient can be used from the command line, useful to test: + + export AUTH_VERSION=2.0 + export OS_USERNAME={SWIFT_ACCOUNT} + export OS_PASSWORD={SWIFT_PASSWORD} + export OS_TENANT_NAME={SWIFT_TENANT_NAME} + export OS_AUTH_URL=https://auth.cloud.ovh.net/v2.0 + swift stat + +More information at +https://docs.openstack.org/python-swiftclient/latest/cli/index.html#swift-usage diff --git a/attachment_swift/__init__.py b/attachment_swift/__init__.py new file mode 100644 index 0000000..0650744 --- /dev/null +++ b/attachment_swift/__init__.py @@ -0,0 +1 @@ +from . import models diff --git a/attachment_swift/__openerp__.py b/attachment_swift/__openerp__.py new file mode 100644 index 0000000..b3490e2 --- /dev/null +++ b/attachment_swift/__openerp__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +{'name': 'Attachments on Swift storage', + 'summary': 'Store assets and attachments on a Swift compatible object store', + 'version': '7.0.1.0.0', + 'author': 'Camptocamp,Odoo Community Association (OCA)', + 'license': 'AGPL-3', + 'category': 'Knowledge Management', + 'depends': ['base_attachment_object_storage'], + 'external_dependencies': { + 'python': ['swiftclient', + 'keystoneclient', + ], + }, + 'website': 'https://www.camptocamp.com', + 'data': [], + 'installable': True, + } diff --git a/attachment_swift/models/__init__.py b/attachment_swift/models/__init__.py new file mode 100644 index 0000000..aaf38a1 --- /dev/null +++ b/attachment_swift/models/__init__.py @@ -0,0 +1 @@ +from . import ir_attachment diff --git a/attachment_swift/models/ir_attachment.py b/attachment_swift/models/ir_attachment.py new file mode 100644 index 0000000..8418161 --- /dev/null +++ b/attachment_swift/models/ir_attachment.py @@ -0,0 +1,120 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + + +import base64 +import logging +import os +from ..swift_uri import SwiftUri + +from openerp.tools.translate import _ +from openerp.osv import osv +from openerp.osv.orm import except_orm + +_logger = logging.getLogger(__name__) + +try: + import swiftclient + from swiftclient.exceptions import ClientException +except ImportError: + swiftclient = None + ClientException = None + _logger.debug("Cannot 'import swiftclient'.") + + +class IrAttachment(osv.osv): + _inherit = 'ir.attachment' + + def _get_stores(self): + return ['swift'] + super(IrAttachment, self)._get_stores() + + def _get_swift_connection(self): + """ Returns a connection object for the Swift object store """ + host = os.environ.get('SWIFT_AUTH_URL') + account = os.environ.get('SWIFT_ACCOUNT') + password = os.environ.get('SWIFT_PASSWORD') + tenant_name = os.environ.get('SWIFT_TENANT_NAME') + region = os.environ.get('SWIFT_REGION_NAME') + os_options = {} + if region: + os_options['region_name'] = region + if not (host and account and password and tenant_name): + raise except_orm( + _("Error"), + _("Problem connecting to Swift store, are the env variables " + "(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, " + "SWIFT_TENANT_NAME) properly set?") + ) + try: + conn = swiftclient.client.Connection(authurl=host, + user=account, + key=password, + tenant_name=tenant_name, + auth_version='2.0', + os_options=os_options, + ) + except ClientException: + _logger.exception('Error connecting to Swift object store') + raise except_orm( + _("Error"), + _('Error on Swift connection')) + return conn + + def _store_file_read(self, fname, bin_size=False): + if fname.startswith('swift://'): + swifturi = SwiftUri(fname) + try: + conn = self._get_swift_connection() + except except_orm: + _logger.exception( + "error reading attachment '%s' from object storage", fname + ) + return '' + try: + resp, obj_content = conn.get_object(swifturi.container(), + swifturi.item()) + read = base64.b64encode(obj_content) + except ClientException: + read = '' + _logger.exception( + 'Error reading object from Swift object store') + return read + else: + return super(IrAttachment, self)._store_file_read(fname, bin_size) + + def _store_file_write(self, storage, key, bin_data): + if storage == 'swift': + container = os.environ.get('SWIFT_WRITE_CONTAINER') + conn = self._get_swift_connection() + conn.put_container(container) + filename = 'swift://{}/{}'.format(container, key) + try: + conn.put_object(container, key, bin_data) + except ClientException: + _logger.exception('Error writing to Swift object store') + raise except_orm( + _("Error"), + _('Error writing to Swift')) + else: + _super = super(IrAttachment, self) + filename = _super._store_file_write(key, bin_data) + return filename + + def _store_file_delete(self, fname): + if fname.startswith('swift://'): + swifturi = SwiftUri(fname) + container = swifturi.container() + # delete the file only if it is on the current configured bucket + # otherwise, we might delete files used on a different environment + if container == os.environ.get('SWIFT_WRITE_CONTAINER'): + conn = self._get_swift_connection() + try: + conn.delete_object(container, swifturi.item()) + except ClientException: + _logger.exception( + _('Error deleting an object on the Swift store')) + # we ignore the error, file will stay on the object + # storage but won't disrupt the process + else: + super(IrAttachment, self)._store_file_delete(fname) diff --git a/attachment_swift/swift_uri.py b/attachment_swift/swift_uri.py new file mode 100644 index 0000000..268eeec --- /dev/null +++ b/attachment_swift/swift_uri.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import re + + +class SwiftUri(object): + + _url_re = re.compile("^swift:///*([^/]*)/?(.*)", + re.IGNORECASE | re.UNICODE) + + def __init__(self, uri): + match = self._url_re.match(uri) + if not match: + raise ValueError("%s: is not a valid Swift URI" % (uri,)) + self._container, self._item = match.groups() + + def container(self): + return self._container + + def item(self): + return self._item diff --git a/requirements.txt b/requirements.txt index 4c98fd2..9a957bb 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,4 @@ boto==2.42.0 python-json-logger==0.1.5 +python-swiftclient==3.4.0 +python-keystoneclient==3.13.0