diff --git a/attachment_swift/README.rst b/attachment_swift/README.rst index a5bcc19..74f6477 100644 --- a/attachment_swift/README.rst +++ b/attachment_swift/README.rst @@ -1,7 +1,7 @@ Attachments on Swift storage ============================ -This addon enables storing attachments (documents and assets) on OpenStack Object Storage (Swift). +This addon enable storing attachments (documents and assets) on OpenStack Object Storage (Swift) Configuration ------------- @@ -13,9 +13,11 @@ Activate Swift storage: Configure accesses with environment variables: * ``SWIFT_AUTH_URL`` : URL of the Swift server -* ``SWIFT_TENANT_NAME`` +* ``SWIFT_TENANT_NAME`` : **!** DEPRECATED **!** Use ``SWIFT_PROJECT_NAME`` instead +* ``SWIFT_PROJECT_NAME`` * ``SWIFT_ACCOUNT`` * ``SWIFT_PASSWORD`` +* ``SWIFT_REGION_NAME`` : optional region * ``SWIFT_WRITE_CONTAINER`` : Name of the container to use in the store (created if not existing) Read-only mode: @@ -30,21 +32,24 @@ credentials) without any risk to alter the production data. This addon must be added in the server wide addons with (``--load`` option): -``--load=web,web_kanban,attachment_swift`` +``--load=web,attachment_swift`` Python Dependencies ------------------- -This module needs the python-swiftclient and the python-keystoneclient (For auth v2.0) to work. +This module needs the python-swiftclient and the python-keystoneclient (For auth v3.0) to work. The python-keystoneclient needs the linux package build-essential and python-dev to install properly. The python-swiftclient can be used from the command line, useful to test: - export AUTH_VERSION=2.0 +.. code-block:: sh + + export AUTH_VERSION=3.0 export OS_USERNAME={SWIFT_ACCOUNT} export OS_PASSWORD={SWIFT_PASSWORD} - export OS_TENANT_NAME={SWIFT_TENANT_NAME} - export OS_AUTH_URL=https://auth.cloud.ovh.net/v2.0 + export OS_PROJECT_NAME={SWIFT_PROJECT_NAME} + export OS_REGION_NAME={SWIFT_REGION_NAME} + export OS_AUTH_URL=https://auth.cloud.ovh.net/v3 swift stat More information at diff --git a/attachment_swift/models/ir_attachment.py b/attachment_swift/models/ir_attachment.py index 62fc203..8fd7f01 100644 --- a/attachment_swift/models/ir_attachment.py +++ b/attachment_swift/models/ir_attachment.py @@ -16,6 +16,7 @@ _logger = logging.getLogger(__name__) try: import swiftclient + import keystoneauth1 from swiftclient.exceptions import ClientException except ImportError: swiftclient = None @@ -23,6 +24,54 @@ except ImportError: _logger.debug("Cannot 'import swiftclient'.") +SWIFT_TIMEOUT = 15 + + +class SwiftSessionStore(object): + """Keep in memory the current Swift Auth session + + The auth endpoint has a rate limit on swift, if every operation + on the filestore authenticate, the limit is exhausted and + operations rejected with an HTTP error code 429. + + Swift connections can reuse the same session by asking a session + matching their connection parameters with ``get_session``. + + The keystoneauth1's session automatically creates a new token + if the previous one is expired. + + The best documentation I found about sessions is + https://docs.openstack.org/keystoneauth/latest/using-sessions.html + """ + + def __init__(self): + self._sessions = {} + + def _get_key(self, auth_url, username, password, project_name): + return (auth_url, username, password, project_name) + + def get_session(self, auth_url=None, username=None, password=None, + project_name=None): + key = self._get_key(auth_url, username, password, project_name) + session = self._sessions.get(key) + if not session: + auth = keystoneauth1.identity.v3.Password( + username=username, + password=password, + project_name=project_name, + auth_url=auth_url, + ) + session = keystoneauth1.session.Session( + auth=auth, + timeout=SWIFT_TIMEOUT, + ) + self._sessions[key] = session + return session + + +swift_session_store = SwiftSessionStore() + + class IrAttachment(osv.osv): _inherit = 'ir.attachment' @@ -34,26 +83,34 @@ class IrAttachment(osv.osv): host = os.environ.get('SWIFT_AUTH_URL') account = os.environ.get('SWIFT_ACCOUNT') password = os.environ.get('SWIFT_PASSWORD') - tenant_name = os.environ.get('SWIFT_TENANT_NAME') + project_name = os.environ.get('SWIFT_PROJECT_NAME') + if not project_name and os.environ.get('SWIFT_TENANT_NAME'): + project_name = os.environ['SWIFT_TENANT_NAME'] + _logger.warning( + "SWIFT_TENANT_NAME is deprecated and " + "must be replaced by SWIFT_PROJECT_NAME" + ) region = os.environ.get('SWIFT_REGION_NAME') os_options = {} if region: os_options['region_name'] = region - if not (host and account and password and tenant_name): + if not (host and account and password and project_name): raise except_orm( _("Error"), _("Problem connecting to Swift store, are the env variables " "(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, " - "SWIFT_TENANT_NAME) properly set?") - ) + "SWIFT_TENANT_NAME) properly set?")) try: - conn = swiftclient.client.Connection(authurl=host, - user=account, - key=password, - tenant_name=tenant_name, - auth_version='2.0', - os_options=os_options, - ) + session = swift_session_store.get_session( + username=account, + password=password, + project_name=project_name, + auth_url=host, + ) + conn = swiftclient.client.Connection( + session=session, + os_options=os_options, + ) except ClientException: _logger.exception('Error connecting to Swift object store') raise except_orm( diff --git a/attachment_swift/tests/__init__.py b/attachment_swift/tests/__init__.py new file mode 100644 index 0000000..506e6b2 --- /dev/null +++ b/attachment_swift/tests/__init__.py @@ -0,0 +1,2 @@ + +from . import test_mock_swift_api diff --git a/attachment_swift/tests/test_mock_swift_api.py b/attachment_swift/tests/test_mock_swift_api.py new file mode 100644 index 0000000..397bf91 --- /dev/null +++ b/attachment_swift/tests/test_mock_swift_api.py @@ -0,0 +1,120 @@ +# Copyright 2017-2019 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import base64 +import mock +import os + +from mock import patch + +import keystoneauth1 + +from openerp.addons.base.tests.test_ir_attachment import TestIrAttachment +from openerp.addons.attachment_swift.models.ir_attachment import SwiftSessionStore +from openerp.addons.attachment_swift.swift_uri import SwiftUri + + +class TestAttachmentSwift(TestIrAttachment): + + def setup(self): + super().setUp() + self.env['ir.config_parameter'].set_param('ir_attachment.location', + 'swift') + + def test_session_store_get_session(self): + auth_url = 'auth_url' + username = 'username' + password = 'password' + project_name = 'project_name' + store = SwiftSessionStore() + session = store.get_session( + auth_url=auth_url, + username=username, + password=password, + project_name=project_name, + ) + self.assertEqual(session.auth.auth_url, auth_url) + self.assertEqual(session.auth.get_cache_id_elements().get( + 'password_username'), username) + self.assertEqual(session.auth.get_cache_id_elements().get( + 'password_password'), password) + self.assertEqual(session.auth.project_name, project_name) + + # get the same session on a second call + self.assertEqual( + store.get_session( + auth_url=auth_url, + username=username, + password=password, + project_name=project_name, + ), + session + ) + + @patch('swiftclient.client') + def test_connection(self, mock_swift_client): + """ Test the connection to the store""" + os.environ['SWIFT_AUTH_URL'] = 'auth_url' + os.environ['SWIFT_ACCOUNT'] = 'account' + os.environ['SWIFT_PASSWORD'] = 'password' + os.environ['SWIFT_PROJECT_NAME'] = 'project_name' + os.environ['SWIFT_REGION_NAME'] = 'NOWHERE' + attachment = self.Attachment + attachment._get_swift_connection() + mock_swift_client.Connection.assert_called_once_with( + session=mock.ANY, + os_options={'region_name': os.environ.get('SWIFT_REGION_NAME')}, + ) + __, kwargs = mock_swift_client.Connection.call_args + session = kwargs['session'] + self.assertTrue(isinstance(session, keystoneauth1.session.Session)) + self.assertEqual(session.auth.auth_url, os.environ['SWIFT_AUTH_URL']) + self.assertEqual(session.auth.get_cache_id_elements().get( + 'password_username'), os.environ['SWIFT_ACCOUNT']) + self.assertEqual(session.auth.get_cache_id_elements().get( + 'password_password'), os.environ['SWIFT_PASSWORD']) + self.assertEqual(session.auth.project_name, + os.environ['SWIFT_PROJECT_NAME']) + + def test_store_file_on_swift(self): + """ + Test writing a file + """ + (self.env['ir.config_parameter']. + set_param('ir_attachment.location', 'swift')) + os.environ['SWIFT_AUTH_URL'] = 'auth_url' + os.environ['SWIFT_ACCOUNT'] = 'account' + os.environ['SWIFT_PASSWORD'] = 'password' + os.environ['SWIFT_PROJECT_NAME'] = 'project_name' + os.environ['SWIFT_WRITE_CONTAINER'] = 'my_container' + container = os.environ.get('SWIFT_WRITE_CONTAINER') + attachment = self.Attachment + bin_data = base64.b64decode(self.blob1_b64) + with patch('swiftclient.client.Connection') as MockConnection: + conn = MockConnection.return_value + attachment.create({'name': 'a5', 'datas': self.blob1_b64}) + conn.put_object.assert_called_with( + container, + attachment._compute_checksum(bin_data), + bin_data) + + def test_delete_file_on_swift(self): + """ + Test deleting a file + """ + (self.env['ir.config_parameter']. + set_param('ir_attachment.location', 'swift')) + os.environ['SWIFT_AUTH_URL'] = 'auth_url' + os.environ['SWIFT_ACCOUNT'] = 'account' + os.environ['SWIFT_PASSWORD'] = 'password' + os.environ['SWIFT_PROJECT_NAME'] = 'project_name' + os.environ['SWIFT_WRITE_CONTAINER'] = 'my_container' + + attachment = self.Attachment + container = os.environ.get('SWIFT_WRITE_CONTAINER') + with patch('swiftclient.client.Connection') as MockConnection: + conn = MockConnection.return_value + a5 = attachment.create({'name': 'a5', 'datas': self.blob1_b64}) + uri = SwiftUri(a5.store_fname) + a5.unlink() + conn.delete_object.assert_called_with(container, uri.item()) diff --git a/attachment_swift/tests/test_with_swift_store.py b/attachment_swift/tests/test_with_swift_store.py new file mode 100644 index 0000000..8320316 --- /dev/null +++ b/attachment_swift/tests/test_with_swift_store.py @@ -0,0 +1,42 @@ +# Copyright 2017-2019 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from openerp.addons.base.tests.test_ir_attachment import TestIrAttachment +from ..swift_uri import SwiftUri +from swiftclient.exceptions import ClientException + + +class TestAttachmentSwift(TestIrAttachment): + """ + Those tests are made to be run against a real Swift store (local or remote) + """ + + def setup(self): + super().setUp() + self.env['ir.config_parameter'].set_param('ir_attachment.location', + 'swift') + + def test_connection(self): + """ Test the connection to the Swift object store """ + conn = self.Attachment._get_swift_connection() + self.assertNotEqual(conn, False) + + def test_store_file_on_swift(self): + """ Test writing a file and then reading it """ + (self.env['ir.config_parameter']. + set_param('ir_attachment.location', 'swift')) + a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64}) + a5bis = self.Attachment.browse(a5.id)[0] + self.assertEqual(a5.datas, a5bis.datas) + + def test_delete_file_on_swift(self): + """ Create a file and then test the deletion """ + (self.env['ir.config_parameter']. + set_param('ir_attachment.location', 'swift')) + a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64}) + uri = SwiftUri(a5.store_fname) + con = self.Attachment._get_swift_connection() + con.get_object(uri.container(), uri.item()) + a5.unlink() + with self.assertRaises(ClientException): + con.get_object(uri.container(), uri.item())