From 440600e89f360adaa05d986f7d72b75b23622494 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 2 May 2019 15:28:35 +0200 Subject: [PATCH 1/4] Add missing variable in documentation example --- attachment_swift/README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/attachment_swift/README.rst b/attachment_swift/README.rst index 6a3190d..edf3505 100644 --- a/attachment_swift/README.rst +++ b/attachment_swift/README.rst @@ -16,6 +16,7 @@ Configure accesses with environment variables: * ``SWIFT_TENANT_NAME`` * ``SWIFT_ACCOUNT`` * ``SWIFT_PASSWORD`` +* ``SWIFT_REGION_NAME`` : optional region * ``SWIFT_WRITE_CONTAINER`` : Name of the container to use in the store (created if not existing) Read-only mode: @@ -44,6 +45,7 @@ The python-swiftclient can be used from the command line, useful to test: export OS_USERNAME={SWIFT_ACCOUNT} export OS_PASSWORD={SWIFT_PASSWORD} export OS_TENANT_NAME={SWIFT_TENANT_NAME} + export SWIFT_REGION_NAME={SWIFT_REGION_NAME} export OS_AUTH_URL=https://auth.cloud.ovh.net/v2.0 swift stat From 8a4e9d1f84d4a33f239cfdb2bc4b903b36c20a1b Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 2 May 2019 15:29:26 +0200 Subject: [PATCH 2/4] attachment_swift: share a session for all connections OVH's Swift applies a rate limit on the authentication. attachment_swift authenticates again each time it has to read/write an attachment. When running upgrades on upgrades of files or installing a new DB, at some point, we get rejected with HTTP 429. This commit introduces a shared storage for Swift Session. All connections will reuses the same authentication token created the first time a connection needs a Session. Note: needs python-swiftclient>=3.7.0 to have https://github.com/openstack/python-swiftclient/commit/1971ef880ff225379d4a91f00f89f323a1605eeb --- attachment_swift/models/ir_attachment.py | 70 ++++++++++++++++--- attachment_swift/tests/test_mock_swift_api.py | 49 +++++++++++-- requirements.txt | 5 +- 3 files changed, 108 insertions(+), 16 deletions(-) diff --git a/attachment_swift/models/ir_attachment.py b/attachment_swift/models/ir_attachment.py index f57cacb..9d79b6f 100644 --- a/attachment_swift/models/ir_attachment.py +++ b/attachment_swift/models/ir_attachment.py @@ -14,6 +14,9 @@ _logger = logging.getLogger(__name__) try: import swiftclient + import keystoneauth1 + import keystoneauth1.identity + import keystoneauth1.session from swiftclient.exceptions import ClientException except ImportError: swiftclient = None @@ -21,6 +24,54 @@ except ImportError: _logger.debug("Cannot 'import swiftclient'.") +SWIFT_TIMEOUT = 15 + + +class SwiftSessionStore(object): + """Keep in memory the current Swift Auth session + + The auth endpoint has a rate limit on swift, if every operation + on the filestore authenticate, the limit is exhausted and + operations rejected with an HTTP error code 429. + + Swift connections can reuse the same session by asking a session + matching their connection parameters with ``get_session``. + + The keystoneauth1's session automatically creates a new token + if the previous one is expired. + + The best documentation I found about sessions is + https://docs.openstack.org/keystoneauth/latest/using-sessions.html + """ + + def __init__(self): + self._sessions = {} + + def _get_key(self, auth_url, username, password, tenant_name): + return (auth_url, username, password, tenant_name) + + def get_session(self, auth_url=None, username=None, password=None, + tenant_name=None): + key = self._get_key(auth_url, username, password, tenant_name) + session = self._sessions.get(key) + if not session: + auth = keystoneauth1.identity.v2.Password( + username=username, + password=password, + tenant_name=tenant_name, + auth_url=auth_url, + ) + session = keystoneauth1.session.Session( + auth=auth, + timeout=SWIFT_TIMEOUT, + ) + self._sessions[key] = session + return session + + +swift_session_store = SwiftSessionStore() + + class IrAttachment(models.Model): _inherit = 'ir.attachment' @@ -45,15 +96,18 @@ class IrAttachment(models.Model): "Problem connecting to Swift store, are the env variables " "(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, " "SWIFT_TENANT_NAME) properly set?" - )) + )) try: - conn = swiftclient.client.Connection(authurl=host, - user=account, - key=password, - tenant_name=tenant_name, - auth_version='2.0', - os_options=os_options, - ) + session = swift_session_store.get_session( + username=account, + password=password, + tenant_name=tenant_name, + auth_url=host, + ) + conn = swiftclient.client.Connection( + session=session, + os_options=os_options, + ) except ClientException: _logger.exception('Error connecting to Swift object store') raise exceptions.UserError(_('Error on Swift connection')) diff --git a/attachment_swift/tests/test_mock_swift_api.py b/attachment_swift/tests/test_mock_swift_api.py index d1d00b9..fa72a53 100644 --- a/attachment_swift/tests/test_mock_swift_api.py +++ b/attachment_swift/tests/test_mock_swift_api.py @@ -3,11 +3,16 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) import base64 +import mock import os from mock import patch + +import keystoneauth1 + from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment -from ..swift_uri import SwiftUri +from odoo.addons.attachment_swift.models.ir_attachment import SwiftSessionStore +from odoo.addons.attachment_swift.swift_uri import SwiftUri class TestAttachmentSwift(TestIrAttachment): @@ -17,6 +22,34 @@ class TestAttachmentSwift(TestIrAttachment): self.env['ir.config_parameter'].set_param('ir_attachment.location', 'swift') + def test_session_store_get_session(self): + auth_url = 'auth_url' + username = 'username' + password = 'password' + tenant_name = 'tenant_name' + store = SwiftSessionStore() + session = store.get_session( + auth_url=auth_url, + username=username, + password=password, + tenant_name=tenant_name, + ) + self.assertEqual(session.auth.auth_url, auth_url) + self.assertEqual(session.auth.username, username) + self.assertEqual(session.auth.password, password) + self.assertEqual(session.auth.tenant_name, tenant_name) + + # get the same session on a second call + self.assertEqual( + store.get_session( + auth_url=auth_url, + username=username, + password=password, + tenant_name=tenant_name, + ), + session + ) + @patch('swiftclient.client') def test_connection(self, mock_swift_client): """ Test the connection to the store""" @@ -28,13 +61,17 @@ class TestAttachmentSwift(TestIrAttachment): attachment = self.Attachment attachment._get_swift_connection() mock_swift_client.Connection.assert_called_once_with( - authurl=os.environ.get('SWIFT_AUTH_URL'), - user=os.environ.get('SWIFT_ACCOUNT'), - key=os.environ.get('SWIFT_PASSWORD'), - tenant_name=os.environ.get('SWIFT_TENANT_NAME'), - auth_version='2.0', + session=mock.ANY, os_options={'region_name': os.environ.get('SWIFT_REGION_NAME')}, ) + __, kwargs = mock_swift_client.Connection.call_args + session = kwargs['session'] + self.assertTrue(isinstance(session, keystoneauth1.session.Session)) + self.assertEqual(session.auth.auth_url, os.environ['SWIFT_AUTH_URL']) + self.assertEqual(session.auth.username, os.environ['SWIFT_ACCOUNT']) + self.assertEqual(session.auth.password, os.environ['SWIFT_PASSWORD']) + self.assertEqual(session.auth.tenant_name, + os.environ['SWIFT_TENANT_NAME']) def test_store_file_on_swift(self): """ diff --git a/requirements.txt b/requirements.txt index e3c77a7..ec8459a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ boto==2.42.0 redis==2.10.5 python-json-logger==0.1.5 statsd==3.2.1 -python-swiftclient==3.4.0 -python-keystoneclient==3.13.0 +python-swiftclient==3.7.0 +python-keystoneclient==3.19.0 +keystoneauth1==3.14.0 From fe827e6b3f29275a71328a4756112a2ecee951e0 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 2 May 2019 16:14:47 +0200 Subject: [PATCH 3/4] Pin PyYAML version because 5.x does not work with Odoo It would raise ConstructorError: could not determine a constructor for the tag '!record' --- .travis.yml | 1 - requirements.txt | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 7f1a418..511c77f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,7 +12,6 @@ addons: - python-lxml # because pip installation is slow - python-simplejson - python-serial - - python-yaml env: matrix: diff --git a/requirements.txt b/requirements.txt index ec8459a..9d0c28d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ statsd==3.2.1 python-swiftclient==3.7.0 python-keystoneclient==3.19.0 keystoneauth1==3.14.0 +# error with 5.x (ConstructorError: could not determine a constructor for the tag '!record') +PyYAML==4.2b4 From b45b4d23a9fbac3b43fb88d71bc7b78752329fbe Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Wed, 8 May 2019 10:21:59 +0200 Subject: [PATCH 4/4] Override dependencies with different name and pin versions The name of the libs and python packages are different, Odoo expects the inner python package in the manifest, but setuptools cannot find the libs in pypi, overrides them with the libs names. --- attachment_swift/__manifest__.py | 1 + setup/attachment_swift/setup.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/attachment_swift/__manifest__.py b/attachment_swift/__manifest__.py index e486064..7d3dc55 100644 --- a/attachment_swift/__manifest__.py +++ b/attachment_swift/__manifest__.py @@ -13,6 +13,7 @@ 'external_dependencies': { 'python': ['swiftclient', 'keystoneclient', + 'keystoneauth1', ], }, 'website': 'https://www.camptocamp.com', diff --git a/setup/attachment_swift/setup.py b/setup/attachment_swift/setup.py index 28c57bb..d6d1eb0 100644 --- a/setup/attachment_swift/setup.py +++ b/setup/attachment_swift/setup.py @@ -2,5 +2,13 @@ import setuptools setuptools.setup( setup_requires=['setuptools-odoo'], - odoo_addon=True, + odoo_addon={ + 'external_dependencies_override': { + 'python': { + 'swiftclient': 'python-swiftclient>=3.7.0', + 'keystoneclient': 'python-keystoneclient>=3.19.0', + 'keystoneauth1': 'keystoneauth1>=3.14.0', + }, + }, + } )