From 03288680559b7755c2e6500c69194c26c654e3a1 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 2 May 2019 15:28:35 +0200 Subject: [PATCH 1/4] Add missing variable in documentation example --- attachment_swift/README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/attachment_swift/README.rst b/attachment_swift/README.rst index 8999290..a05768d 100644 --- a/attachment_swift/README.rst +++ b/attachment_swift/README.rst @@ -16,6 +16,7 @@ Configure accesses with environment variables: * ``SWIFT_TENANT_NAME`` * ``SWIFT_ACCOUNT`` * ``SWIFT_PASSWORD`` +* ``SWIFT_REGION_NAME`` : optional region * ``SWIFT_WRITE_CONTAINER`` : Name of the container to use in the store (created if not existing) Read-only mode: @@ -44,6 +45,7 @@ The python-swiftclient can be used from the command line, useful to test: export OS_USERNAME={SWIFT_ACCOUNT} export OS_PASSWORD={SWIFT_PASSWORD} export OS_TENANT_NAME={SWIFT_TENANT_NAME} + export SWIFT_REGION_NAME={SWIFT_REGION_NAME} export OS_AUTH_URL=https://auth.cloud.ovh.net/v2.0 swift stat From 5a0c554749b8cc7c0fa8f0e6bbbb7f6daed035d6 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 2 May 2019 15:29:26 +0200 Subject: [PATCH 2/4] attachment_swift: share a session for all connections OVH's Swift applies a rate limit on the authentication. attachment_swift authenticates again each time it has to read/write an attachment. When running upgrades on upgrades of files or installing a new DB, at some point, we get rejected with HTTP 429. This commit introduces a shared storage for Swift Session. All connections will reuses the same authentication token created the first time a connection needs a Session. Note: needs python-swiftclient>=3.7.0 to have https://github.com/openstack/python-swiftclient/commit/1971ef880ff225379d4a91f00f89f323a1605eeb --- attachment_swift/models/ir_attachment.py | 70 ++++++++++++++++--- attachment_swift/tests/test_mock_swift_api.py | 49 +++++++++++-- requirements.txt | 5 +- 3 files changed, 108 insertions(+), 16 deletions(-) diff --git a/attachment_swift/models/ir_attachment.py b/attachment_swift/models/ir_attachment.py index f57cacb..9d79b6f 100644 --- a/attachment_swift/models/ir_attachment.py +++ b/attachment_swift/models/ir_attachment.py @@ -14,6 +14,9 @@ _logger = logging.getLogger(__name__) try: import swiftclient + import keystoneauth1 + import keystoneauth1.identity + import keystoneauth1.session from swiftclient.exceptions import ClientException except ImportError: swiftclient = None @@ -21,6 +24,54 @@ except ImportError: _logger.debug("Cannot 'import swiftclient'.") +SWIFT_TIMEOUT = 15 + + +class SwiftSessionStore(object): + """Keep in memory the current Swift Auth session + + The auth endpoint has a rate limit on swift, if every operation + on the filestore authenticate, the limit is exhausted and + operations rejected with an HTTP error code 429. + + Swift connections can reuse the same session by asking a session + matching their connection parameters with ``get_session``. + + The keystoneauth1's session automatically creates a new token + if the previous one is expired. + + The best documentation I found about sessions is + https://docs.openstack.org/keystoneauth/latest/using-sessions.html + """ + + def __init__(self): + self._sessions = {} + + def _get_key(self, auth_url, username, password, tenant_name): + return (auth_url, username, password, tenant_name) + + def get_session(self, auth_url=None, username=None, password=None, + tenant_name=None): + key = self._get_key(auth_url, username, password, tenant_name) + session = self._sessions.get(key) + if not session: + auth = keystoneauth1.identity.v2.Password( + username=username, + password=password, + tenant_name=tenant_name, + auth_url=auth_url, + ) + session = keystoneauth1.session.Session( + auth=auth, + timeout=SWIFT_TIMEOUT, + ) + self._sessions[key] = session + return session + + +swift_session_store = SwiftSessionStore() + + class IrAttachment(models.Model): _inherit = 'ir.attachment' @@ -45,15 +96,18 @@ class IrAttachment(models.Model): "Problem connecting to Swift store, are the env variables " "(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, " "SWIFT_TENANT_NAME) properly set?" - )) + )) try: - conn = swiftclient.client.Connection(authurl=host, - user=account, - key=password, - tenant_name=tenant_name, - auth_version='2.0', - os_options=os_options, - ) + session = swift_session_store.get_session( + username=account, + password=password, + tenant_name=tenant_name, + auth_url=host, + ) + conn = swiftclient.client.Connection( + session=session, + os_options=os_options, + ) except ClientException: _logger.exception('Error connecting to Swift object store') raise exceptions.UserError(_('Error on Swift connection')) diff --git a/attachment_swift/tests/test_mock_swift_api.py b/attachment_swift/tests/test_mock_swift_api.py index 7c093f9..31402eb 100644 --- a/attachment_swift/tests/test_mock_swift_api.py +++ b/attachment_swift/tests/test_mock_swift_api.py @@ -2,11 +2,16 @@ # Copyright 2017 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +import mock import os from mock import patch + +import keystoneauth1 + from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment -from ..swift_uri import SwiftUri +from odoo.addons.attachment_swift.models.ir_attachment import SwiftSessionStore +from odoo.addons.attachment_swift.swift_uri import SwiftUri class TestAttachmentSwift(TestIrAttachment): @@ -16,6 +21,34 @@ class TestAttachmentSwift(TestIrAttachment): self.env['ir.config_parameter'].set_param('ir_attachment.location', 'swift') + def test_session_store_get_session(self): + auth_url = 'auth_url' + username = 'username' + password = 'password' + tenant_name = 'tenant_name' + store = SwiftSessionStore() + session = store.get_session( + auth_url=auth_url, + username=username, + password=password, + tenant_name=tenant_name, + ) + self.assertEqual(session.auth.auth_url, auth_url) + self.assertEqual(session.auth.username, username) + self.assertEqual(session.auth.password, password) + self.assertEqual(session.auth.tenant_name, tenant_name) + + # get the same session on a second call + self.assertEqual( + store.get_session( + auth_url=auth_url, + username=username, + password=password, + tenant_name=tenant_name, + ), + session + ) + @patch('swiftclient.client') def test_connection(self, mock_swift_client): """ Test the connection to the store""" @@ -27,13 +60,17 @@ class TestAttachmentSwift(TestIrAttachment): attachment = self.Attachment attachment._get_swift_connection() mock_swift_client.Connection.assert_called_once_with( - authurl=os.environ.get('SWIFT_AUTH_URL'), - user=os.environ.get('SWIFT_ACCOUNT'), - key=os.environ.get('SWIFT_PASSWORD'), - tenant_name=os.environ.get('SWIFT_TENANT_NAME'), - auth_version='2.0', + session=mock.ANY, os_options={'region_name': os.environ.get('SWIFT_REGION_NAME')}, ) + __, kwargs = mock_swift_client.Connection.call_args + session = kwargs['session'] + self.assertTrue(isinstance(session, keystoneauth1.session.Session)) + self.assertEqual(session.auth.auth_url, os.environ['SWIFT_AUTH_URL']) + self.assertEqual(session.auth.username, os.environ['SWIFT_ACCOUNT']) + self.assertEqual(session.auth.password, os.environ['SWIFT_PASSWORD']) + self.assertEqual(session.auth.tenant_name, + os.environ['SWIFT_TENANT_NAME']) def test_store_file_on_swift(self): """ diff --git a/requirements.txt b/requirements.txt index e3c77a7..ec8459a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ boto==2.42.0 redis==2.10.5 python-json-logger==0.1.5 statsd==3.2.1 -python-swiftclient==3.4.0 -python-keystoneclient==3.13.0 +python-swiftclient==3.7.0 +python-keystoneclient==3.19.0 +keystoneauth1==3.14.0 From 1178b6d9a4d8803e88b29cfcd3993fbcfb7b442d Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 2 May 2019 16:14:47 +0200 Subject: [PATCH 3/4] Pin PyYAML version because 5.x does not work with Odoo It would raise ConstructorError: could not determine a constructor for the tag '!record' --- .travis.yml | 1 - requirements.txt | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 4b30e51..e7485a6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,6 @@ addons: - python-lxml # because pip installation is slow - python-simplejson - python-serial - - python-yaml python: - '2.7' diff --git a/requirements.txt b/requirements.txt index ec8459a..9d0c28d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ statsd==3.2.1 python-swiftclient==3.7.0 python-keystoneclient==3.19.0 keystoneauth1==3.14.0 +# error with 5.x (ConstructorError: could not determine a constructor for the tag '!record') +PyYAML==4.2b4 From c405f57cc264e59a9fc1857fcc5b7451e33b82c0 Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Wed, 8 May 2019 10:21:59 +0200 Subject: [PATCH 4/4] Override dependencies with different name and pin versions The name of the libs and python packages are different, Odoo expects the inner python package in the manifest, but setuptools cannot find the libs in pypi, overrides them with the libs names. --- attachment_swift/__manifest__.py | 1 + setup/attachment_swift/setup.py | 10 +++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/attachment_swift/__manifest__.py b/attachment_swift/__manifest__.py index 1a63b4b..bed6a37 100644 --- a/attachment_swift/__manifest__.py +++ b/attachment_swift/__manifest__.py @@ -13,6 +13,7 @@ 'external_dependencies': { 'python': ['swiftclient', 'keystoneclient', + 'keystoneauth1', ], }, 'website': 'https://www.camptocamp.com', diff --git a/setup/attachment_swift/setup.py b/setup/attachment_swift/setup.py index 28c57bb..d6d1eb0 100644 --- a/setup/attachment_swift/setup.py +++ b/setup/attachment_swift/setup.py @@ -2,5 +2,13 @@ import setuptools setuptools.setup( setup_requires=['setuptools-odoo'], - odoo_addon=True, + odoo_addon={ + 'external_dependencies_override': { + 'python': { + 'swiftclient': 'python-swiftclient>=3.7.0', + 'keystoneclient': 'python-keystoneclient>=3.19.0', + 'keystoneauth1': 'keystoneauth1>=3.14.0', + }, + }, + } )