attachment_swift: share a session for all connections

OVH's Swift applies a rate limit on the authentication.

attachment_swift authenticates again each time it has to read/write an
attachment. When running upgrades on upgrades of files or installing a
new DB, at some point, we get rejected with HTTP 429.

This commit introduces a shared storage for Swift Session. All
connections will reuses the same authentication token created the first
time a connection needs a Session.

Note: needs python-swiftclient>=3.7.0 to have
https://github.com/openstack/python-swiftclient/commit/1971ef880ff225379d4a91f00f89f323a1605eeb

attachment_swift/models/ir_attachment.py

@@ -13,6 +13,9 @@ _logger = logging.getLogger(__name__)
 
 try:
     import swiftclient
+    import keystoneauth1
+    import keystoneauth1.identity
+    import keystoneauth1.session
     from swiftclient.exceptions import ClientException
 except ImportError:
     swiftclient = None
@@ -20,6 +23,54 @@ except ImportError:
     _logger.debug("Cannot 'import swiftclient'.")
 
 
+SWIFT_TIMEOUT = 15
+
+
+class SwiftSessionStore(object):
+    """Keep in memory the current Swift Auth session
+
+    The auth endpoint has a rate limit on swift, if every operation
+    on the filestore authenticate, the limit is exhausted and
+    operations rejected with an HTTP error code 429.
+
+    Swift connections can reuse the same session by asking a session
+    matching their connection parameters with ``get_session``.
+
+    The keystoneauth1's session automatically creates a new token
+    if the previous one is expired.
+
+    The best documentation I found about sessions is
+    https://docs.openstack.org/keystoneauth/latest/using-sessions.html
+    """
+
+    def __init__(self):
+        self._sessions = {}
+
+    def _get_key(self, auth_url, username, password, tenant_name):
+        return (auth_url, username, password, tenant_name)
+
+    def get_session(self, auth_url=None, username=None, password=None,
+                    tenant_name=None):
+        key = self._get_key(auth_url, username, password, tenant_name)
+        session = self._sessions.get(key)
+        if not session:
+            auth = keystoneauth1.identity.v2.Password(
+                username=username,
+                password=password,
+                tenant_name=tenant_name,
+                auth_url=auth_url,
+            )
+            session = keystoneauth1.session.Session(
+                auth=auth,
+                timeout=SWIFT_TIMEOUT,
+            )
+            self._sessions[key] = session
+        return session
+
+
+swift_session_store = SwiftSessionStore()
+
+
 class IrAttachment(models.Model):
     _inherit = 'ir.attachment'
 
@@ -44,15 +95,18 @@ class IrAttachment(models.Model):
                 "Problem connecting to Swift store, are the env variables "
                 "(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, "
                 "SWIFT_TENANT_NAME) properly set?"
-                ))
+            ))
         try:
-            conn = swiftclient.client.Connection(authurl=host,
-                                                 user=account,
-                                                 key=password,
-                                                 tenant_name=tenant_name,
-                                                 auth_version='2.0',
-                                                 os_options=os_options,
-                                                 )
+            session = swift_session_store.get_session(
+                username=account,
+                password=password,
+                tenant_name=tenant_name,
+                auth_url=host,
+            )
+            conn = swiftclient.client.Connection(
+                session=session,
+                os_options=os_options,
+            )
         except ClientException:
             _logger.exception('Error connecting to Swift object store')
             raise exceptions.UserError(_('Error on Swift connection'))

attachment_swift/tests/test_mock_swift_api.py

@@ -2,11 +2,16 @@
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
 import base64
+import mock
 import os
 
 from mock import patch
+
+import keystoneauth1
+
 from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment
-from ..swift_uri import SwiftUri
+from odoo.addons.attachment_swift.models.ir_attachment import SwiftSessionStore
+from odoo.addons.attachment_swift.swift_uri import SwiftUri
 
 
 class TestAttachmentSwift(TestIrAttachment):
@@ -16,6 +21,34 @@ class TestAttachmentSwift(TestIrAttachment):
         self.env['ir.config_parameter'].set_param('ir_attachment.location',
                                                   'swift')
 
+    def test_session_store_get_session(self):
+        auth_url = 'auth_url'
+        username = 'username'
+        password = 'password'
+        tenant_name = 'tenant_name'
+        store = SwiftSessionStore()
+        session = store.get_session(
+            auth_url=auth_url,
+            username=username,
+            password=password,
+            tenant_name=tenant_name,
+        )
+        self.assertEqual(session.auth.auth_url, auth_url)
+        self.assertEqual(session.auth.username, username)
+        self.assertEqual(session.auth.password, password)
+        self.assertEqual(session.auth.tenant_name, tenant_name)
+
+        # get the same session on a second call
+        self.assertEqual(
+            store.get_session(
+                auth_url=auth_url,
+                username=username,
+                password=password,
+                tenant_name=tenant_name,
+            ),
+            session
+        )
+
     @patch('swiftclient.client')
     def test_connection(self, mock_swift_client):
         """ Test the connection to the store"""
@@ -27,13 +60,17 @@ class TestAttachmentSwift(TestIrAttachment):
         attachment = self.Attachment
         attachment._get_swift_connection()
         mock_swift_client.Connection.assert_called_once_with(
-            authurl=os.environ.get('SWIFT_AUTH_URL'),
-            user=os.environ.get('SWIFT_ACCOUNT'),
-            key=os.environ.get('SWIFT_PASSWORD'),
-            tenant_name=os.environ.get('SWIFT_TENANT_NAME'),
-            auth_version='2.0',
+            session=mock.ANY,
             os_options={'region_name': os.environ.get('SWIFT_REGION_NAME')},
         )
+        __, kwargs = mock_swift_client.Connection.call_args
+        session = kwargs['session']
+        self.assertTrue(isinstance(session, keystoneauth1.session.Session))
+        self.assertEqual(session.auth.auth_url, os.environ['SWIFT_AUTH_URL'])
+        self.assertEqual(session.auth.username, os.environ['SWIFT_ACCOUNT'])
+        self.assertEqual(session.auth.password, os.environ['SWIFT_PASSWORD'])
+        self.assertEqual(session.auth.tenant_name,
+                         os.environ['SWIFT_TENANT_NAME'])
 
     def test_store_file_on_swift(self):
         """

requirements.txt

@@ -2,5 +2,6 @@ boto3==1.9.102
 redis==2.10.5
 python-json-logger==0.1.5
 statsd==3.2.1
-python-swiftclient==3.4.0
-python-keystoneclient==3.13.0
+python-swiftclient==3.7.0
+python-keystoneclient==3.19.0
+keystoneauth1==3.14.0