Merge pull request #58 from guewen/12.0-swift-auth-session

[12.0] attachment_swift: share a session for all connections

.travis.yml

@@ -13,7 +13,6 @@ addons:
       - python-lxml # because pip installation is slow
       - python-simplejson
       - python-serial
-      - python-yaml
 
 env:
   matrix:

attachment_swift/README.rst

@@ -16,6 +16,7 @@ Configure accesses with environment variables:
 * ``SWIFT_TENANT_NAME``
 * ``SWIFT_ACCOUNT``
 * ``SWIFT_PASSWORD``
+* ``SWIFT_REGION_NAME``         : optional region
 * ``SWIFT_WRITE_CONTAINER``     : Name of the container to use in the store (created if not existing)
 
 Read-only mode:
@@ -44,6 +45,7 @@ The python-swiftclient can be used from the command line, useful to test:
     export OS_USERNAME={SWIFT_ACCOUNT}
     export OS_PASSWORD={SWIFT_PASSWORD}
     export OS_TENANT_NAME={SWIFT_TENANT_NAME}
+    export SWIFT_REGION_NAME={SWIFT_REGION_NAME}
     export OS_AUTH_URL=https://auth.cloud.ovh.net/v2.0
     swift stat
 

attachment_swift/__manifest__.py

@@ -12,6 +12,7 @@
  'external_dependencies': {
      'python': ['swiftclient',
                 'keystoneclient',
+                'keystoneauth1',
                 ],
  },
  'website': 'https://www.camptocamp.com',

attachment_swift/models/ir_attachment.py

@@ -13,6 +13,9 @@ _logger = logging.getLogger(__name__)
 
 try:
     import swiftclient
+    import keystoneauth1
+    import keystoneauth1.identity
+    import keystoneauth1.session
     from swiftclient.exceptions import ClientException
 except ImportError:
     swiftclient = None
@@ -20,6 +23,54 @@ except ImportError:
     _logger.debug("Cannot 'import swiftclient'.")
 
 
+SWIFT_TIMEOUT = 15
+
+
+class SwiftSessionStore(object):
+    """Keep in memory the current Swift Auth session
+
+    The auth endpoint has a rate limit on swift, if every operation
+    on the filestore authenticate, the limit is exhausted and
+    operations rejected with an HTTP error code 429.
+
+    Swift connections can reuse the same session by asking a session
+    matching their connection parameters with ``get_session``.
+
+    The keystoneauth1's session automatically creates a new token
+    if the previous one is expired.
+
+    The best documentation I found about sessions is
+    https://docs.openstack.org/keystoneauth/latest/using-sessions.html
+    """
+
+    def __init__(self):
+        self._sessions = {}
+
+    def _get_key(self, auth_url, username, password, tenant_name):
+        return (auth_url, username, password, tenant_name)
+
+    def get_session(self, auth_url=None, username=None, password=None,
+                    tenant_name=None):
+        key = self._get_key(auth_url, username, password, tenant_name)
+        session = self._sessions.get(key)
+        if not session:
+            auth = keystoneauth1.identity.v2.Password(
+                username=username,
+                password=password,
+                tenant_name=tenant_name,
+                auth_url=auth_url,
+            )
+            session = keystoneauth1.session.Session(
+                auth=auth,
+                timeout=SWIFT_TIMEOUT,
+            )
+            self._sessions[key] = session
+        return session
+
+
+swift_session_store = SwiftSessionStore()
+
+
 class IrAttachment(models.Model):
     _inherit = 'ir.attachment'
 
@@ -44,15 +95,18 @@ class IrAttachment(models.Model):
                 "Problem connecting to Swift store, are the env variables "
                 "(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, "
                 "SWIFT_TENANT_NAME) properly set?"
-                ))
+            ))
         try:
-            conn = swiftclient.client.Connection(authurl=host,
-                                                 user=account,
-                                                 key=password,
-                                                 tenant_name=tenant_name,
-                                                 auth_version='2.0',
-                                                 os_options=os_options,
-                                                 )
+            session = swift_session_store.get_session(
+                username=account,
+                password=password,
+                tenant_name=tenant_name,
+                auth_url=host,
+            )
+            conn = swiftclient.client.Connection(
+                session=session,
+                os_options=os_options,
+            )
         except ClientException:
             _logger.exception('Error connecting to Swift object store')
             raise exceptions.UserError(_('Error on Swift connection'))

attachment_swift/tests/test_mock_swift_api.py

@@ -2,11 +2,16 @@
 # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
 
 import base64
+import mock
 import os
 
 from mock import patch
+
+import keystoneauth1
+
 from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment
-from ..swift_uri import SwiftUri
+from odoo.addons.attachment_swift.models.ir_attachment import SwiftSessionStore
+from odoo.addons.attachment_swift.swift_uri import SwiftUri
 
 
 class TestAttachmentSwift(TestIrAttachment):
@@ -16,6 +21,34 @@ class TestAttachmentSwift(TestIrAttachment):
         self.env['ir.config_parameter'].set_param('ir_attachment.location',
                                                   'swift')
 
+    def test_session_store_get_session(self):
+        auth_url = 'auth_url'
+        username = 'username'
+        password = 'password'
+        tenant_name = 'tenant_name'
+        store = SwiftSessionStore()
+        session = store.get_session(
+            auth_url=auth_url,
+            username=username,
+            password=password,
+            tenant_name=tenant_name,
+        )
+        self.assertEqual(session.auth.auth_url, auth_url)
+        self.assertEqual(session.auth.username, username)
+        self.assertEqual(session.auth.password, password)
+        self.assertEqual(session.auth.tenant_name, tenant_name)
+
+        # get the same session on a second call
+        self.assertEqual(
+            store.get_session(
+                auth_url=auth_url,
+                username=username,
+                password=password,
+                tenant_name=tenant_name,
+            ),
+            session
+        )
+
     @patch('swiftclient.client')
     def test_connection(self, mock_swift_client):
         """ Test the connection to the store"""
@@ -27,13 +60,17 @@ class TestAttachmentSwift(TestIrAttachment):
         attachment = self.Attachment
         attachment._get_swift_connection()
         mock_swift_client.Connection.assert_called_once_with(
-            authurl=os.environ.get('SWIFT_AUTH_URL'),
-            user=os.environ.get('SWIFT_ACCOUNT'),
-            key=os.environ.get('SWIFT_PASSWORD'),
-            tenant_name=os.environ.get('SWIFT_TENANT_NAME'),
-            auth_version='2.0',
+            session=mock.ANY,
             os_options={'region_name': os.environ.get('SWIFT_REGION_NAME')},
         )
+        __, kwargs = mock_swift_client.Connection.call_args
+        session = kwargs['session']
+        self.assertTrue(isinstance(session, keystoneauth1.session.Session))
+        self.assertEqual(session.auth.auth_url, os.environ['SWIFT_AUTH_URL'])
+        self.assertEqual(session.auth.username, os.environ['SWIFT_ACCOUNT'])
+        self.assertEqual(session.auth.password, os.environ['SWIFT_PASSWORD'])
+        self.assertEqual(session.auth.tenant_name,
+                         os.environ['SWIFT_TENANT_NAME'])
 
     def test_store_file_on_swift(self):
         """

requirements.txt

@@ -2,5 +2,8 @@ boto3==1.9.102
 redis==2.10.5
 python-json-logger==0.1.5
 statsd==3.2.1
-python-swiftclient==3.4.0
-python-keystoneclient==3.13.0
+python-swiftclient==3.7.0
+python-keystoneclient==3.19.0
+keystoneauth1==3.14.0
+# error with 5.x (ConstructorError: could not determine a constructor for the tag '!record')
+PyYAML==4.2b4

setup/attachment_swift/setup.py

@@ -2,5 +2,13 @@ import setuptools
 
 setuptools.setup(
     setup_requires=['setuptools-odoo'],
-    odoo_addon=True,
+    odoo_addon={
+        'external_dependencies_override': {
+            'python': {
+                'swiftclient': 'python-swiftclient>=3.7.0',
+                'keystoneclient': 'python-keystoneclient>=3.19.0',
+                'keystoneauth1': 'keystoneauth1>=3.14.0',
+            },
+        },
+    }
 )