Merge pull request #55 from guewen/swift-auth-session

attachment_swift: share a session for all connections

.travis.yml

@@ -13,7 +13,6 @@ addons:
       - python-lxml # because pip installation is slow
       - python-simplejson
       - python-serial
-      - python-yaml
 
 python:
   - '2.7'

attachment_swift/README.rst

@@ -16,6 +16,7 @@ Configure accesses with environment variables:
 * ``SWIFT_TENANT_NAME``
 * ``SWIFT_ACCOUNT``
 * ``SWIFT_PASSWORD``
+* ``SWIFT_REGION_NAME``         : optional region
 * ``SWIFT_WRITE_CONTAINER``     : Name of the container to use in the store (created if not existing)
 
 Read-only mode:
@@ -44,6 +45,7 @@ The python-swiftclient can be used from the command line, useful to test:
     export OS_USERNAME={SWIFT_ACCOUNT}
     export OS_PASSWORD={SWIFT_PASSWORD}
     export OS_TENANT_NAME={SWIFT_TENANT_NAME}
+    export SWIFT_REGION_NAME={SWIFT_REGION_NAME}
     export OS_AUTH_URL=https://auth.cloud.ovh.net/v2.0
     swift stat
 

attachment_swift/models/ir_attachment.py

@@ -14,6 +14,9 @@ _logger = logging.getLogger(__name__)
 
 try:
     import swiftclient
+    import keystoneauth1
+    import keystoneauth1.identity
+    import keystoneauth1.session
     from swiftclient.exceptions import ClientException
 except ImportError:
     swiftclient = None
@@ -21,6 +24,54 @@ except ImportError:
     _logger.debug("Cannot 'import swiftclient'.")
 
 
+SWIFT_TIMEOUT = 15
+
+
+class SwiftSessionStore(object):
+    """Keep in memory the current Swift Auth session
+
+    The auth endpoint has a rate limit on swift, if every operation
+    on the filestore authenticate, the limit is exhausted and
+    operations rejected with an HTTP error code 429.
+
+    Swift connections can reuse the same session by asking a session
+    matching their connection parameters with ``get_session``.
+
+    The keystoneauth1's session automatically creates a new token
+    if the previous one is expired.
+
+    The best documentation I found about sessions is
+    https://docs.openstack.org/keystoneauth/latest/using-sessions.html
+    """
+
+    def __init__(self):
+        self._sessions = {}
+
+    def _get_key(self, auth_url, username, password, tenant_name):
+        return (auth_url, username, password, tenant_name)
+
+    def get_session(self, auth_url=None, username=None, password=None,
+                    tenant_name=None):
+        key = self._get_key(auth_url, username, password, tenant_name)
+        session = self._sessions.get(key)
+        if not session:
+            auth = keystoneauth1.identity.v2.Password(
+                username=username,
+                password=password,
+                tenant_name=tenant_name,
+                auth_url=auth_url,
+            )
+            session = keystoneauth1.session.Session(
+                auth=auth,
+                timeout=SWIFT_TIMEOUT,
+            )
+            self._sessions[key] = session
+        return session
+
+
+swift_session_store = SwiftSessionStore()
+
+
 class IrAttachment(models.Model):
     _inherit = 'ir.attachment'
 
@@ -45,15 +96,18 @@ class IrAttachment(models.Model):
                 "Problem connecting to Swift store, are the env variables "
                 "(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, "
                 "SWIFT_TENANT_NAME) properly set?"
-                ))
+            ))
         try:
-            conn = swiftclient.client.Connection(authurl=host,
+            session = swift_session_store.get_session(
-                                                 user=account,
+                username=account,
-                                                 key=password,
+                password=password,
-                                                 tenant_name=tenant_name,
+                tenant_name=tenant_name,
-                                                 auth_version='2.0',
+                auth_url=host,
-                                                 os_options=os_options,
+            )
-                                                 )
+            conn = swiftclient.client.Connection(
+                session=session,
+                os_options=os_options,
+            )
         except ClientException:
             _logger.exception('Error connecting to Swift object store')
             raise exceptions.UserError(_('Error on Swift connection'))

requirements.txt

@@ -2,5 +2,8 @@ boto==2.42.0
 redis==2.10.5
 python-json-logger==0.1.5
 statsd==3.2.1
-python-swiftclient==3.4.0
+python-swiftclient==3.7.0
-python-keystoneclient==3.13.0
+python-keystoneclient==3.19.0
+keystoneauth1==3.14.0
+# error with 5.x (ConstructorError: could not determine a constructor for the tag '!record')
+PyYAML==4.2b4