Merge pull request #157 from p-tombez/7.0-keystoneauth_v3

[7.0] Use keystoneauth v3 for Swift attachments

attachment_swift/README.rst

@@ -1,7 +1,7 @@
 Attachments on Swift storage
 ============================
 
-This addon enables storing attachments (documents and assets) on OpenStack Object Storage (Swift).
+This addon enable storing attachments (documents and assets) on OpenStack Object Storage (Swift)
 
 Configuration
 -------------
@@ -13,9 +13,11 @@ Activate Swift storage:
 Configure accesses with environment variables:
 
 * ``SWIFT_AUTH_URL``            : URL of the Swift server
-* ``SWIFT_TENANT_NAME``
+* ``SWIFT_TENANT_NAME``         : **!** DEPRECATED **!** Use ``SWIFT_PROJECT_NAME`` instead
+* ``SWIFT_PROJECT_NAME``
 * ``SWIFT_ACCOUNT``
 * ``SWIFT_PASSWORD``
+* ``SWIFT_REGION_NAME``         : optional region
 * ``SWIFT_WRITE_CONTAINER``     : Name of the container to use in the store (created if not existing)
 
 Read-only mode:
@@ -30,21 +32,24 @@ credentials) without any risk to alter the production data.
 
 This addon must be added in the server wide addons with (``--load`` option):
 
-``--load=web,web_kanban,attachment_swift``
+``--load=web,attachment_swift``
 
 Python Dependencies
 -------------------
 
-This module needs the python-swiftclient and the python-keystoneclient (For auth v2.0) to work.
+This module needs the python-swiftclient and the python-keystoneclient (For auth v3.0) to work.
 The python-keystoneclient needs the linux package build-essential and python-dev to install properly.
 
 The python-swiftclient can be used from the command line, useful to test:
 
-    export AUTH_VERSION=2.0
+.. code-block:: sh
+
+    export AUTH_VERSION=3.0
     export OS_USERNAME={SWIFT_ACCOUNT}
     export OS_PASSWORD={SWIFT_PASSWORD}
-    export OS_TENANT_NAME={SWIFT_TENANT_NAME}
+    export OS_PROJECT_NAME={SWIFT_PROJECT_NAME}
-    export OS_AUTH_URL=https://auth.cloud.ovh.net/v2.0
+    export OS_REGION_NAME={SWIFT_REGION_NAME}
+    export OS_AUTH_URL=https://auth.cloud.ovh.net/v3
     swift stat
 
 More information at

attachment_swift/models/ir_attachment.py

@@ -16,6 +16,7 @@ _logger = logging.getLogger(__name__)
 
 try:
     import swiftclient
+    import keystoneauth1
     from swiftclient.exceptions import ClientException
 except ImportError:
     swiftclient = None
@@ -23,6 +24,54 @@ except ImportError:
     _logger.debug("Cannot 'import swiftclient'.")
 
 
+SWIFT_TIMEOUT = 15
+
+
+class SwiftSessionStore(object):
+    """Keep in memory the current Swift Auth session
+
+    The auth endpoint has a rate limit on swift, if every operation
+    on the filestore authenticate, the limit is exhausted and
+    operations rejected with an HTTP error code 429.
+
+    Swift connections can reuse the same session by asking a session
+    matching their connection parameters with ``get_session``.
+
+    The keystoneauth1's session automatically creates a new token
+    if the previous one is expired.
+
+    The best documentation I found about sessions is
+    https://docs.openstack.org/keystoneauth/latest/using-sessions.html
+    """
+
+    def __init__(self):
+        self._sessions = {}
+
+    def _get_key(self, auth_url, username, password, project_name):
+        return (auth_url, username, password, project_name)
+
+    def get_session(self, auth_url=None, username=None, password=None,
+                    project_name=None):
+        key = self._get_key(auth_url, username, password, project_name)
+        session = self._sessions.get(key)
+        if not session:
+            auth = keystoneauth1.identity.v3.Password(
+                username=username,
+                password=password,
+                project_name=project_name,
+                auth_url=auth_url,
+            )
+            session = keystoneauth1.session.Session(
+                auth=auth,
+                timeout=SWIFT_TIMEOUT,
+            )
+            self._sessions[key] = session
+        return session
+
+
+swift_session_store = SwiftSessionStore()
+
+
 class IrAttachment(osv.osv):
     _inherit = 'ir.attachment'
 
@@ -34,24 +83,32 @@ class IrAttachment(osv.osv):
         host = os.environ.get('SWIFT_AUTH_URL')
         account = os.environ.get('SWIFT_ACCOUNT')
         password = os.environ.get('SWIFT_PASSWORD')
-        tenant_name = os.environ.get('SWIFT_TENANT_NAME')
+        project_name = os.environ.get('SWIFT_PROJECT_NAME')
+        if not project_name and os.environ.get('SWIFT_TENANT_NAME'):
+            project_name = os.environ['SWIFT_TENANT_NAME']
+            _logger.warning(
+                "SWIFT_TENANT_NAME is deprecated and "
+                "must be replaced by SWIFT_PROJECT_NAME"
+            )
         region = os.environ.get('SWIFT_REGION_NAME')
         os_options = {}
         if region:
             os_options['region_name'] = region
-        if not (host and account and password and tenant_name):
+        if not (host and account and password and project_name):
             raise except_orm(
                 _("Error"),
                 _("Problem connecting to Swift store, are the env variables "
                   "(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, "
-                  "SWIFT_TENANT_NAME) properly set?")
+                  "SWIFT_TENANT_NAME) properly set?"))
-            )
         try:
-            conn = swiftclient.client.Connection(authurl=host,
+            session = swift_session_store.get_session(
-                                                 user=account,
+                username=account,
-                                                 key=password,
+                password=password,
-                                                 tenant_name=tenant_name,
+                project_name=project_name,
-                                                 auth_version='2.0',
+                auth_url=host,
+            )
+            conn = swiftclient.client.Connection(
+                session=session,
                 os_options=os_options,
             )
         except ClientException:

attachment_swift/tests/__init__.py

@@ -0,0 +1,2 @@
+
+from . import test_mock_swift_api

attachment_swift/tests/test_mock_swift_api.py

@@ -0,0 +1,120 @@
+# Copyright 2017-2019 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import base64
+import mock
+import os
+
+from mock import patch
+
+import keystoneauth1
+
+from openerp.addons.base.tests.test_ir_attachment import TestIrAttachment
+from openerp.addons.attachment_swift.models.ir_attachment import SwiftSessionStore
+from openerp.addons.attachment_swift.swift_uri import SwiftUri
+
+
+class TestAttachmentSwift(TestIrAttachment):
+
+    def setup(self):
+        super().setUp()
+        self.env['ir.config_parameter'].set_param('ir_attachment.location',
+                                                  'swift')
+
+    def test_session_store_get_session(self):
+        auth_url = 'auth_url'
+        username = 'username'
+        password = 'password'
+        project_name = 'project_name'
+        store = SwiftSessionStore()
+        session = store.get_session(
+            auth_url=auth_url,
+            username=username,
+            password=password,
+            project_name=project_name,
+        )
+        self.assertEqual(session.auth.auth_url, auth_url)
+        self.assertEqual(session.auth.get_cache_id_elements().get(
+            'password_username'), username)
+        self.assertEqual(session.auth.get_cache_id_elements().get(
+            'password_password'), password)
+        self.assertEqual(session.auth.project_name, project_name)
+
+        # get the same session on a second call
+        self.assertEqual(
+            store.get_session(
+                auth_url=auth_url,
+                username=username,
+                password=password,
+                project_name=project_name,
+            ),
+            session
+        )
+
+    @patch('swiftclient.client')
+    def test_connection(self, mock_swift_client):
+        """ Test the connection to the store"""
+        os.environ['SWIFT_AUTH_URL'] = 'auth_url'
+        os.environ['SWIFT_ACCOUNT'] = 'account'
+        os.environ['SWIFT_PASSWORD'] = 'password'
+        os.environ['SWIFT_PROJECT_NAME'] = 'project_name'
+        os.environ['SWIFT_REGION_NAME'] = 'NOWHERE'
+        attachment = self.Attachment
+        attachment._get_swift_connection()
+        mock_swift_client.Connection.assert_called_once_with(
+            session=mock.ANY,
+            os_options={'region_name': os.environ.get('SWIFT_REGION_NAME')},
+        )
+        __, kwargs = mock_swift_client.Connection.call_args
+        session = kwargs['session']
+        self.assertTrue(isinstance(session, keystoneauth1.session.Session))
+        self.assertEqual(session.auth.auth_url, os.environ['SWIFT_AUTH_URL'])
+        self.assertEqual(session.auth.get_cache_id_elements().get(
+            'password_username'), os.environ['SWIFT_ACCOUNT'])
+        self.assertEqual(session.auth.get_cache_id_elements().get(
+            'password_password'), os.environ['SWIFT_PASSWORD'])
+        self.assertEqual(session.auth.project_name,
+                         os.environ['SWIFT_PROJECT_NAME'])
+
+    def test_store_file_on_swift(self):
+        """
+            Test writing a file
+        """
+        (self.env['ir.config_parameter'].
+            set_param('ir_attachment.location', 'swift'))
+        os.environ['SWIFT_AUTH_URL'] = 'auth_url'
+        os.environ['SWIFT_ACCOUNT'] = 'account'
+        os.environ['SWIFT_PASSWORD'] = 'password'
+        os.environ['SWIFT_PROJECT_NAME'] = 'project_name'
+        os.environ['SWIFT_WRITE_CONTAINER'] = 'my_container'
+        container = os.environ.get('SWIFT_WRITE_CONTAINER')
+        attachment = self.Attachment
+        bin_data = base64.b64decode(self.blob1_b64)
+        with patch('swiftclient.client.Connection') as MockConnection:
+            conn = MockConnection.return_value
+            attachment.create({'name': 'a5', 'datas': self.blob1_b64})
+            conn.put_object.assert_called_with(
+                container,
+                attachment._compute_checksum(bin_data),
+                bin_data)
+
+    def test_delete_file_on_swift(self):
+        """
+            Test deleting a file
+        """
+        (self.env['ir.config_parameter'].
+            set_param('ir_attachment.location', 'swift'))
+        os.environ['SWIFT_AUTH_URL'] = 'auth_url'
+        os.environ['SWIFT_ACCOUNT'] = 'account'
+        os.environ['SWIFT_PASSWORD'] = 'password'
+        os.environ['SWIFT_PROJECT_NAME'] = 'project_name'
+        os.environ['SWIFT_WRITE_CONTAINER'] = 'my_container'
+
+        attachment = self.Attachment
+        container = os.environ.get('SWIFT_WRITE_CONTAINER')
+        with patch('swiftclient.client.Connection') as MockConnection:
+            conn = MockConnection.return_value
+            a5 = attachment.create({'name': 'a5', 'datas': self.blob1_b64})
+            uri = SwiftUri(a5.store_fname)
+            a5.unlink()
+            conn.delete_object.assert_called_with(container, uri.item())

attachment_swift/tests/test_with_swift_store.py

@@ -0,0 +1,42 @@
+# Copyright 2017-2019 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+from openerp.addons.base.tests.test_ir_attachment import TestIrAttachment
+from ..swift_uri import SwiftUri
+from swiftclient.exceptions import ClientException
+
+
+class TestAttachmentSwift(TestIrAttachment):
+    """
+    Those tests are made to be run against a real Swift store (local or remote)
+    """
+
+    def setup(self):
+        super().setUp()
+        self.env['ir.config_parameter'].set_param('ir_attachment.location',
+                                                  'swift')
+
+    def test_connection(self):
+        """ Test the connection to the Swift object store """
+        conn = self.Attachment._get_swift_connection()
+        self.assertNotEqual(conn, False)
+
+    def test_store_file_on_swift(self):
+        """ Test writing a file and then reading it """
+        (self.env['ir.config_parameter'].
+            set_param('ir_attachment.location', 'swift'))
+        a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64})
+        a5bis = self.Attachment.browse(a5.id)[0]
+        self.assertEqual(a5.datas, a5bis.datas)
+
+    def test_delete_file_on_swift(self):
+        """ Create a file and then test the deletion """
+        (self.env['ir.config_parameter'].
+            set_param('ir_attachment.location', 'swift'))
+        a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64})
+        uri = SwiftUri(a5.store_fname)
+        con = self.Attachment._get_swift_connection()
+        con.get_object(uri.container(), uri.item())
+        a5.unlink()
+        with self.assertRaises(ClientException):
+            con.get_object(uri.container(), uri.item())