Burtiraven2/session_redis_public
1
0
Fork 0
mirror of https://github.com/camptocamp/odoo-cloud-platform.git synced 2026-06-24 08:47:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #156 from p-tombez/8.0-keystoneauth_v3

Browse Source 
[8.0] Use keystoneauth v3 for Swift attachments
This commit is contained in:
Patrick Tombez
2020-03-04 13:24:57 +01:00
committed by GitHub
co-authored by GitHub
parent 0c9d5d6abe 9d6e38381d
commit cca45667ec
5 changed files with 244 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files
attachment_swift/README.rst
+12 -7
View File
@@ -1,7 +1,7 @@
Attachments on Swift storage
============================
This addon enables storing attachments (documents and assets) on OpenStack Object Storage (Swift).
This addon enable storing attachments (documents and assets) on OpenStack Object Storage (Swift)
Configuration
-------------
@@ -13,9 +13,11 @@ Activate Swift storage:
Configure accesses with environment variables:
* ``SWIFT_AUTH_URL`` : URL of the Swift server
* ``SWIFT_TENANT_NAME``
* ``SWIFT_TENANT_NAME`` : **!** DEPRECATED **!** Use ``SWIFT_PROJECT_NAME`` instead
* ``SWIFT_PROJECT_NAME``
* ``SWIFT_ACCOUNT``
* ``SWIFT_PASSWORD``
* ``SWIFT_REGION_NAME`` : optional region
* ``SWIFT_WRITE_CONTAINER`` : Name of the container to use in the store (created if not existing)
Read-only mode:
@@ -30,21 +32,24 @@ credentials) without any risk to alter the production data.
This addon must be added in the server wide addons with (``--load`` option):
``--load=web,web_kanban,attachment_swift``
``--load=web,attachment_swift``
Python Dependencies
-------------------
This module needs the python-swiftclient and the python-keystoneclient (For auth v2.0) to work.
This module needs the python-swiftclient and the python-keystoneclient (For auth v3.0) to work.
The python-keystoneclient needs the linux package build-essential and python-dev to install properly.
The python-swiftclient can be used from the command line, useful to test:
export AUTH_VERSION=2.0
.. code-block:: sh
export AUTH_VERSION=3.0
export OS_USERNAME={SWIFT_ACCOUNT}
export OS_PASSWORD={SWIFT_PASSWORD}
export OS_TENANT_NAME={SWIFT_TENANT_NAME}
export OS_AUTH_URL=https://auth.cloud.ovh.net/v2.0
export OS_PROJECT_NAME={SWIFT_PROJECT_NAME}
export OS_REGION_NAME={SWIFT_REGION_NAME}
export OS_AUTH_URL=https://auth.cloud.ovh.net/v3
swift stat
More information at
attachment_swift/models/ir_attachment.py
+68 -11
View File
@@ -16,6 +16,7 @@ _logger = logging.getLogger(__name__)
try:
import swiftclient
import keystoneauth1
from swiftclient.exceptions import ClientException
except ImportError:
swiftclient = None
@@ -23,6 +24,54 @@ except ImportError:
_logger.debug("Cannot 'import swiftclient'.")
SWIFT_TIMEOUT = 15
class SwiftSessionStore(object):
"""Keep in memory the current Swift Auth session
The auth endpoint has a rate limit on swift, if every operation
on the filestore authenticate, the limit is exhausted and
operations rejected with an HTTP error code 429.
Swift connections can reuse the same session by asking a session
matching their connection parameters with ``get_session``.
The keystoneauth1's session automatically creates a new token
if the previous one is expired.
The best documentation I found about sessions is
https://docs.openstack.org/keystoneauth/latest/using-sessions.html
"""
def __init__(self):
self._sessions = {}
def _get_key(self, auth_url, username, password, project_name):
return (auth_url, username, password, project_name)
def get_session(self, auth_url=None, username=None, password=None,
project_name=None):
key = self._get_key(auth_url, username, password, project_name)
session = self._sessions.get(key)
if not session:
auth = keystoneauth1.identity.v3.Password(
username=username,
password=password,
project_name=project_name,
auth_url=auth_url,
)
session = keystoneauth1.session.Session(
auth=auth,
timeout=SWIFT_TIMEOUT,
)
self._sessions[key] = session
return session
swift_session_store = SwiftSessionStore()
class IrAttachment(osv.osv):
_inherit = 'ir.attachment'
@@ -34,26 +83,34 @@ class IrAttachment(osv.osv):
host = os.environ.get('SWIFT_AUTH_URL')
account = os.environ.get('SWIFT_ACCOUNT')
password = os.environ.get('SWIFT_PASSWORD')
tenant_name = os.environ.get('SWIFT_TENANT_NAME')
project_name = os.environ.get('SWIFT_PROJECT_NAME')
if not project_name and os.environ.get('SWIFT_TENANT_NAME'):
project_name = os.environ['SWIFT_TENANT_NAME']
_logger.warning(
"SWIFT_TENANT_NAME is deprecated and "
"must be replaced by SWIFT_PROJECT_NAME"
)
region = os.environ.get('SWIFT_REGION_NAME')
os_options = {}
if region:
os_options['region_name'] = region
if not (host and account and password and tenant_name):
if not (host and account and password and project_name):
raise except_orm(
_("Error"),
_("Problem connecting to Swift store, are the env variables "
"(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, "
"SWIFT_TENANT_NAME) properly set?")
)
"SWIFT_TENANT_NAME) properly set?"))
try:
conn = swiftclient.client.Connection(authurl=host,
user=account,
key=password,
tenant_name=tenant_name,
auth_version='2.0',
os_options=os_options,
)
session = swift_session_store.get_session(
username=account,
password=password,
project_name=project_name,
auth_url=host,
)
conn = swiftclient.client.Connection(
session=session,
os_options=os_options,
)
except ClientException:
_logger.exception('Error connecting to Swift object store')
raise except_orm(
attachment_swift/tests/__init__.py
+2
View File
@@ -0,0 +1,2 @@
from . import test_mock_swift_api
attachment_swift/tests/test_mock_swift_api.py
+120
View File
@@ -0,0 +1,120 @@
# Copyright 2017-2019 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
import base64
import mock
import os
from mock import patch
import keystoneauth1
from openerp.addons.base.tests.test_ir_attachment import TestIrAttachment
from openerp.addons.attachment_swift.models.ir_attachment import SwiftSessionStore
from openerp.addons.attachment_swift.swift_uri import SwiftUri
class TestAttachmentSwift(TestIrAttachment):
def setup(self):
super().setUp()
self.env['ir.config_parameter'].set_param('ir_attachment.location',
'swift')
def test_session_store_get_session(self):
auth_url = 'auth_url'
username = 'username'
password = 'password'
project_name = 'project_name'
store = SwiftSessionStore()
session = store.get_session(
auth_url=auth_url,
username=username,
password=password,
project_name=project_name,
)
self.assertEqual(session.auth.auth_url, auth_url)
self.assertEqual(session.auth.get_cache_id_elements().get(
'password_username'), username)
self.assertEqual(session.auth.get_cache_id_elements().get(
'password_password'), password)
self.assertEqual(session.auth.project_name, project_name)
# get the same session on a second call
self.assertEqual(
store.get_session(
auth_url=auth_url,
username=username,
password=password,
project_name=project_name,
),
session
)
@patch('swiftclient.client')
def test_connection(self, mock_swift_client):
""" Test the connection to the store"""
os.environ['SWIFT_AUTH_URL'] = 'auth_url'
os.environ['SWIFT_ACCOUNT'] = 'account'
os.environ['SWIFT_PASSWORD'] = 'password'
os.environ['SWIFT_PROJECT_NAME'] = 'project_name'
os.environ['SWIFT_REGION_NAME'] = 'NOWHERE'
attachment = self.Attachment
attachment._get_swift_connection()
mock_swift_client.Connection.assert_called_once_with(
session=mock.ANY,
os_options={'region_name': os.environ.get('SWIFT_REGION_NAME')},
)
__, kwargs = mock_swift_client.Connection.call_args
session = kwargs['session']
self.assertTrue(isinstance(session, keystoneauth1.session.Session))
self.assertEqual(session.auth.auth_url, os.environ['SWIFT_AUTH_URL'])
self.assertEqual(session.auth.get_cache_id_elements().get(
'password_username'), os.environ['SWIFT_ACCOUNT'])
self.assertEqual(session.auth.get_cache_id_elements().get(
'password_password'), os.environ['SWIFT_PASSWORD'])
self.assertEqual(session.auth.project_name,
os.environ['SWIFT_PROJECT_NAME'])
def test_store_file_on_swift(self):
"""
Test writing a file
"""
(self.env['ir.config_parameter'].
set_param('ir_attachment.location', 'swift'))
os.environ['SWIFT_AUTH_URL'] = 'auth_url'
os.environ['SWIFT_ACCOUNT'] = 'account'
os.environ['SWIFT_PASSWORD'] = 'password'
os.environ['SWIFT_PROJECT_NAME'] = 'project_name'
os.environ['SWIFT_WRITE_CONTAINER'] = 'my_container'
container = os.environ.get('SWIFT_WRITE_CONTAINER')
attachment = self.Attachment
bin_data = base64.b64decode(self.blob1_b64)
with patch('swiftclient.client.Connection') as MockConnection:
conn = MockConnection.return_value
attachment.create({'name': 'a5', 'datas': self.blob1_b64})
conn.put_object.assert_called_with(
container,
attachment._compute_checksum(bin_data),
bin_data)
def test_delete_file_on_swift(self):
"""
Test deleting a file
"""
(self.env['ir.config_parameter'].
set_param('ir_attachment.location', 'swift'))
os.environ['SWIFT_AUTH_URL'] = 'auth_url'
os.environ['SWIFT_ACCOUNT'] = 'account'
os.environ['SWIFT_PASSWORD'] = 'password'
os.environ['SWIFT_PROJECT_NAME'] = 'project_name'
os.environ['SWIFT_WRITE_CONTAINER'] = 'my_container'
attachment = self.Attachment
container = os.environ.get('SWIFT_WRITE_CONTAINER')
with patch('swiftclient.client.Connection') as MockConnection:
conn = MockConnection.return_value
a5 = attachment.create({'name': 'a5', 'datas': self.blob1_b64})
uri = SwiftUri(a5.store_fname)
a5.unlink()
conn.delete_object.assert_called_with(container, uri.item())
attachment_swift/tests/test_with_swift_store.py
+42
View File
@@ -0,0 +1,42 @@
# Copyright 2017-2019 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
from openerp.addons.base.tests.test_ir_attachment import TestIrAttachment
from ..swift_uri import SwiftUri
from swiftclient.exceptions import ClientException
class TestAttachmentSwift(TestIrAttachment):
"""
Those tests are made to be run against a real Swift store (local or remote)
"""
def setup(self):
super().setUp()
self.env['ir.config_parameter'].set_param('ir_attachment.location',
'swift')
def test_connection(self):
""" Test the connection to the Swift object store """
conn = self.Attachment._get_swift_connection()
self.assertNotEqual(conn, False)
def test_store_file_on_swift(self):
""" Test writing a file and then reading it """
(self.env['ir.config_parameter'].
set_param('ir_attachment.location', 'swift'))
a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64})
a5bis = self.Attachment.browse(a5.id)[0]
self.assertEqual(a5.datas, a5bis.datas)
def test_delete_file_on_swift(self):
""" Create a file and then test the deletion """
(self.env['ir.config_parameter'].
set_param('ir_attachment.location', 'swift'))
a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64})
uri = SwiftUri(a5.store_fname)
con = self.Attachment._get_swift_connection()
con.get_object(uri.container(), uri.item())
a5.unlink()
with self.assertRaises(ClientException):
con.get_object(uri.container(), uri.item())