From 85d32e44c6b6923b4e7fe1bea163f79faa1d038f Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 2 May 2019 15:28:35 +0200 Subject: [PATCH 1/3] Add missing variable in documentation example --- attachment_swift/README.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/attachment_swift/README.rst b/attachment_swift/README.rst index 8999290..a05768d 100644 --- a/attachment_swift/README.rst +++ b/attachment_swift/README.rst @@ -16,6 +16,7 @@ Configure accesses with environment variables: * ``SWIFT_TENANT_NAME`` * ``SWIFT_ACCOUNT`` * ``SWIFT_PASSWORD`` +* ``SWIFT_REGION_NAME`` : optional region * ``SWIFT_WRITE_CONTAINER`` : Name of the container to use in the store (created if not existing) Read-only mode: @@ -44,6 +45,7 @@ The python-swiftclient can be used from the command line, useful to test: export OS_USERNAME={SWIFT_ACCOUNT} export OS_PASSWORD={SWIFT_PASSWORD} export OS_TENANT_NAME={SWIFT_TENANT_NAME} + export SWIFT_REGION_NAME={SWIFT_REGION_NAME} export OS_AUTH_URL=https://auth.cloud.ovh.net/v2.0 swift stat From 4e919d85cbd7ee0c88f53aa69b5987750f7781be Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 2 May 2019 15:29:26 +0200 Subject: [PATCH 2/3] attachment_swift: share a session for all connections OVH's Swift applies a rate limit on the authentication. attachment_swift authenticates again each time it has to read/write an attachment. When running upgrades on upgrades of files or installing a new DB, at some point, we get rejected with HTTP 429. This commit introduces a shared storage for Swift Session. All connections will reuses the same authentication token created the first time a connection needs a Session. Note: needs python-swiftclient>=3.7.0 to have https://github.com/openstack/python-swiftclient/commit/1971ef880ff225379d4a91f00f89f323a1605eeb --- attachment_swift/models/ir_attachment.py | 70 +++++++++++++++++++++--- requirements.txt | 5 +- 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/attachment_swift/models/ir_attachment.py b/attachment_swift/models/ir_attachment.py index e307a8a..7616def 100644 --- a/attachment_swift/models/ir_attachment.py +++ b/attachment_swift/models/ir_attachment.py @@ -14,6 +14,9 @@ _logger = logging.getLogger(__name__) try: import swiftclient + import keystoneauth1 + import keystoneauth1.identity + import keystoneauth1.session from swiftclient.exceptions import ClientException except ImportError: swiftclient = None @@ -21,6 +24,54 @@ except ImportError: _logger.debug("Cannot 'import swiftclient'.") +SWIFT_TIMEOUT = 15 + + +class SwiftSessionStore(object): + """Keep in memory the current Swift Auth session + + The auth endpoint has a rate limit on swift, if every operation + on the filestore authenticate, the limit is exhausted and + operations rejected with an HTTP error code 429. + + Swift connections can reuse the same session by asking a session + matching their connection parameters with ``get_session``. + + The keystoneauth1's session automatically creates a new token + if the previous one is expired. + + The best documentation I found about sessions is + https://docs.openstack.org/keystoneauth/latest/using-sessions.html + """ + + def __init__(self): + self._sessions = {} + + def _get_key(self, auth_url, username, password, tenant_name): + return (auth_url, username, password, tenant_name) + + def get_session(self, auth_url=None, username=None, password=None, + tenant_name=None): + key = self._get_key(auth_url, username, password, tenant_name) + session = self._sessions.get(key) + if not session: + auth = keystoneauth1.identity.v2.Password( + username=username, + password=password, + tenant_name=tenant_name, + auth_url=auth_url, + ) + session = keystoneauth1.session.Session( + auth=auth, + timeout=SWIFT_TIMEOUT, + ) + self._sessions[key] = session + return session + + +swift_session_store = SwiftSessionStore() + + class IrAttachment(models.Model): _inherit = 'ir.attachment' @@ -45,15 +96,18 @@ class IrAttachment(models.Model): "Problem connecting to Swift store, are the env variables " "(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, " "SWIFT_TENANT_NAME) properly set?" - )) + )) try: - conn = swiftclient.client.Connection(authurl=host, - user=account, - key=password, - tenant_name=tenant_name, - auth_version='2.0', - os_options=os_options, - ) + session = swift_session_store.get_session( + username=account, + password=password, + tenant_name=tenant_name, + auth_url=host, + ) + conn = swiftclient.client.Connection( + session=session, + os_options=os_options, + ) except ClientException: _logger.exception('Error connecting to Swift object store') raise exceptions.UserError(_('Error on Swift connection')) diff --git a/requirements.txt b/requirements.txt index e3c77a7..ec8459a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,5 +2,6 @@ boto==2.42.0 redis==2.10.5 python-json-logger==0.1.5 statsd==3.2.1 -python-swiftclient==3.4.0 -python-keystoneclient==3.13.0 +python-swiftclient==3.7.0 +python-keystoneclient==3.19.0 +keystoneauth1==3.14.0 From 3e20f1563f354796ff6ce921a5108e5f995dee2a Mon Sep 17 00:00:00 2001 From: Guewen Baconnier Date: Thu, 2 May 2019 16:14:47 +0200 Subject: [PATCH 3/3] Pin PyYAML version because 5.x does not work with Odoo It would raise ConstructorError: could not determine a constructor for the tag '!record' --- .travis.yml | 1 - requirements.txt | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2655a84..d4ce526 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,6 @@ addons: - python-lxml # because pip installation is slow - python-simplejson - python-serial - - python-yaml python: - '2.7' diff --git a/requirements.txt b/requirements.txt index ec8459a..9d0c28d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,3 +5,5 @@ statsd==3.2.1 python-swiftclient==3.7.0 python-keystoneclient==3.19.0 keystoneauth1==3.14.0 +# error with 5.x (ConstructorError: could not determine a constructor for the tag '!record') +PyYAML==4.2b4