Burtiraven2/session_redis_public
1
0
Fork 0
mirror of https://github.com/camptocamp/odoo-cloud-platform.git synced 2026-06-23 18:04:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Create base_attachment_object_storage to extract common code to store implementations

Browse Source
This commit is contained in:
Thierry Ducrest
2017-09-01 11:14:21 +02:00
committed by Guewen Baconnier
co-authored by Guewen Baconnier
parent beea07d44f
commit fea698057a
11 changed files with 360 additions and 78 deletions
Download Patch File Download Diff File Expand all files Collapse all files
attachment_swift/README.rst
+34
View File
@@ -0,0 +1,34 @@
Attachments on Swift storage
============================
This addon allows to store the attachments (documents and assets) on
OpenStack Object Storage (Swift)
Configuration
-------------
Activate Swift storage:
* Create or set the system parameter with the key ``ir_attachment.location``
and the value in the form ``swift``.
Configure accesses with environment variables:
* ``SWIFT_HOST``
* ``SWIFT_ACCOUNT``
* ``SWIFT_PASSWORD``
* ``SWIFT_WRITE_CONTAINER``
Read-only mode:
The container name and the key are stored in the attachment. So if you change the
``SWIFT_WRITE_CONTAINER`` or the ``ir_attachment.location``, the existing attachments
will still be read on their former container. But as soon as they are written over
or new attachments are created, they will be created on the new container or on
the other location (db or filesystem). This is a convenient way to be able to
read the production attachments on a replication (since you have the
credentials) without any risk to alter the production data.
This addon must be added in the server wide addons with (``--load`` option):
``--load=web,web_kanban,attachment_swift``
attachment_swift/__init__.py
+1 -1
View File
@@ -1 +1 @@
import models
from . import models
attachment_swift/__manifest__.py
+6 -3
View File
@@ -4,12 +4,15 @@
{'name': 'Attachments on Swift storage',
'summary': 'Store assets and attachments on a Swift compatible object storage',
'version': '10.0.0.0',
'summary': 'Store assets and attachments on a Swift compatible object store',
'version': '10.0.1.1.0',
'author': 'Camptocamp,Odoo Community Association (OCA)',
'license': 'AGPL-3',
'category': 'Knowledge Management',
'depends': ['base'],
'depends': ['base_attachment_object_storage'],
'external_dependencies': {
'python': ['swiftclient'],
},
'website': 'http://www.camptocamp.com',
'data': [],
'installable': True,
attachment_swift/models/ir_attachment.py
+55 -68
View File
@@ -6,23 +6,25 @@
import base64
import logging
import os
import swiftclient
from swiftclient.exceptions import ClientException
from ..swift_uri import SwiftUri
from odoo import api, exceptions, models, _
_logger = logging.getLogger(__name__)
try:
import swiftclient
from swiftclient.exceptions import ClientException
except ImportError:
swiftclient = None
ClientException = None
_logger.debug("Cannot 'import swiftclient'.")
class IrAttachment(models.Model):
_inherit = 'ir.attachment'
@api.multi
def _store_in_db_when_swift(self):
# For testing lets save everything in Swift object store
# TODO: same than in attachment_s3
return False
_SWIFT_STORAGE = 'swift'
@api.model
def _get_swift_connection(self):
@@ -31,86 +33,71 @@ class IrAttachment(models.Model):
account = os.environ.get('SWIFT_ACCOUNT')
password = os.environ.get('SWIFT_PASSWORD')
if not (host and account and password):
raise exceptions.UserError(
_('Problem connecting to Swift store, are not the env variables set ?'))
print 'Connection to host: {}, account: {}, password: {}'.format(host, account, password)
raise exceptions.UserError(_(
'''Problem connecting to Swift store, are the env variables
(SWIFT_HOST, SWIFT_ACCOUNT, SWIFT_PASSWORD) properly set ?
'''))
try:
conn = swiftclient.client.Connection(authurl=host, user=account, key=password)
conn = swiftclient.client.Connection(authurl=host,
user=account,
key=password)
except ClientException:
_logger.exception('Error connecting to Swift object store')
raise exceptions.UserError('Error connection to Swift')
raise exceptions.UserError(_('Error on Swift connection'))
return conn
@api.model
def _file_read_swift(self, fname, bin_size=False):
swifturi = SwiftUri(fname)
conn = self._get_swift_connection()
print 'Swift reading on {} of {} '.format(swifturi.container(), swifturi.item())
try:
resp_headers, obj_content = conn.get_object(swifturi.container(), swifturi.item())
read = base64.b64encode(obj_content)
except ClientException:
_logger.exception('Error reading object from Swift object store');
#raise exceptions.UserError('Error reading to Swift')
return ''
return read
@api.model
def _file_read(self, fname, bin_size=False):
def _store_file_read(self, fname, bin_size=False):
if fname.startswith('swift://'):
return self._file_read_swift(fname, bin_size=bin_size)
swifturi = SwiftUri(fname)
conn = self._get_swift_connection()
try:
resp, obj_content = conn.get_object(swifturi.container(),
swifturi.item())
read = base64.b64encode(obj_content)
except ClientException:
_logger.exception(
'Error reading object from Swift object store')
raise exceptions.UserError(_('Error reading on Swift'))
return read
else:
return super(IrAttachment, self)._store_file_read(fname, bin_size)
def _store_file_write(self, value, checksum):
if self._storage() == self._SWIFT_STORAGE:
container = os.environ.get('SWIFT_WRITE_CONTAINER')
conn = self._get_swift_connection()
conn.put_container(container)
bin_data = value.decode('base64')
key = self._compute_checksum(bin_data)
filename = 'swift://{}/{}'.format(container, key)
try:
conn.put_object(container, key, bin_data)
except ClientException:
_logger.exception('Error writing to Swift object store')
raise exceptions.UserError(_('Error writing to Swift'))
else:
_super = super(IrAttachment, self)
return _super._file_read(fname, bin_size=bin_size)
def _file_write_swift(self, value, checksum):
container = os.environ.get('SWIFT_WRITE_CONTAINER')
conn = self._get_swift_connection()
conn.put_container(container)
bin_data = value.decode('base64')
# No keys given by the store, use checksum !?
key = self._compute_checksum(bin_data)
filename = 'swift://{}/{}'.format(container, key)
print 'Saving {}'.format(filename)
try:
conn.put_object(container, key, bin_data)
except ClientException:
_logger.exception('Error connecting to Swift object store')
raise exceptions.UserError('Error writting to Swift')
return filename
def _file_write(self, value, checksum):
storage = self._storage()
if storage == 'swift':
filename = self._file_write_swift(value, checksum)
else:
filename = super(IrAttachment, self)._file_write(value, checksum)
filename = _super._store_file_write(value, checksum)
return filename
@api.model
def _file_delete(self, fname):
def _store_file_delete(self, fname):
if fname.startswith('swift://'):
swifturi = SwiftUri(fname)
container = swifturi.container()
print 'Deleting... container: {} | filename: {}'.format(container, swifturi.item())
if container == os.environ.get('SWIFT_WRITE_CONTAINER'):
conn = self._get_swift_connection()
try:
conn.delete_object(container, swifturi.item())
except ClientException as error:
_logger.exception('Error connecting to Swift object store');
raise exceptions.UserError('Error deleting in Swift')
except ClientException:
_logger.exception(
_('Error deleting an object on the Swift store'))
raise exceptions.UserError(_('Error deleting on Swift'))
else:
super(IrAttachment, self)._file_delete(fname)
@api.model
def _force_storage_swift(self, new_cr=False):
return
@api.model
def force_storage(self):
storage = self._storage()
if storage == 'swift':
self._force_storage_swift()
else:
return super(IrAttachment, self).force_storage()
def _get_stores(self):
l = [self._SWIFT_STORAGE]
l += super(IrAttachment, self)._get_stores()
return l
attachment_swift/swift_uri.py
+2 -1
View File
@@ -7,7 +7,8 @@ import re
class SwiftUri(object):
_url_re = re.compile("^swift:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE)
_url_re = re.compile("^swift:///*([^/]*)/?(.*)",
re.IGNORECASE | re.UNICODE)
def __init__(self, uri):
match = self._url_re.match(uri)
attachment_swift/tests/tests.py
+10 -5
View File
@@ -1,8 +1,8 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Camptocamp SA
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
import unittest
from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment
from ..models import ir_attachment
from ..swift_uri import SwiftUri
from swiftclient.exceptions import ClientException
@@ -11,7 +11,8 @@ class TestAttachmentSwift(TestIrAttachment):
def setup(self):
super(TestAttachmentSwift, self).setUp()
self.env['ir.config_parameter'].set_param('ir_attachment.location', 'swift')
self.env['ir.config_parameter'].set_param('ir_attachment.location',
'swift')
def test_connection(self):
""" Test the connection to the Swift object store """
@@ -19,15 +20,19 @@ class TestAttachmentSwift(TestIrAttachment):
self.assertNotEquals(conn, False)
def test_store_file_on_swift(self):
(self.env['ir.config_parameter'].
set_param('ir_attachment.location', 'swift'))
a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64})
a5bis = self.Attachment.browse(a5.id)[0]
self.assertEquals(a5.datas, a5bis.datas)
def test_delete_file_on_swift(self):
self.env['ir.config_parameter'].set_param('ir_attachment.location', 'swift')
(self.env['ir.config_parameter'].
set_param('ir_attachment.location', 'swift'))
a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64})
uri = SwiftUri(a5.store_fname)
con = self.Attachment._get_swift_connection()
data = con.get_object(uri.container(), uri.item())
con.get_object(uri.container(), uri.item())
a5.unlink()
with self.assertRaises(ClientException):
con.get_object(uri.container(), uri.item())