mirror of
https://github.com/camptocamp/odoo-cloud-platform.git
synced 2026-06-23 18:04:34 +00:00
Guewen Baconnier
9ca3f87614
When we return a bin_size (such as 28672), the web client does not accept it as as bin size and try to use it as value for the b64 src of the image. Though, returning the image as b64 in all cases does seem to be supported.
308 lines
12 KiB
Python
308 lines
12 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright 2016 Camptocamp SA
|
|
# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
|
|
|
|
|
|
import base64
|
|
import logging
|
|
import os
|
|
import xml.dom.minidom
|
|
from functools import partial
|
|
|
|
from distutils.util import strtobool
|
|
|
|
import boto
|
|
from boto.exception import S3ResponseError
|
|
|
|
from openerp import _, api, exceptions, fields, models
|
|
|
|
_logger = logging.getLogger(__name__)
|
|
|
|
|
|
class IrAttachment(models.Model):
|
|
_inherit = "ir.attachment"
|
|
|
|
datas = fields.Binary(
|
|
compute='_compute_datas',
|
|
inverse='_inverse_datas',
|
|
string='File Content',
|
|
nodrop=True,
|
|
)
|
|
|
|
@api.model
|
|
def _s3_readonly(self):
|
|
|
|
def is_true(strval):
|
|
return bool(strtobool(strval or '0'.lower()))
|
|
|
|
params = self.env['ir.config_parameter'].sudo()
|
|
storage = params.get_param('ir_attachment.location', default='')
|
|
env_ro = is_true(os.environ.get('AWS_ATTACHMENT_READONLY'))
|
|
param_ro = is_true(params.get_param('ir_attachment.s3.readonly'))
|
|
return storage.startswith('s3://') and (env_ro or param_ro)
|
|
|
|
@api.depends('store_fname', 'db_datas')
|
|
def _compute_datas(self):
|
|
bin_size = self.env.context.get('bin_size')
|
|
if self._s3_readonly():
|
|
for attach in self:
|
|
# look first in db_datas in case a file has been modified
|
|
# locally
|
|
data = attach.db_datas
|
|
if data:
|
|
attach.datas = data
|
|
else:
|
|
params = self.env['ir.config_parameter'].sudo()
|
|
bucket_url = params.get_param('ir_attachment.location')
|
|
bucket = self._get_s3_bucket(bucket_url)
|
|
attach.datas = self._file_read_s3(bucket,
|
|
attach.store_fname,
|
|
bin_size)
|
|
else:
|
|
values = self._data_get('datas', None)
|
|
for attach in self:
|
|
attach.datas = values.get(attach.id)
|
|
|
|
def _inverse_datas(self):
|
|
for attach in self:
|
|
self._data_set('datas', attach.datas, None)
|
|
|
|
@api.model
|
|
def _storage(self):
|
|
if self._s3_readonly():
|
|
# When the S3 readonly mode is active, we force the storage
|
|
# to be in the database. We'll override the read method
|
|
# to look in S3 if we have a value though.
|
|
return 'db'
|
|
else:
|
|
return super(IrAttachment, self)._storage()
|
|
|
|
@api.model
|
|
def _get_s3_bucket(self, bucket_url):
|
|
"""Connect to S3 and return the bucket
|
|
|
|
It expects the ``bucket_url`` to be in the form:
|
|
|
|
``s3://<access-key>:<secret-key>@<bucket-name>``
|
|
|
|
Alternatively, we can also use environment variables, in that case,
|
|
you must set the parameter to ``s3://``.
|
|
|
|
If the S3 provider is not AWS, the key
|
|
``ir_attachment.s3.host`` can be configured in the System
|
|
Parameters with the hostname.
|
|
|
|
The following environment variables can be set:
|
|
* ``AWS_HOST``
|
|
* ``AWS_ACCESS_KEY_ID``
|
|
* ``AWS_SECRET_ACCESS_KEY``
|
|
* ``AWS_BUCKETNAME``
|
|
|
|
"""
|
|
assert bucket_url.startswith('s3://')
|
|
host = os.environ.get('AWS_HOST')
|
|
if not host:
|
|
host = self.env['ir.config_parameter'].sudo().get_param(
|
|
'ir_attachment.s3.host', default=None
|
|
)
|
|
if host:
|
|
connect_s3 = partial(boto.connect_s3, host=host)
|
|
else:
|
|
connect_s3 = boto.connect_s3
|
|
|
|
if bucket_url == 's3://':
|
|
access_key = os.environ.get('AWS_ACCESS_KEY_ID')
|
|
secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY')
|
|
bucket_name = os.environ.get('AWS_BUCKETNAME')
|
|
if not (access_key and secret_key and bucket_name):
|
|
raise exceptions.UserError(
|
|
_('The following environment variables must be set:\n'
|
|
'* AWS_ACCESS_KEY_ID\n'
|
|
'* AWS_SECRET_ACCESS_KEY\n'
|
|
'* AWS_BUCKETNAME\n'
|
|
'* AWS_HOST (optional)\n'
|
|
)
|
|
)
|
|
else:
|
|
malformed_msg = _(
|
|
'S3 bucket %s is malformed, the expected form is '
|
|
's3://<access-key>:<secret-key>@<bucket-name>'
|
|
)
|
|
params = bucket_url[5:].split('@')
|
|
if not len(params) == 2:
|
|
raise exceptions.UserError(malformed_msg % bucket_url)
|
|
keys, bucket_name = params
|
|
keys = keys.split(':')
|
|
if not len(keys) == 2:
|
|
raise exceptions.UserError(malformed_msg % bucket_url)
|
|
access_key, secret_key = keys
|
|
|
|
try:
|
|
conn = connect_s3(aws_access_key_id=access_key,
|
|
aws_secret_access_key=secret_key)
|
|
|
|
except S3ResponseError as error:
|
|
# log verbose error from s3, return short message for user
|
|
_logger.exception('Error during connection on S3')
|
|
raise exceptions.UserError(self._parse_s3_error(error))
|
|
|
|
bucket = conn.lookup(bucket_name)
|
|
if not bucket:
|
|
bucket = conn.create_bucket(bucket_name)
|
|
return bucket
|
|
|
|
@staticmethod
|
|
def _parse_s3_error(s3error):
|
|
msg = s3error.reason
|
|
# S3 error message is a XML message...
|
|
doc = xml.dom.minidom.parseString(s3error.body)
|
|
msg_node = doc.getElementsByTagName('Message')
|
|
if msg_node:
|
|
msg = '%s: %s' % (msg, msg_node[0].childNodes[0].data)
|
|
return msg
|
|
|
|
@api.model
|
|
def _file_read_s3(self, bucket, fname, bin_size=False):
|
|
filekey = bucket.get_key(fname)
|
|
if filekey:
|
|
read = base64.b64encode(filekey.get_contents_as_string())
|
|
else:
|
|
# If the attachment has been created before the installation
|
|
# of the addon, it might be stored on the filesystem.
|
|
# Fallback on the filesystem read.
|
|
# Consider running ``force_storage()`` to move all the
|
|
# attachments on the Object Storage
|
|
try:
|
|
_super = super(IrAttachment, self)
|
|
read = _super._file_read(fname, bin_size=bin_size)
|
|
except (IOError, OSError):
|
|
# File is missing
|
|
read = ''
|
|
return read
|
|
|
|
@api.model
|
|
def _file_read(self, fname, bin_size=False):
|
|
storage = self._storage()
|
|
if storage.startswith('s3://'):
|
|
storage = self._storage()
|
|
bucket = self._get_s3_bucket(storage)
|
|
read = self._file_read_s3(bucket, fname, bin_size=bin_size)
|
|
else:
|
|
_super = super(IrAttachment, self)
|
|
read = _super._file_read(fname, bin_size=bin_size)
|
|
return read
|
|
|
|
@api.model
|
|
def _file_write(self, value, checksum):
|
|
storage = self._storage()
|
|
if storage.startswith('s3://'):
|
|
bucket = self._get_s3_bucket(storage)
|
|
bin_data = value.decode('base64')
|
|
filename = self._compute_checksum(bin_data)
|
|
filekey = bucket.get_key(filename) or bucket.new_key(filename)
|
|
try:
|
|
filekey.set_contents_from_string(bin_data)
|
|
except S3ResponseError as error:
|
|
# log verbose error from s3, return short message for user
|
|
_logger.exception(
|
|
'Error during storage of the file %s' % filename
|
|
)
|
|
raise exceptions.UserError(
|
|
_('The file could not be stored: %s') %
|
|
(self._parse_s3_error(error),)
|
|
)
|
|
else:
|
|
filename = super(IrAttachment, self)._file_write(value, checksum)
|
|
return filename
|
|
|
|
@api.model
|
|
def _file_delete(self, fname):
|
|
storage = self._storage()
|
|
if storage.startswith('s3://'):
|
|
bucket = self._get_s3_bucket(storage)
|
|
filekey = bucket.get_key(fname)
|
|
if filekey:
|
|
try:
|
|
filekey.delete()
|
|
except S3ResponseError as error:
|
|
# log verbose error from s3, return short message for user
|
|
_logger.exception(
|
|
'Error during deletion of the file %s' % fname
|
|
)
|
|
raise exceptions.UserError(
|
|
_('The file could not be deleted: %s') %
|
|
(self._parse_s3_error(error),)
|
|
)
|
|
else:
|
|
# If the attachment has been created before the installation
|
|
# of the addon, it might be stored on the filesystem.
|
|
# Fallback on the filesystem delete method.
|
|
# Consider running ``force_storage()`` to move all the
|
|
# attachments on the Object Storage
|
|
super(IrAttachment, self)._file_delete(fname)
|
|
else:
|
|
super(IrAttachment, self)._file_delete(fname)
|
|
|
|
@api.model
|
|
def force_storage(self):
|
|
if not self.env['res.users'].browse(self.env.uid)._is_admin():
|
|
raise exceptions.AccessError(
|
|
_('Only administrators can execute this action.')
|
|
)
|
|
|
|
storage = self._storage()
|
|
if storage.startswith('s3://'):
|
|
_logger.info('migrating files to the object storage')
|
|
s3_bucket = self._get_s3_bucket(storage)
|
|
domain = ['|',
|
|
('res_field', '=', False),
|
|
('res_field', '!=', False)]
|
|
ids = self.search(domain).ids
|
|
for attachment_id in ids:
|
|
# This is a trick to avoid having the 'datas' function fields
|
|
# computed for every attachment on each iteration of the loop.
|
|
# The former issue being that it reads the content of the file
|
|
# of ALL the attachments on each loop.
|
|
self.env.clear()
|
|
attachment = self.browse(attachment_id)
|
|
_logger.info('inspecting attachment %s (%d)',
|
|
attachment.name, attachment.id)
|
|
fname = attachment.store_fname
|
|
if fname:
|
|
# migrating from filestore
|
|
s3_key = s3_bucket.get_key(fname)
|
|
if s3_key:
|
|
_logger.info('file %s already on the object storage',
|
|
fname)
|
|
else:
|
|
_logger.info('moving %s on the object storage', fname)
|
|
attachment.write({'datas': attachment.datas,
|
|
# this is required otherwise the
|
|
# mimetype gets overriden with
|
|
# 'application/octet-stream'
|
|
# on assets
|
|
'mimetype': attachment.mimetype})
|
|
_logger.info('moved %s on the object storage', fname)
|
|
full_path = self._full_path(fname)
|
|
_logger.info('cleaning fs attachment')
|
|
if os.path.exists(full_path):
|
|
try:
|
|
os.unlink(full_path)
|
|
except OSError:
|
|
_logger.info(
|
|
"_file_delete could not unlink %s",
|
|
full_path, exc_info=True
|
|
)
|
|
except IOError:
|
|
# Harmless and needed for race conditions
|
|
_logger.info(
|
|
"_file_delete could not unlink %s",
|
|
full_path, exc_info=True
|
|
)
|
|
elif attachment.db_datas:
|
|
_logger.info('moving on the object storage from database')
|
|
attachment.write({'datas': attachment.datas})
|
|
self.env.cr.commit()
|
|
else:
|
|
return super(IrAttachment, self).force_storage()
|