diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index df9807c..106246e 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -1,10 +1,10 @@
exclude: |
(?x)
# NOT INSTALLABLE ADDONS
+ ^attachment_s3/|
^base_fileurl_field/|
^monitoring_log_requests/|
^monitoring_statsd/|
- ^test_base_fileurl_field/|
# END NOT INSTALLABLE ADDONS
# Files and folders generated by bots, to avoid loops
^setup/|/static/description/index\.html$|
diff --git a/attachment_s3/README.rst b/attachment_s3/README.rst
new file mode 100644
index 0000000..0d03699
--- /dev/null
+++ b/attachment_s3/README.rst
@@ -0,0 +1,58 @@
+Attachments on S3 storage
+=========================
+
+This addon allows to store the attachments (documents and assets) on S3 or any
+other S3-compatible Object Storage.
+
+Configuration
+-------------
+
+Activate S3 storage:
+
+* Create or set the system parameter with the key ``ir_attachment.location``
+ and the value in the form ``s3``.
+
+Configure accesses with environment variables:
+
+* ``AWS_HOST`` (not required if using AWS services)
+* ``AWS_REGION`` (required if using AWS services)
+* ``AWS_ACCESS_KEY_ID``
+* ``AWS_SECRET_ACCESS_KEY``
+* ``AWS_BUCKETNAME`` (optional {db} placeholder)
+
+Read-only mode:
+
+The bucket and the file key are stored in the attachment. So if you change the
+``AWS_BUCKETNAME`` or the ``ir_attachment.location``, the existing attachments
+will still be read on their former bucket. But as soon as they are written over
+or new attachments are created, they will be created on the new bucket or on
+the other location (db or filesystem). This is a convenient way to be able to
+read the production attachments on a replication (since you have the
+credentials) without any risk to alter the production data.
+
+This addon must be added in the server wide addons with (``--load`` option):
+
+``--load=web,attachment_s3``
+
+The System Parameter ``ir_attachment.storage.force.database`` can be customized to
+force storage of files in the database. See the documentation of the module
+``base_attachment_object_storage``.
+
+Multi-tenancy
+-------------
+
+Use the `{db}` placeholder to handle multi-tenancy.
+
+On instances that hold multiple databases, it's preferable to have one bucket per database.
+
+To handle this, you can insert the `{db}` placeholder in your bucket name variable ``AWS_BUCKETNAME``.
+It will be replaced by the database name.
+This will give you a unique bucketname per database.
+
+
+Limitations
+-----------
+
+* You need to call ``env['ir.attachment'].force_storage()`` after
+ having changed the ``ir_attachment.location`` configuration in order to
+ migrate the existing attachments to S3.
diff --git a/attachment_s3/__init__.py b/attachment_s3/__init__.py
new file mode 100644
index 0000000..0650744
--- /dev/null
+++ b/attachment_s3/__init__.py
@@ -0,0 +1 @@
+from . import models
diff --git a/attachment_s3/__manifest__.py b/attachment_s3/__manifest__.py
new file mode 100644
index 0000000..7958e62
--- /dev/null
+++ b/attachment_s3/__manifest__.py
@@ -0,0 +1,19 @@
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{
+ "name": "Attachments on S3 storage",
+ "summary": "Store assets and attachments on a S3 compatible object storage",
+ "version": "15.0.1.0.0",
+ "author": "Camptocamp,Odoo Community Association (OCA)",
+ "license": "AGPL-3",
+ "category": "Knowledge Management",
+ "depends": ["base", "base_attachment_object_storage"],
+ "external_dependencies": {
+ "python": ["boto3"],
+ },
+ "website": "https://github.com/camptocamp/odoo-cloud-platform",
+ "data": [],
+ "installable": False,
+}
diff --git a/attachment_s3/models/__init__.py b/attachment_s3/models/__init__.py
new file mode 100644
index 0000000..aaf38a1
--- /dev/null
+++ b/attachment_s3/models/__init__.py
@@ -0,0 +1 @@
+from . import ir_attachment
diff --git a/attachment_s3/models/ir_attachment.py b/attachment_s3/models/ir_attachment.py
new file mode 100644
index 0000000..d18c0e2
--- /dev/null
+++ b/attachment_s3/models/ir_attachment.py
@@ -0,0 +1,177 @@
+# Copyright 2016-2019 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+import io
+import logging
+import os
+from urllib.parse import urlsplit
+
+from odoo import _, api, exceptions, models
+
+from ..s3uri import S3Uri
+
+_logger = logging.getLogger(__name__)
+
+try:
+ import boto3
+ from botocore.exceptions import ClientError, EndpointConnectionError
+except ImportError:
+ boto3 = None # noqa
+ ClientError = None # noqa
+ EndpointConnectionError = None # noqa
+ _logger.debug("Cannot 'import boto3'.")
+
+
+class IrAttachment(models.Model):
+ _inherit = "ir.attachment"
+
+ def _get_stores(self):
+ return ["s3"] + super()._get_stores()
+
+ @api.model
+ def _get_s3_bucket(self, name=None):
+ """Connect to S3 and return the bucket
+
+ The following environment variables can be set:
+ * ``AWS_HOST``
+ * ``AWS_REGION``
+ * ``AWS_ACCESS_KEY_ID``
+ * ``AWS_SECRET_ACCESS_KEY``
+ * ``AWS_BUCKETNAME``
+
+ If a name is provided, we'll read this bucket, otherwise, the bucket
+ from the environment variable ``AWS_BUCKETNAME`` will be read.
+
+ """
+ host = os.environ.get("AWS_HOST")
+
+ # Ensure host is prefixed with a scheme (use https as default)
+ if host and not urlsplit(host).scheme:
+ host = "https://%s" % host
+
+ region_name = os.environ.get("AWS_REGION")
+ access_key = os.environ.get("AWS_ACCESS_KEY_ID")
+ secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY")
+ bucket_name = name or os.environ.get("AWS_BUCKETNAME")
+ # replaces {db} by the database name to handle multi-tenancy
+ bucket_name = bucket_name.format(db=self.env.cr.dbname)
+
+ params = {
+ "aws_access_key_id": access_key,
+ "aws_secret_access_key": secret_key,
+ }
+ if host:
+ params["endpoint_url"] = host
+ if region_name:
+ params["region_name"] = region_name
+ if not (access_key and secret_key and bucket_name):
+ msg = _(
+ "If you want to read from the %(bucket_name)s S3 bucket, the following "
+ "environment variables must be set:\n"
+ "* AWS_ACCESS_KEY_ID\n"
+ "* AWS_SECRET_ACCESS_KEY\n"
+ "If you want to write in the %(bucket_name)s S3 bucket, this variable "
+ "must be set as well:\n"
+ "* AWS_BUCKETNAME\n"
+ "Optionally, the S3 host can be changed with:\n"
+ "* AWS_HOST\n"
+ ).format(bucket_name=bucket_name)
+
+ raise exceptions.UserError(msg)
+ # try:
+ s3 = boto3.resource("s3", **params)
+ bucket = s3.Bucket(bucket_name)
+ exists = True
+ try:
+ s3.meta.client.head_bucket(Bucket=bucket_name)
+ except ClientError as e:
+ # If a client error is thrown, then check that it was a 404 error.
+ # If it was a 404 error, then the bucket does not exist.
+ error_code = e.response["Error"]["Code"]
+ if error_code == "404":
+ exists = False
+ except EndpointConnectionError as error:
+ # log verbose error from s3, return short message for user
+ msg = _logger.exception("Error during connection on S3")
+ raise exceptions.UserError(str(error)) from None
+
+ if not exists:
+ if not region_name:
+ bucket = s3.create_bucket(Bucket=bucket_name)
+ else:
+ bucket = s3.create_bucket(
+ Bucket=bucket_name,
+ CreateBucketConfiguration={"LocationConstraint": region_name},
+ )
+ return bucket
+
+ @api.model
+ def _store_file_read(self, fname):
+ if fname.startswith("s3://"):
+ s3uri = S3Uri(fname)
+ try:
+ bucket = self._get_s3_bucket(name=s3uri.bucket())
+ except exceptions.UserError:
+ _logger.exception(
+ "error reading attachment '%s' from object storage", fname
+ )
+ return ""
+ try:
+ key = s3uri.item()
+ bucket.meta.client.head_object(Bucket=bucket.name, Key=key)
+ with io.BytesIO() as res:
+ bucket.download_fileobj(key, res)
+ res.seek(0)
+ read = res.read()
+ except ClientError:
+ read = ""
+ _logger.info("attachment '%s' missing on object storage", fname)
+ return read
+ else:
+ return super()._store_file_read(fname)
+
+ @api.model
+ def _store_file_write(self, key, bin_data):
+ location = self.env.context.get("storage_location") or self._storage()
+ if location == "s3":
+ bucket = self._get_s3_bucket()
+ obj = bucket.Object(key=key)
+ with io.BytesIO() as file:
+ file.write(bin_data)
+ file.seek(0)
+ filename = "s3://%s/%s" % (bucket.name, key)
+ try:
+ obj.upload_fileobj(file)
+ except ClientError as error:
+ # log verbose error from s3, return short message for user
+ _logger.exception("Error during storage of the file %s" % filename)
+ raise exceptions.UserError(
+ _("The file could not be stored: %s") % str(error)
+ ) from None
+ else:
+ _super = super()
+ filename = _super._store_file_write(key, bin_data)
+ return filename
+
+ @api.model
+ def _store_file_delete(self, fname):
+ if fname.startswith("s3://"):
+ s3uri = S3Uri(fname)
+ bucket_name = s3uri.bucket()
+ item_name = s3uri.item()
+ # delete the file only if it is on the current configured bucket
+ # otherwise, we might delete files used on a different environment
+ if bucket_name == os.environ.get("AWS_BUCKETNAME"):
+ bucket = self._get_s3_bucket()
+ obj = bucket.Object(key=item_name)
+ try:
+ bucket.meta.client.head_object(Bucket=bucket.name, Key=item_name)
+ obj.delete()
+ _logger.info("file %s deleted on the object storage" % (fname,))
+ except ClientError:
+ # log verbose error from s3, return short message for
+ # user
+ _logger.exception("Error during deletion of the file %s" % fname)
+ else:
+ return super()._store_file_delete(fname)
diff --git a/attachment_s3/s3uri.py b/attachment_s3/s3uri.py
new file mode 100644
index 0000000..5bbe11a
--- /dev/null
+++ b/attachment_s3/s3uri.py
@@ -0,0 +1,20 @@
+# Copyright 2016-2019 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import re
+
+
+class S3Uri:
+ _url_re = re.compile("^s3:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE)
+
+ def __init__(self, uri):
+ match = self._url_re.match(uri)
+ if not match:
+ raise ValueError("%s: is not a valid S3 URI" % (uri,))
+ self._bucket, self._item = match.groups()
+
+ def bucket(self):
+ return self._bucket
+
+ def item(self):
+ return self._item
diff --git a/test_base_fileurl_field/README.rst b/test_base_fileurl_field/README.rst
deleted file mode 100644
index 0061d73..0000000
--- a/test_base_fileurl_field/README.rst
+++ /dev/null
@@ -1,4 +0,0 @@
-Test Base FileURL Field
-=======================
-
-This module serves as implementation example for `base_fileurl_field` and to run its tests.
diff --git a/test_base_fileurl_field/__init__.py b/test_base_fileurl_field/__init__.py
deleted file mode 100644
index 7967689..0000000
--- a/test_base_fileurl_field/__init__.py
+++ /dev/null
@@ -1,3 +0,0 @@
-# Copyright 2019 Camptocamp SA
-# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl)
-from . import models
diff --git a/test_base_fileurl_field/__manifest__.py b/test_base_fileurl_field/__manifest__.py
deleted file mode 100644
index 4c885b0..0000000
--- a/test_base_fileurl_field/__manifest__.py
+++ /dev/null
@@ -1,18 +0,0 @@
-# Copyright 2019 Camptocamp SA
-# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl)
-{
- "name": "test base fileurl fields",
- "summary": """A module to verify fileurl field.""",
- "version": "17.0.1.0.0",
- "category": "Tests",
- "author": "Camptocamp,Odoo Community Association (OCA)",
- "website": "https://github.com/camptocamp/odoo-cloud-platform",
- "license": "AGPL-3",
- "depends": ["base_fileurl_field"],
- "data": [
- "views/res_partner.xml",
- "views/res_users.xml",
- ],
- "installable": False,
- "auto_install": False,
-}
diff --git a/test_base_fileurl_field/data/pattern.png b/test_base_fileurl_field/data/pattern.png
deleted file mode 100644
index ce52599..0000000
Binary files a/test_base_fileurl_field/data/pattern.png and /dev/null differ
diff --git a/test_base_fileurl_field/data/sample.txt b/test_base_fileurl_field/data/sample.txt
deleted file mode 100644
index 8a03e0e..0000000
--- a/test_base_fileurl_field/data/sample.txt
+++ /dev/null
@@ -1 +0,0 @@
-This is a simple text file.
diff --git a/test_base_fileurl_field/models/__init__.py b/test_base_fileurl_field/models/__init__.py
deleted file mode 100644
index 906bfc7..0000000
--- a/test_base_fileurl_field/models/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from . import res_partner
-from . import res_users
diff --git a/test_base_fileurl_field/models/res_partner.py b/test_base_fileurl_field/models/res_partner.py
deleted file mode 100644
index 48ca0a4..0000000
--- a/test_base_fileurl_field/models/res_partner.py
+++ /dev/null
@@ -1,46 +0,0 @@
-# Copyright 2019 Camptocamp SA
-# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl)
-from odoo import _, api, fields, models
-from odoo.exceptions import ValidationError
-
-
-class ResPartner(models.Model):
-
- _inherit = "res.partner"
-
- name = fields.Char()
- url_file = fields.FileURL(
- storage_location="s3", filename="url_file_fname", storage_path="partner"
- )
- url_file_fname = fields.Char()
-
- url_image = fields.FileURL(
- storage_location="s3",
- filename="url_image_fname",
- storage_path="partner_image",
- )
- url_image_fname = fields.Char()
-
- @api.constrains("url_file", "url_file_fname")
- def _check_url_file_fname(self):
- rec = self.search([("url_file_fname", "=", self.url_file_fname)])
- if len(rec) > 1:
- raise ValidationError(
- _(
- "This file name is already used on an existing record. "
- "Please use another file name or delete the url_file on :\n"
- "Model: %s Id: %s" % (self._name, rec.id)
- )
- )
-
- @api.constrains("url_image", "url_image_fname")
- def _check_url_image_fname(self):
- rec = self.search([("url_image_fname", "=", self.url_image_fname)])
- if len(rec) > 1:
- raise ValidationError(
- _(
- "This file name is already used on an existing record. "
- "Please use another file name or delete the url_image on :\n"
- "Model: %s Id: %s" % (self._name, rec.id)
- )
- )
diff --git a/test_base_fileurl_field/models/res_users.py b/test_base_fileurl_field/models/res_users.py
deleted file mode 100644
index c8bc324..0000000
--- a/test_base_fileurl_field/models/res_users.py
+++ /dev/null
@@ -1,11 +0,0 @@
-# Copyright 2019 Camptocamp SA
-# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl)
-from odoo import fields, models
-
-
-class ResUsers(models.Model):
-
- _inherit = "res.users"
-
- partner_url_file = fields.FileURL(related="partner_id.url_file")
- partner_url_file_fname = fields.Char(related="partner_id.url_file_fname")
diff --git a/test_base_fileurl_field/tests/__init__.py b/test_base_fileurl_field/tests/__init__.py
deleted file mode 100644
index 56a55f4..0000000
--- a/test_base_fileurl_field/tests/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from . import ir_attachment
-from . import test_fileurl_fields
diff --git a/test_base_fileurl_field/tests/ir_attachment.py b/test_base_fileurl_field/tests/ir_attachment.py
deleted file mode 100644
index 8803571..0000000
--- a/test_base_fileurl_field/tests/ir_attachment.py
+++ /dev/null
@@ -1,44 +0,0 @@
-# Copyright 2019 Camptocamp SA
-# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl)
-
-import logging
-
-from odoo import api, models
-
-_logger = logging.getLogger(__name__)
-
-FAKE_S3_BUCKET = {}
-
-
-class IrAttachment(models.Model):
- _inherit = "ir.attachment"
-
- def _get_stores(self):
- l = ["s3"]
- l += super(IrAttachment, self)._get_stores()
- return l
-
- @api.model
- def _store_file_read(self, fname, bin_size=False):
- if fname.startswith("s3://"):
- return FAKE_S3_BUCKET.get(fname)
- else:
- return super(IrAttachment, self)._store_file_read(fname, bin_size)
-
- @api.model
- def _store_file_write(self, key, bin_data):
- location = self.env.context.get("storage_location") or self._storage()
- if location == "s3":
- FAKE_S3_BUCKET[key] = bin_data
- filename = "s3://fake_bucket/%s" % key
- else:
- _super = super(IrAttachment, self)
- filename = _super._store_file_write(key, bin_data)
- return filename
-
- @api.model
- def _store_file_delete(self, fname):
- if fname.startswith("s3://"):
- FAKE_S3_BUCKET.pop(fname)
- else:
- super(IrAttachment, self)._store_file_delete(fname)
diff --git a/test_base_fileurl_field/tests/test_fileurl_fields.py b/test_base_fileurl_field/tests/test_fileurl_fields.py
deleted file mode 100644
index 8be2bec..0000000
--- a/test_base_fileurl_field/tests/test_fileurl_fields.py
+++ /dev/null
@@ -1,42 +0,0 @@
-# Copyright 2019 Camptocamp SA
-# License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl)
-import base64
-
-from odoo.exceptions import ValidationError
-from odoo.modules.module import get_module_resource
-from odoo.tests import TransactionCase
-
-
-class TestFileUrlFields(TransactionCase):
- def test_fileurl_fields(self):
- file_path = get_module_resource("test_base_fileurl_field", "data", "sample.txt")
- image_path = get_module_resource(
- "test_base_fileurl_field", "data", "pattern.png"
- )
- partner = self.env.ref("base.main_partner")
- with open(file_path, "rb") as f:
- with open(image_path, "rb") as i:
- partner.write(
- {
- "url_file": base64.b64encode(f.read()),
- "url_file_fname": "sample.txt",
- "url_image": base64.b64encode(i.read()),
- "url_image_fname": "pattern.png",
- }
- )
-
- with open(file_path, "rb") as f:
- self.assertEqual(base64.decodebytes(partner.url_file), f.read())
-
- with open(image_path, "rb") as i:
- self.assertEqual(base64.decodebytes(partner.url_image), i.read())
-
- partner2 = self.env.ref("base.partner_admin")
- with open(file_path, "rb") as f:
- with self.assertRaises(ValidationError):
- partner2.write(
- {
- "url_file": base64.b64encode(f.read()),
- "url_file_fname": "sample.txt",
- }
- )
diff --git a/test_base_fileurl_field/views/res_partner.xml b/test_base_fileurl_field/views/res_partner.xml
deleted file mode 100644
index f3367e9..0000000
--- a/test_base_fileurl_field/views/res_partner.xml
+++ /dev/null
@@ -1,26 +0,0 @@
-
-
-
- res.partner.form.inherit
- res.partner
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/test_base_fileurl_field/views/res_users.xml b/test_base_fileurl_field/views/res_users.xml
deleted file mode 100644
index cbfdc72..0000000
--- a/test_base_fileurl_field/views/res_users.xml
+++ /dev/null
@@ -1,18 +0,0 @@
-
-
-
- res.users.form.inherit
- res.users
-
-
-
-
-
-
-
-
-
-
-
-
-