Merge commit 'refs/pull/477/head' of github.com:camptocamp/odoo-cloud-platform into merge-branch-1151-18_fixes-84d106f1

.pre-commit-config.yaml

@@ -1,9 +1,16 @@
 exclude: |
   (?x)
   # NOT INSTALLABLE ADDONS
+  ^attachment_azure/|
+  ^base_attachment_object_storage/|
   ^base_fileurl_field/|
+  ^cloud_platform/|
+  ^cloud_platform_azure/|
+  ^logging_json/|
   ^monitoring_log_requests/|
+  ^monitoring_prometheus/|
   ^monitoring_statsd/|
+  ^monitoring_status/|
   ^test_base_fileurl_field/|
   # END NOT INSTALLABLE ADDONS
   # Files and folders generated by bots, to avoid loops

attachment_s3/README.rst

@@ -0,0 +1,58 @@
+Attachments on S3 storage
+=========================
+
+This addon allows to store the attachments (documents and assets) on S3 or any
+other S3-compatible Object Storage.
+
+Configuration
+-------------
+
+Activate S3 storage:
+
+* Create or set the system parameter with the key ``ir_attachment.location``
+  and the value in the form ``s3``.
+
+Configure accesses with environment variables:
+
+* ``AWS_HOST`` (not required if using AWS services)
+* ``AWS_REGION`` (required if using AWS services)
+* ``AWS_ACCESS_KEY_ID``
+* ``AWS_SECRET_ACCESS_KEY``
+* ``AWS_BUCKETNAME`` (optional {db} placeholder)
+
+Read-only mode:
+
+The bucket and the file key are stored in the attachment. So if you change the
+``AWS_BUCKETNAME`` or the ``ir_attachment.location``, the existing attachments
+will still be read on their former bucket. But as soon as they are written over
+or new attachments are created, they will be created on the new bucket or on
+the other location (db or filesystem). This is a convenient way to be able to
+read the production attachments on a replication (since you have the
+credentials) without any risk to alter the production data.
+
+This addon must be added in the server wide addons with (``--load`` option):
+
+``--load=web,attachment_s3``
+
+The System Parameter ``ir_attachment.storage.force.database`` can be customized to
+force storage of files in the database. See the documentation of the module
+``base_attachment_object_storage``.
+
+Multi-tenancy
+-------------
+
+Use the `{db}` placeholder to handle multi-tenancy.
+
+On instances that hold multiple databases, it's preferable to have one bucket per database.
+
+To handle this, you can insert the `{db}` placeholder in your bucket name variable ``AWS_BUCKETNAME``.
+It will be replaced by the database name.
+This will give you a unique bucketname per database.
+
+
+Limitations
+-----------
+
+* You need to call ``env['ir.attachment'].force_storage()`` after
+  having changed the ``ir_attachment.location`` configuration in order to
+  migrate the existing attachments to S3.

attachment_s3/__init__.py

@@ -0,0 +1 @@
+from . import models

attachment_s3/__manifest__.py

@@ -0,0 +1,18 @@
+# Copyright 2016-2021 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+{
+    "name": "Attachments on S3 storage",
+    "summary": "Store assets and attachments on a S3 compatible object storage",
+    "version": "18.0.1.0.0",
+    "author": "Camptocamp,Odoo Community Association (OCA)",
+    "license": "AGPL-3",
+    "category": "Knowledge Management",
+    "depends": ["base", "base_attachment_object_storage"],
+    "external_dependencies": {
+        "python": ["boto3"],
+    },
+    "website": "https://github.com/camptocamp/odoo-cloud-platform",
+    "installable": True,
+}

attachment_s3/models/__init__.py

@@ -0,0 +1 @@
+from . import ir_attachment

attachment_s3/models/ir_attachment.py

@@ -0,0 +1,179 @@
+# Copyright 2016-2019 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+
+import io
+import logging
+import os
+from urllib.parse import urlsplit
+from typing import Optional
+
+from odoo import api, models
+from odoo.exceptions import UserError
+
+from ..s3uri import S3Uri
+
+_logger = logging.getLogger(__name__)
+
+try:
+    import boto3
+    from botocore.exceptions import ClientError, EndpointConnectionError
+except ImportError:
+    boto3 = None  # noqa
+    ClientError = None  # noqa
+    EndpointConnectionError = None  # noqa
+    _logger.debug("Cannot 'import boto3'.")
+
+
+class IrAttachment(models.Model):
+    _inherit = "ir.attachment"
+
+    def _get_stores(self):
+        return ["s3"] + super()._get_stores()
+
+    @api.model
+    def _get_s3_bucket(self, name: Optional[str] = None):
+        """Connect to S3 and return the bucket
+
+        The following environment variables can be set:
+        * ``AWS_HOST``
+        * ``AWS_REGION``
+        * ``AWS_ACCESS_KEY_ID``
+        * ``AWS_SECRET_ACCESS_KEY``
+        * ``AWS_BUCKETNAME``
+
+        If a name is provided, we'll read this bucket, otherwise, the bucket
+        from the environment variable ``AWS_BUCKETNAME`` will be read.
+
+        """
+        if not boto3:
+            raise UserError(self.env._("boto3 library is required for S3 integration."))
+
+        host = os.getenv("AWS_HOST").strip()
+
+        # Ensure `host`` is prefixed with a scheme (use https as default)
+        if host and not urlsplit(host).scheme:
+            host = f"https://{host}"
+
+        region_name = os.getenv("AWS_REGION")
+        access_key = os.getenv("AWS_ACCESS_KEY_ID")
+        secret_key = os.getenv("AWS_SECRET_ACCESS_KEY")
+        bucket_name = (name or os.getenv("AWS_BUCKETNAME", "")).format(db=self.env.cr.dbname)
+
+        if not all([access_key, secret_key, bucket_name]):
+            msg = self.env._(
+                "Missing AWS credentials."
+                "If you want to read from the %(bucket_name)s S3 bucket, the following "
+                "environment variables must be set:\n"
+                "* AWS_ACCESS_KEY_ID\n"
+                "* AWS_SECRET_ACCESS_KEY\n"
+                "If you want to write in the %(bucket_name)s S3 bucket, this variable "
+                "must be set as well:\n"
+                "* AWS_BUCKETNAME\n"
+                "Optionally, the S3 host can be changed with:\n"
+                "* AWS_HOST\n"
+            ).format(bucket_name=bucket_name)
+            raise UserError(msg)
+
+        s3_params = {
+            "aws_access_key_id": access_key,
+            "aws_secret_access_key": secret_key,
+        }
+        if host:
+            s3_params["endpoint_url"] = host
+        if region_name:
+            s3_params["region_name"] = region_name
+
+        s3 = boto3.resource("s3", **s3_params)
+        bucket = s3.Bucket(bucket_name)
+
+        try:
+            s3.meta.client.head_bucket(Bucket=bucket_name)
+        except ClientError as e:
+            # If a client error is thrown, then check that it was a 404 error.
+            # If it was a 404 error, then the bucket does not exist.
+            if e.response.get("Error", {}).get("Code") == "404":
+                _logger.warning(f"S3 bucket '{bucket_name}' does not exist.")
+                return self._create_s3_bucket(s3, bucket_name, region_name)
+            raise UserError(f"Failed to connect to S3 bucket: {str(e)}") from None
+        except EndpointConnectionError as error:
+            # log verbose error from s3, return short message for user
+            _logger.exception("Error during S3 connection.")
+            raise UserError(str(error)) from None
+
+        return bucket
+
+    def _create_s3_bucket(self, s3, bucket_name: str, region_name: Optional[str]):
+        """Create an S3 bucket if it does not exist."""
+        params = {"Bucket": bucket_name}
+        if region_name:
+            params["CreateBucketConfiguration"] = {"LocationConstraint": region_name}
+        
+        try:
+            return s3.create_bucket(**params)
+        except ClientError as e:
+            _logger.exception(f"Failed to create S3 bucket '{bucket_name}'")
+            raise UserError(f"Bucket creation failed: {str(e)}") from None
+
+    @api.model
+    def _store_file_read(self, fname: str):
+        if fname.startswith("s3://"):
+            s3uri = S3Uri(fname)
+            try:
+                bucket = self._get_s3_bucket(name=s3uri.bucket)
+            except UserError:
+                _logger.exception(f"Error reading attachment '{fname}' from S3.")
+                return ""
+
+            key = s3uri.item()
+            try:
+                bucket.meta.client.head_object(Bucket=bucket.name, Key=key)
+                with io.BytesIO() as res:
+                    bucket.download_fileobj(key, res)
+                    res.seek(0)
+                    return res.read()
+            except ClientError:
+                _logger.info(f"Attachment '{fname}' missing on S3.")
+                return ""
+        return super()._store_file_read(fname)
+
+    @api.model
+    def _store_file_write(self, key: str, bin_data: bytes) -> str:
+        location = self.env.context.get("storage_location") or self._storage()
+        if location == "s3":
+            bucket = self._get_s3_bucket()
+            obj = bucket.Object(key=key)
+            filename = f"s3://{bucket.name}/{key}"
+
+            try:
+                with io.BytesIO(bin_data) as file:
+                    obj.upload_fileobj(file)
+            except ClientError as error:
+                # log verbose error from s3, return short message for user
+                _logger.exception(f"Error storing file {filename} on S3.")
+                raise UserError(self.env._("The file could not be stored: %s") % str(error)) from None
+
+            return filename
+        return super()._store_file_write(key, bin_data)
+
+    @api.model
+    def _store_file_delete(self, fname: str):
+        if fname.startswith("s3://"):
+            s3uri = S3Uri(fname)
+            bucket_name = s3uri.bucket()
+            item_name = s3uri.item()
+            # delete the file only if it is on the current configured bucket
+            # otherwise, we might delete files used on a different environment
+            if bucket_name == os.getenv("AWS_BUCKETNAME"):
+                bucket = self._get_s3_bucket()
+                obj = bucket.Object(key=item_name)
+
+                try:
+                    bucket.meta.client.head_object(Bucket=bucket.name, Key=item_name)
+                    obj.delete()
+                    _logger.info(f"File {fname} deleted from S3.")
+                except ClientError:
+                    # log verbose error from s3, return short message for user
+                    _logger.exception(f"Error deleting file {fname} from S3.")
+        else:
+            return super()._store_file_delete(fname)

attachment_s3/pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["whool"]
+build-backend = "whool.buildapi"

attachment_s3/s3uri.py

@@ -0,0 +1,24 @@
+# Copyright 2016-2019 Camptocamp SA
+# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html)
+
+import re
+from typing import Optional
+
+class S3Uri:
+    _url_re = re.compile(r"^s3://+([^/]+)/?(.*)", re.IGNORECASE | re.UNICODE)
+
+    def __init__(self, uri: str) -> None:
+        match = self._url_re.match(uri)
+        if not match:
+            raise ValueError(f"{uri}: is not a valid S3 URI")
+        
+        self._bucket: str = match.group(1)
+        self._item: Optional[str] = match.group(2) if match.group(2) else None
+
+    @property
+    def bucket(self) -> str:
+        return self._bucket
+
+    @property
+    def item(self) -> Optional[str]:
+        return self._item

requirements.txt

@@ -1,6 +1,3 @@
 # generated from manifests external_dependencies
-azure-identity
-azure-storage-blob
-prometheus_client
-python-json-logger
+boto3
 redis

session_redis/http.py

@@ -3,6 +3,8 @@
 
 import logging
 import os
+from pathlib import Path
+from typing import Optional
 
 from odoo import http
 from odoo.tools import config
@@ -21,8 +23,9 @@ except ImportError:
     _logger.debug("Cannot 'import redis'.")
 
 
-def is_true(strval):
-    return bool(strtobool(strval or "0".lower()))
+def is_true(strval: Optional[str]) -> bool:
+    """Convert string value to boolean."""
+    return bool(strtobool((strval or "0").lower()))
 
 
 sentinel_host = os.getenv("ODOO_SESSION_REDIS_SENTINEL_HOST")
@@ -43,7 +46,8 @@ anon_expiration = os.getenv("ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS")
 
 
 @lazy_property
-def session_store(self):
+def session_store(self) -> RedisSessionStore:
+    """Configure Redis session storage."""
     if sentinel_host:
         sentinel = Sentinel([(sentinel_host, sentinel_port)], password=password)
         redis_client = sentinel.master_for(sentinel_master_name)
@@ -61,34 +65,34 @@ def session_store(self):
 
 
 def purge_fs_sessions(path):
-    if not os.path.isdir(path):
-        _logger.warning(f"Session directory '{path}' does not exist.")
+    """Remove old file-based sessions."""
+    session_path = Path(path)
+    if not session_path.exists():
+        _logger.warning(f"Session directory '{session_path}' does not exist.")
         return
 
-    for fname in os.listdir(path):
-        path = os.path.join(path, fname)
+    for session_file in session_path.iterdir():
         try:
-            os.unlink(path)
-        except OSError:
-            _logger.warning("OS Error during purge of redis sessions.")
+            session_file.unlink()
+            _logger.debug(f"Deleted session file: {session_file}")
+        except PermissionError:
+            _logger.warning(
+                f"Permission denied while deleting session file: {session_file}"
+            )
+        except OSError as e:
+            _logger.warning(f"Error deleting session file {session_file}: {str(e)}")
 
 
 if is_true(os.getenv("ODOO_SESSION_REDIS")):
+    storage_info = f"Redis with prefix '{prefix}' on "
     if sentinel_host:
-        _logger.debug(
-            "HTTP sessions stored in Redis with prefix '%s'. "
-            "Using Sentinel on %s:%s",
-            prefix or "",
-            sentinel_host,
-            sentinel_port,
-        )
+        storage_info += f"Sentinel {sentinel_host}:{sentinel_port}"
     else:
-        _logger.debug(
-            "HTTP sessions stored in Redis with prefix '%s' on " "%s:%s",
-            prefix or "",
-            host,
-            port,
-        )
+        storage_info += f"{host}:{port}"
+
+    _logger.debug("HTTP sessions stored in %s.", storage_info)
+
     http.Application.session_store = session_store
-    # clean the existing sessions on the file system
+
+    # Clean existing sessions stored in the file system
     purge_fs_sessions(config.session_dir)

session_redis/json_encoding.py

@@ -3,8 +3,9 @@
 
 import json
 from datetime import date, datetime
+from typing import Any
 
-import dateutil
+import dateutil.parser
 
 
 class SessionEncoder(json.JSONEncoder):
@@ -13,30 +14,30 @@ class SessionEncoder(json.JSONEncoder):
     So that we can later recompose them if they were stored in the session
     """
 
-    def default(self, obj):
+    def default(self, obj: Any) -> Any:
         if isinstance(obj, datetime):
             return {"_type": "datetime_isoformat", "value": obj.isoformat()}
-        elif isinstance(obj, date):
+        if isinstance(obj, date):
             return {"_type": "date_isoformat", "value": obj.isoformat()}
-        elif isinstance(obj, set):
-            return {"_type": "set", "value": tuple(obj)}
-        return json.JSONEncoder.default(self, obj)
+        if isinstance(obj, set):
+            return {"_type": "set", "value": tuple(sorted(obj))}
+        return super().default(obj)
 
 
 class SessionDecoder(json.JSONDecoder):
     """Decode json, recomposing recordsets and date/datetime"""
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, object_hook=self.object_hook, **kwargs)
 
-    def object_hook(self, obj):
+    def object_hook(self, obj: dict[str, Any]) -> Any:
+        """Convert serialized data back into its original Python type."""
         if "_type" not in obj:
             return obj
-        type_ = obj["_type"]
-        if type_ == "datetime_isoformat":
+        if obj["_type"] == "datetime_isoformat":
             return dateutil.parser.parse(obj["value"])
-        elif type_ == "date_isoformat":
+        if obj["_type"] == "date_isoformat":
             return dateutil.parser.parse(obj["value"]).date()
-        elif type_ == "set":
+        if obj["_type"] == "set":
             return set(obj["value"])
         return obj

session_redis/session.py

@@ -3,6 +3,7 @@
 
 import json
 import logging
+from typing import Optional
 
 from odoo.service import security
 from odoo.tools._vendor.sessions import SessionStore
@@ -24,41 +25,42 @@ class RedisSessionStore(SessionStore):
         self,
         redis,
         session_class=None,
-        prefix="",
-        expiration=None,
-        anon_expiration=None,
-    ):
+        prefix: str = "",
+        expiration: Optional[int] = None,
+        anon_expiration: Optional[int] = None,
+    ) -> None:
         super().__init__(session_class=session_class)
         self.redis = redis
-        if expiration is None:
-            self.expiration = DEFAULT_SESSION_TIMEOUT
-        else:
-            self.expiration = expiration
-        if anon_expiration is None:
-            self.anon_expiration = DEFAULT_SESSION_TIMEOUT_ANONYMOUS
-        else:
-            self.anon_expiration = anon_expiration
-        self.prefix = "session:"
-        if prefix:
-            self.prefix = f"{self.prefix}:{prefix}:"
+        self.expiration = (
+            expiration if expiration is not None else DEFAULT_SESSION_TIMEOUT
+        )
+        self.anon_expiration = (
+            anon_expiration
+            if anon_expiration is not None
+            else DEFAULT_SESSION_TIMEOUT_ANONYMOUS
+        )
+        self.prefix = f"session:{prefix}:" if prefix else "session:"
 
-    def build_key(self, sid):
+    def build_key(self, sid: str) -> str:
+        """Build the Redis key for a session ID."""
         return f"{self.prefix}{sid}"
 
-    def save(self, session):
+    def save(self, session) -> Optional[bool]:
+        """Save session data in Redis with an expiration time."""
         key = self.build_key(session.sid)
 
         # allow to set a custom expiration for a session
         # such as a very short one for monitoring requests
-        if session.uid:
-            expiration = session.expiration or self.expiration
-        else:
-            expiration = session.expiration or self.anon_expiration
+        expiration = session.expiration or (
+            self.expiration if session.uid else self.anon_expiration
+        )
+
         if _logger.isEnabledFor(logging.DEBUG):
-            if session.uid:
-                user_msg = f"user '{session.login}' (id: {session.uid})"
-            else:
-                user_msg = "anonymous user"
+            user_msg = (
+                f"user '{session.login}' (id: {session.uid})"
+                if session.uid
+                else "anonymous user"
+            )
             _logger.debug(
                 f"saving session with key '{key}' and "
                 f"expiration of {expiration} seconds for {user_msg}"
@@ -69,13 +71,16 @@ class RedisSessionStore(SessionStore):
         )
         if self.redis.set(key, data):
             return self.redis.expire(key, expiration)
+        return None
 
-    def delete(self, session):
+    def delete(self, session) -> int:
+        """Delete a session from Redis."""
         key = self.build_key(session.sid)
         _logger.debug(f"deleting session with key {key}")
         return self.redis.delete(key)
 
-    def get(self, sid):
+    def get(self, sid: str):
+        """Retrieve a session from Redis, or return a new one if not found."""
         if not self.is_valid_key(sid):
             _logger.debug(
                 f"session with invalid sid '{sid}' has been asked, "
@@ -91,6 +96,7 @@ class RedisSessionStore(SessionStore):
                 "returning a new one"
             )
             return self.new()
+
         try:
             data = json.loads(saved.decode("utf-8"), cls=json_encoding.SessionDecoder)
         except ValueError:
@@ -99,14 +105,18 @@ class RedisSessionStore(SessionStore):
                 "content could not be read, it has been reset"
             )
             data = {}
+
         return self.session_class(data, sid, False)
 
-    def list(self):
-        keys = self.redis.keys("%s*" % self.prefix)
+    def list(self) -> list[str]:
+        """List all session keys in Redis."""
+        # More efficient scanning
+        keys = [key for key in self.redis.scan_iter(f"{self.prefix}*")]
         _logger.debug("a listing redis keys has been called")
-        return [key[len(self.prefix) :] for key in keys]
+        return [key.decode("utf-8")[len(self.prefix) :] for key in keys]
 
-    def rotate(self, session, env):
+    def rotate(self, session, env) -> None:
+        """Rotate session ID and regenerate session token if user is logged in."""
         self.delete(session)
         session.sid = self.generate_key()
         if session.uid and env:

session_redis/strtobool.py

@@ -1,3 +1,5 @@
+from typing import Union
+
 _MAP = {
     "y": True,
     "yes": True,

test_base_fileurl_field/models/res_partner.py

@@ -5,7 +5,6 @@ from odoo.exceptions import ValidationError
 
 
 class ResPartner(models.Model):
-
     _inherit = "res.partner"
 
     name = fields.Char()

test_base_fileurl_field/models/res_users.py

@@ -4,7 +4,6 @@ from odoo import fields, models
 
 
 class ResUsers(models.Model):
-
     _inherit = "res.users"
 
     partner_url_file = fields.FileURL(related="partner_id.url_file")