diff --git a/.copier-answers.yml b/.copier-answers.yml new file mode 100644 index 0000000..86197e9 --- /dev/null +++ b/.copier-answers.yml @@ -0,0 +1,27 @@ +# Do NOT update manually; changes here will be overwritten by Copier +_commit: v1.14.2 +_src_path: https://github.com/OCA/oca-addons-repo-template.git +ci: GitHub +dependency_installation_mode: PIP +generate_requirements_txt: false +github_check_license: true +github_ci_extra_env: {} +github_enable_codecov: true +github_enable_makepot: true +github_enable_stale_action: true +github_enforce_dev_status_compatibility: false +include_wkhtmltopdf: false +odoo_version: 14.0 +org_name: Camptocamp +org_slug: camptocamp +rebel_module_groups: +- attachment_s3,cloud_platform_exoscale +- attachment_swift,cloud_platform_ovh +- attachment_azure,cloud_platform_azure +repo_description: '' +repo_name: Odoo Cloud Addons +repo_slug: odoo-cloud-platform +repo_website: https://github.com/camptocamp/odoo-cloud-platform +travis_apt_packages: [] +travis_apt_sources: [] + diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..bfd7ac5 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,20 @@ +# Configuration for known file extensions +[*.{css,js,json,less,md,py,rst,sass,scss,xml,yaml,yml}] +charset = utf-8 +end_of_line = lf +indent_size = 4 +indent_style = space +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{json,yml,yaml,rst,md}] +indent_size = 2 + +# Do not configure editor for libs and autogenerated content +[{*/static/{lib,src/lib}/**,*/static/description/index.html,*/readme/../README.rst}] +charset = unset +end_of_line = unset +indent_size = unset +indent_style = unset +insert_final_newline = false +trim_trailing_whitespace = false diff --git a/.eslintrc.yml b/.eslintrc.yml new file mode 100644 index 0000000..9429bc6 --- /dev/null +++ b/.eslintrc.yml @@ -0,0 +1,187 @@ +env: + browser: true + es6: true + +# See https://github.com/OCA/odoo-community.org/issues/37#issuecomment-470686449 +parserOptions: + ecmaVersion: 2019 + +overrides: + - files: + - "**/*.esm.js" + parserOptions: + sourceType: module + +# Globals available in Odoo that shouldn't produce errorings +globals: + _: readonly + $: readonly + fuzzy: readonly + jQuery: readonly + moment: readonly + odoo: readonly + openerp: readonly + owl: readonly + +# Styling is handled by Prettier, so we only need to enable AST rules; +# see https://github.com/OCA/maintainer-quality-tools/pull/618#issuecomment-558576890 +rules: + accessor-pairs: warn + array-callback-return: warn + callback-return: warn + capitalized-comments: + - warn + - always + - ignoreConsecutiveComments: true + ignoreInlineComments: true + complexity: + - warn + - 15 + constructor-super: warn + dot-notation: warn + eqeqeq: warn + global-require: warn + handle-callback-err: warn + id-blacklist: warn + id-match: warn + init-declarations: error + max-depth: warn + max-nested-callbacks: warn + max-statements-per-line: warn + no-alert: warn + no-array-constructor: warn + no-caller: warn + no-case-declarations: warn + no-class-assign: warn + no-cond-assign: error + no-const-assign: error + no-constant-condition: warn + no-control-regex: warn + no-debugger: error + no-delete-var: warn + no-div-regex: warn + no-dupe-args: error + no-dupe-class-members: error + no-dupe-keys: error + no-duplicate-case: error + no-duplicate-imports: error + no-else-return: warn + no-empty-character-class: warn + no-empty-function: error + no-empty-pattern: error + no-empty: warn + no-eq-null: error + no-eval: error + no-ex-assign: error + no-extend-native: warn + no-extra-bind: warn + no-extra-boolean-cast: warn + no-extra-label: warn + no-fallthrough: warn + no-func-assign: error + no-global-assign: error + no-implicit-coercion: + - warn + - allow: ["~"] + no-implicit-globals: warn + no-implied-eval: warn + no-inline-comments: warn + no-inner-declarations: warn + no-invalid-regexp: warn + no-irregular-whitespace: warn + no-iterator: warn + no-label-var: warn + no-labels: warn + no-lone-blocks: warn + no-lonely-if: error + no-mixed-requires: error + no-multi-str: warn + no-native-reassign: error + no-negated-condition: warn + no-negated-in-lhs: error + no-new-func: warn + no-new-object: warn + no-new-require: warn + no-new-symbol: warn + no-new-wrappers: warn + no-new: warn + no-obj-calls: warn + no-octal-escape: warn + no-octal: warn + no-param-reassign: warn + no-path-concat: warn + no-process-env: warn + no-process-exit: warn + no-proto: warn + no-prototype-builtins: warn + no-redeclare: warn + no-regex-spaces: warn + no-restricted-globals: warn + no-restricted-imports: warn + no-restricted-modules: warn + no-restricted-syntax: warn + no-return-assign: error + no-script-url: warn + no-self-assign: warn + no-self-compare: warn + no-sequences: warn + no-shadow-restricted-names: warn + no-shadow: warn + no-sparse-arrays: warn + no-sync: warn + no-this-before-super: warn + no-throw-literal: warn + no-undef-init: warn + no-undef: error + no-unmodified-loop-condition: warn + no-unneeded-ternary: error + no-unreachable: error + no-unsafe-finally: error + no-unused-expressions: error + no-unused-labels: error + no-unused-vars: error + no-use-before-define: error + no-useless-call: warn + no-useless-computed-key: warn + no-useless-concat: warn + no-useless-constructor: warn + no-useless-escape: warn + no-useless-rename: warn + no-void: warn + no-with: warn + operator-assignment: [error, always] + prefer-const: warn + radix: warn + require-yield: warn + sort-imports: warn + spaced-comment: [error, always] + strict: [error, function] + use-isnan: error + valid-jsdoc: + - warn + - prefer: + arg: param + argument: param + augments: extends + constructor: class + exception: throws + func: function + method: function + prop: property + return: returns + virtual: abstract + yield: yields + preferType: + array: Array + bool: Boolean + boolean: Boolean + number: Number + object: Object + str: String + string: String + requireParamDescription: false + requireReturn: false + requireReturnDescription: false + requireReturnType: false + valid-typeof: warn + yoda: warn diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..e397e8e --- /dev/null +++ b/.flake8 @@ -0,0 +1,12 @@ +[flake8] +max-line-length = 88 +max-complexity = 16 +# B = bugbear +# B9 = bugbear opinionated (incl line length) +select = C,E,F,W,B,B9 +# E203: whitespace before ':' (black behaviour) +# E501: flake8 line length (covered by bugbear B950) +# W503: line break before binary operator (black behaviour) +ignore = E203,E501,W503 +per-file-ignores= + __init__.py:F401 diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml new file mode 100644 index 0000000..755f8a2 --- /dev/null +++ b/.github/workflows/pre-commit.yml @@ -0,0 +1,35 @@ +name: pre-commit + +on: + pull_request: + branches: + - "14.0*" + push: + branches: + - "14.0" + - "14.0-ocabot-*" + +jobs: + pre-commit: + runs-on: ubuntu-22.04 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - name: Get python version + run: echo "PY=$(python -VV | sha256sum | cut -d' ' -f1)" >> $GITHUB_ENV + - uses: actions/cache@v1 + with: + path: ~/.cache/pre-commit + key: pre-commit|${{ env.PY }}|${{ hashFiles('.pre-commit-config.yaml') }} + - name: Install pre-commit + run: pip install pre-commit + - name: Run pre-commit + run: pre-commit run --all-files --show-diff-on-failure --color=always + - name: Check that all files generated by pre-commit are in git + run: | + newfiles="$(git ls-files --others --exclude-from=.gitignore)" + if [ "$newfiles" != "" ] ; then + echo "Please check-in the following files:" + echo "$newfiles" + exit 1 + fi diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000..1693a12 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,69 @@ +name: Mark stale issues and pull requests + +on: + schedule: + - cron: "0 12 * * 0" + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - name: Stale PRs and issues policy + uses: actions/stale@v4 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + # General settings. + ascending: true + remove-stale-when-updated: true + # Pull Requests settings. + # 120+30 day stale policy for PRs + # * Except PRs marked as "no stale" + days-before-pr-stale: 120 + days-before-pr-close: 30 + exempt-pr-labels: "no stale" + stale-pr-label: "stale" + stale-pr-message: > + There hasn't been any activity on this pull request in the past 4 months, so + it has been marked as stale and it will be closed automatically if no + further activity occurs in the next 30 days. + + If you want this PR to never become stale, please ask a PSC member to apply + the "no stale" label. + # Issues settings. + # 180+30 day stale policy for open issues + # * Except Issues marked as "no stale" + days-before-issue-stale: 180 + days-before-issue-close: 30 + exempt-issue-labels: "no stale,needs more information" + stale-issue-label: "stale" + stale-issue-message: > + There hasn't been any activity on this issue in the past 6 months, so it has + been marked as stale and it will be closed automatically if no further + activity occurs in the next 30 days. + + If you want this issue to never become stale, please ask a PSC member to + apply the "no stale" label. + + # 15+30 day stale policy for issues pending more information + # * Issues that are pending more information + # * Except Issues marked as "no stale" + - name: Needs more information stale issues policy + uses: actions/stale@v4 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + ascending: true + only-labels: "needs more information" + exempt-issue-labels: "no stale" + days-before-stale: 15 + days-before-close: 30 + days-before-pr-stale: -1 + days-before-pr-close: -1 + remove-stale-when-updated: true + stale-issue-label: "stale" + stale-issue-message: > + This issue needs more information and there hasn't been any activity + recently, so it has been marked as stale and it will be closed automatically + if no further activity occurs in the next 30 days. + + If you think this is a mistake, please ask a PSC member to remove the "needs + more information" label. diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000..a35ce30 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,96 @@ +name: tests + +on: + pull_request: + branches: + - "14.0*" + push: + branches: + - "14.0" + - "14.0-ocabot-*" + +jobs: + unreleased-deps: + runs-on: ubuntu-latest + name: Detect unreleased dependencies + steps: + - uses: actions/checkout@v2 + - run: | + for reqfile in requirements.txt test-requirements.txt ; do + if [ -f ${reqfile} ] ; then + result=0 + # reject non-comment lines that contain a / (i.e. URLs, relative paths) + grep "^[^#].*/" ${reqfile} || result=$? + if [ $result -eq 0 ] ; then + echo "Unreleased dependencies found in ${reqfile}." + exit 1 + fi + fi + done + test: + runs-on: ubuntu-22.04 + container: ${{ matrix.container }} + name: ${{ matrix.name }} + strategy: + fail-fast: false + matrix: + include: + - container: ghcr.io/oca/oca-ci/py3.6-odoo14.0:latest + include: "attachment_s3,cloud_platform_exoscale" + makepot: "true" + name: test exoscale S3 with Odoo + - container: ghcr.io/oca/oca-ci/py3.6-ocb14.0:latest + include: "attachment_s3,cloud_platform_exoscale" + name: test exoscale S3 with OCB + - container: ghcr.io/oca/oca-ci/py3.6-odoo14.0:latest + include: "attachment_swift,cloud_platform_ovh" + makepot: "true" + name: test OVH with Odoo + - container: ghcr.io/oca/oca-ci/py3.6-ocb14.0:latest + include: "attachment_swift,cloud_platform_ovh" + name: test azure with OCB + - container: ghcr.io/oca/oca-ci/py3.6-odoo14.0:latest + include: "attachment_azure,cloud_platform_azure" + makepot: "true" + name: test azure with Odoo + - container: ghcr.io/oca/oca-ci/py3.6-ocb14.0:latest + include: "attachment_azure,cloud_platform_azure" + name: test OVH with OCB + - container: ghcr.io/oca/oca-ci/py3.6-odoo14.0:latest + exclude: "attachment_azure,attachment_s3,attachment_swift,cloud_platform_exoscale,cloud_platform_ovh" + makepot: "true" + name: test others with Odoo + - container: ghcr.io/oca/oca-ci/py3.6-ocb14.0:latest + exclude: "attachment_azure,attachment_s3,attachment_swift,cloud_platform_exoscale,cloud_platform_ovh" + name: test others with OCB + services: + postgres: + image: postgres:9.6 + env: + POSTGRES_USER: odoo + POSTGRES_PASSWORD: odoo + POSTGRES_DB: odoo + ports: + - 5432:5432 + env: + INCLUDE: "${{ matrix.include }}" + EXCLUDE: "${{ matrix.exclude }}" + steps: + - uses: actions/checkout@v2 + with: + persist-credentials: false + - name: Install addons and dependencies + run: oca_install_addons + - name: Check licenses + run: manifestoo -d . check-licenses + - name: Check development status + run: manifestoo -d . check-dev-status --default-dev-status=Beta + continue-on-error: true + - name: Initialize test db + run: oca_init_test_database + - name: Run tests + run: oca_run_tests + - uses: codecov/codecov-action@v1 + - name: Update .pot files + run: oca_export_and_push_pot https://x-access-token:${{ secrets.GIT_PUSH_TOKEN }}@github.com/${{ github.repository }} + if: ${{ matrix.makepot == 'true' && github.event_name == 'push' && github.repository_owner == 'camptocamp' }} diff --git a/.gitignore b/.gitignore index 50b17c6..9c283fd 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] +/.venv +/.pytest_cache # C extensions *.so @@ -13,8 +15,6 @@ build/ develop-eggs/ dist/ eggs/ -.eggs/ -lib/ lib64/ parts/ sdist/ @@ -22,6 +22,7 @@ var/ *.egg-info/ .installed.cfg *.egg +*.eggs # Installer logs pip-log.txt @@ -41,6 +42,19 @@ coverage.xml # Pycharm .idea +# Eclipse +.settings + +# Visual Studio cache/options directory +.vs/ +.vscode + +# OSX Files +.DS_Store + +# Django stuff: +*.log + # Mr Developer .mr.developer.cfg .project @@ -50,8 +64,11 @@ coverage.xml .ropeproject # Sphinx documentation -connector/doc/_build/ +docs/_build/ # Backup files *~ *.swp + +# OCA rules +!static/lib/ diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 0000000..0ec187e --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,13 @@ +[settings] +; see https://github.com/psf/black +multi_line_output=3 +include_trailing_comma=True +force_grid_wrap=0 +combine_as_imports=True +use_parentheses=True +line_length=88 +known_odoo=odoo +known_odoo_addons=odoo.addons +sections=FUTURE,STDLIB,THIRDPARTY,ODOO,ODOO_ADDONS,FIRSTPARTY,LOCALFOLDER +default_section=THIRDPARTY +ensure_newline_before_comments = True diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..f6798c3 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,132 @@ +exclude: | + (?x) + # NOT INSTALLABLE ADDONS + # END NOT INSTALLABLE ADDONS + # Files and folders generated by bots, to avoid loops + ^setup/|/static/description/index\.html$| + # We don't want to mess with tool-generated files + .svg$|/tests/([^/]+/)?cassettes/|^.copier-answers.yml$|^.github/| + # Maybe reactivate this when all README files include prettier ignore tags? + ^README\.md$| + # Library files can have extraneous formatting (even minimized) + /static/(src/)?lib/| + # Repos using Sphinx to generate docs don't need prettying + ^docs/_templates/.*\.html$| + # You don't usually want a bot to modify your legal texts + (LICENSE.*|COPYING.*) +default_language_version: + python: python3 + node: "14.13.0" +repos: + - repo: local + hooks: + # These files are most likely copier diff rejection junks; if found, + # review them manually, fix the problem (if needed) and remove them + - id: forbidden-files + name: forbidden files + entry: found forbidden files; remove them + language: fail + files: "\\.rej$" + - id: en-po-files + name: en.po files cannot exist + entry: found a en.po file + language: fail + files: '[a-zA-Z0-9_]*/i18n/en\.po$' + - repo: https://github.com/oca/maintainer-tools + rev: ab1d7f6 + hooks: + # update the NOT INSTALLABLE ADDONS section above + - id: oca-update-pre-commit-excluded-addons + - id: oca-fix-manifest-website + args: ["https://github.com/camptocamp/odoo-cloud-platform"] + - repo: https://github.com/myint/autoflake + rev: v1.4 + hooks: + - id: autoflake + args: + - --expand-star-imports + - --ignore-init-module-imports + - --in-place + - --remove-all-unused-imports + - --remove-duplicate-keys + - --remove-unused-variables + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + - repo: https://github.com/pre-commit/mirrors-prettier + rev: v2.1.2 + hooks: + - id: prettier + name: prettier (with plugin-xml) + additional_dependencies: + - "prettier@2.1.2" + - "@prettier/plugin-xml@0.12.0" + args: + - --plugin=@prettier/plugin-xml + files: \.(css|htm|html|js|json|jsx|less|md|scss|toml|ts|xml|yaml|yml)$ + - repo: https://github.com/pre-commit/mirrors-eslint + rev: v7.8.1 + hooks: + - id: eslint + verbose: true + args: + - --color + - --fix + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v3.2.0 + hooks: + - id: trailing-whitespace + # exclude autogenerated files + exclude: /README\.rst$|\.pot?$ + - id: end-of-file-fixer + # exclude autogenerated files + exclude: /README\.rst$|\.pot?$ + - id: debug-statements + - id: fix-encoding-pragma + args: ["--remove"] + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-merge-conflict + # exclude files where underlines are not distinguishable from merge conflicts + exclude: /README\.rst$|^docs/.*\.rst$ + - id: check-symlinks + - id: check-xml + - id: mixed-line-ending + args: ["--fix=lf"] + - repo: https://github.com/asottile/pyupgrade + rev: v2.7.2 + hooks: + - id: pyupgrade + args: ["--keep-percent-format"] + - repo: https://github.com/PyCQA/isort + rev: 5.12.0 + hooks: + - id: isort + name: isort except __init__.py + args: + - --settings=. + exclude: /__init__\.py$ + - repo: https://github.com/acsone/setuptools-odoo + rev: 3.1.8 + hooks: + - id: setuptools-odoo-make-default + - repo: https://github.com/PyCQA/flake8 + rev: 3.8.3 + hooks: + - id: flake8 + name: flake8 + additional_dependencies: ["flake8-bugbear==20.1.4"] + - repo: https://github.com/OCA/pylint-odoo + rev: 7.0.2 + hooks: + - id: pylint_odoo + name: pylint with optional checks + args: + - --rcfile=.pylintrc + - --exit-zero + verbose: true + - id: pylint_odoo + args: + - --rcfile=.pylintrc-mandatory diff --git a/.prettierrc.yml b/.prettierrc.yml new file mode 100644 index 0000000..5b6d4b3 --- /dev/null +++ b/.prettierrc.yml @@ -0,0 +1,8 @@ +# Defaults for all prettier-supported languages. +# Prettier will complete this with settings from .editorconfig file. +bracketSpacing: false +printWidth: 88 +proseWrap: always +semi: true +trailingComma: "es5" +xmlWhitespaceSensitivity: "strict" diff --git a/.pylintrc b/.pylintrc new file mode 100644 index 0000000..58f06e7 --- /dev/null +++ b/.pylintrc @@ -0,0 +1,90 @@ + + +[MASTER] +load-plugins=pylint_odoo +score=n + +[ODOOLINT] +readme_template_url="https://github.com/OCA/maintainer-tools/blob/master/template/module/README.rst" +manifest_required_authors=Camptocamp +manifest_required_keys=license +manifest_deprecated_keys=description,active +license_allowed=AGPL-3,GPL-2,GPL-2 or any later version,GPL-3,GPL-3 or any later version,LGPL-3 +valid_odoo_versions=14.0 + +[MESSAGES CONTROL] +disable=all + +# This .pylintrc contains optional AND mandatory checks and is meant to be +# loaded in an IDE to have it check everything, in the hope this will make +# optional checks more visible to contributors who otherwise never look at a +# green travis to see optional checks that failed. +# .pylintrc-mandatory containing only mandatory checks is used the pre-commit +# config as a blocking check. + +enable=anomalous-backslash-in-string, + api-one-deprecated, + api-one-multi-together, + assignment-from-none, + attribute-deprecated, + class-camelcase, + dangerous-default-value, + dangerous-view-replace-wo-priority, + development-status-allowed, + duplicate-id-csv, + duplicate-key, + duplicate-xml-fields, + duplicate-xml-record-id, + eval-referenced, + eval-used, + incoherent-interpreter-exec-perm, + license-allowed, + manifest-author-string, + manifest-deprecated-key, + manifest-required-author, + manifest-required-key, + manifest-version-format, + method-compute, + method-inverse, + method-required-super, + method-search, + openerp-exception-warning, + pointless-statement, + pointless-string-statement, + print-used, + redundant-keyword-arg, + redundant-modulename-xml, + reimported, + relative-import, + return-in-init, + rst-syntax-error, + sql-injection, + too-few-format-args, + translation-field, + translation-required, + unreachable, + use-vim-comment, + wrong-tabs-instead-of-spaces, + xml-syntax-error, + # messages that do not cause the lint step to fail + consider-merging-classes-inherited, + create-user-wo-reset-password, + dangerous-filter-wo-user, + deprecated-module, + file-not-used, + invalid-commit, + missing-manifest-dependency, + missing-newline-extrafiles, + missing-readme, + no-utf8-coding-comment, + odoo-addons-relative-import, + old-api7-method-defined, + redefined-builtin, + too-complex, + unnecessary-utf8-coding-comment + + +[REPORTS] +msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg} +output-format=colorized +reports=no diff --git a/.pylintrc-mandatory b/.pylintrc-mandatory new file mode 100644 index 0000000..755b649 --- /dev/null +++ b/.pylintrc-mandatory @@ -0,0 +1,65 @@ + +[MASTER] +load-plugins=pylint_odoo +score=n + +[ODOOLINT] +readme_template_url="https://github.com/OCA/maintainer-tools/blob/master/template/module/README.rst" +manifest_required_authors=Camptocamp +manifest_required_keys=license +manifest_deprecated_keys=description,active +license_allowed=AGPL-3,GPL-2,GPL-2 or any later version,GPL-3,GPL-3 or any later version,LGPL-3 +valid_odoo_versions=14.0 + +[MESSAGES CONTROL] +disable=all + +enable=anomalous-backslash-in-string, + api-one-deprecated, + api-one-multi-together, + assignment-from-none, + attribute-deprecated, + class-camelcase, + dangerous-default-value, + dangerous-view-replace-wo-priority, + development-status-allowed, + duplicate-id-csv, + duplicate-key, + duplicate-xml-fields, + duplicate-xml-record-id, + eval-referenced, + eval-used, + incoherent-interpreter-exec-perm, + license-allowed, + manifest-author-string, + manifest-deprecated-key, + manifest-required-author, + manifest-required-key, + manifest-version-format, + method-compute, + method-inverse, + method-required-super, + method-search, + openerp-exception-warning, + pointless-statement, + pointless-string-statement, + print-used, + redundant-keyword-arg, + redundant-modulename-xml, + reimported, + relative-import, + return-in-init, + rst-syntax-error, + sql-injection, + too-few-format-args, + translation-field, + translation-required, + unreachable, + use-vim-comment, + wrong-tabs-instead-of-spaces, + xml-syntax-error + +[REPORTS] +msg-template={path}:{line}: [{msg_id}({symbol}), {obj}] {msg} +output-format=colorized +reports=no diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index c90f2f2..0000000 --- a/.travis.yml +++ /dev/null @@ -1,42 +0,0 @@ -language: python -sudo: false -cache: pip - -branches: - only: - - "/^[[:digit:]]{1,2}.[[:digit:]]$/" - -python: - - "3.6" - -addons: - postgresql: "9.5" - apt: - packages: - - expect-dev # provides unbuffer utility - - python-lxml # because pip installation is slow - - python-simplejson - - python-serial - -env: - matrix: - - LINT_CHECK="1" - - TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="cloud_platform_exoscale" - - TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="cloud_platform_exoscale" - - TESTS="1" ODOO_REPO="odoo/odoo" INCLUDE="cloud_platform_ovh" - - TESTS="1" ODOO_REPO="OCA/OCB" INCLUDE="cloud_platform_ovh" - - TESTS="1" ODOO_REPO="odoo/odoo" EXCLUDE="cloud_platform,cloud_platform_ovh,cloud_platform_exoscale" - - TESTS="1" ODOO_REPO="OCA/OCB" EXCLUDE="cloud_platform,cloud_platform_ovh,cloud_platform_exoscale" - global: - - VERSION="14.0" LINT_CHECK="0" TESTS="0" - -install: - - git clone --depth=1 https://github.com/OCA/maintainer-quality-tools.git ${HOME}/maintainer-quality-tools - - export PATH=${HOME}/maintainer-quality-tools/travis:${PATH} - - travis_install_nightly - -script: - - travis_run_tests - -after_success: - - travis_after_test_success diff --git a/LICENSE b/LICENSE index 3ffc567..be3f7b2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,7 +1,7 @@ -GNU AFFERO GENERAL PUBLIC LICENSE + GNU AFFERO GENERAL PUBLIC LICENSE Version 3, 19 November 2007 - Copyright (C) 2007 Free Software Foundation, Inc. + Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. @@ -633,8 +633,8 @@ the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software: you can redistribute it and/or modify - it under the terms of the GNU Affero General Public License as published - by the Free Software Foundation, either version 3 of the License, or + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, @@ -643,7 +643,7 @@ the "copyright" line and a pointer to where the full notice is found. GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License - along with this program. If not, see . + along with this program. If not, see . Also add information on how to contact you by electronic and paper mail. @@ -658,4 +658,4 @@ specific requirements. You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see -. \ No newline at end of file +. diff --git a/README.md b/README.md index a3e6490..53f69c8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,11 @@ -[![Build Status](https://travis-ci.com/camptocamp/odoo-cloud-platform.svg?token=Lpp9PcS5on9AGbp76WKB&branch=12.0)](https://travis-ci.com/camptocamp/odoo-cloud-platform) + + +[![Pre-commit Status](https://github.com/camptocamp/odoo-cloud-platform/actions/workflows/pre-commit.yml/badge.svg?branch=14.0)](https://github.com/camptocamp/odoo-cloud-platform/actions/workflows/pre-commit.yml?query=branch%3A14.0) +[![Build Status](https://github.com/camptocamp/odoo-cloud-platform/actions/workflows/test.yml/badge.svg?branch=14.0)](https://github.com/camptocamp/odoo-cloud-platform/actions/workflows/test.yml?query=branch%3A14.0) +[![codecov](https://codecov.io/gh/camptocamp/odoo-cloud-platform/branch/14.0/graph/badge.svg)](https://codecov.io/gh/camptocamp/odoo-cloud-platform) + + + # Odoo Cloud Addons @@ -167,3 +174,26 @@ The checks can be bypassed with the environment variable To prevent object storage to be accessed while failing for any kind of reason set this environment variable `DISABLE_ATTACHMENT_STORAGE` set to `1`. + + + + + +[//]: # (addons) + +This part will be replaced when running the oca-gen-addons-table script from OCA/maintainer-tools. + +[//]: # (end addons) + + + +## Licenses + +This repository is licensed under [AGPL-3.0](LICENSE). + +However, each module can have a totally different license, as long as they adhere to Camptocamp +policy. Consult each module's `__manifest__.py` file, which contains a `license` key +that explains its license. + +---- + diff --git a/attachment_azure/__manifest__.py b/attachment_azure/__manifest__.py index 9264084..6a601fe 100644 --- a/attachment_azure/__manifest__.py +++ b/attachment_azure/__manifest__.py @@ -9,13 +9,11 @@ "Open Source Integrators, " "Serpent Consulting Services, " "Odoo Community Association (OCA)", + "website": "https://github.com/camptocamp/odoo-cloud-platform", "license": "AGPL-3", "category": "Knowledge Management", "depends": ["base_attachment_object_storage"], - "external_dependencies": { - "python": ["azure-storage-blob", "azure-identity"], - }, - "website": "https://github.com/camptocamp/odoo-cloud-platform", + "external_dependencies": {"python": ["azure-storage-blob", "azure-identity"]}, "installable": True, "development_status": "Beta", "post_init_hook": "_post_init_hook", diff --git a/attachment_azure/controllers/main.py b/attachment_azure/controllers/main.py index 2a28987..a0816c9 100644 --- a/attachment_azure/controllers/main.py +++ b/attachment_azure/controllers/main.py @@ -1,11 +1,11 @@ import logging import os -from odoo.addons.web.controllers.main import Database -from odoo import http -from odoo import exceptions +from odoo import exceptions, http from odoo.http import request +from odoo.addons.web.controllers.main import Database + _logger = logging.getLogger(__name__) diff --git a/attachment_azure/models/ir_attachment.py b/attachment_azure/models/ir_attachment.py index 2fc03c8..2a358db 100644 --- a/attachment_azure/models/ir_attachment.py +++ b/attachment_azure/models/ir_attachment.py @@ -12,13 +12,13 @@ from odoo import _, api, exceptions, models _logger = logging.getLogger(__name__) try: + from azure.core.exceptions import HttpResponseError, ResourceExistsError from azure.storage.blob import ( - BlobServiceClient, - generate_account_sas, - ResourceTypes, AccountSasPermissions, + BlobServiceClient, + ResourceTypes, + generate_account_sas, ) - from azure.core.exceptions import ResourceExistsError, HttpResponseError except ImportError: _logger.debug("Cannot 'import azure-storage-blob'.") @@ -32,9 +32,9 @@ class IrAttachment(models.Model): _inherit = "ir.attachment" def _get_stores(self): - l = ["azure"] - l += super(IrAttachment, self)._get_stores() - return l + stores = ["azure"] + stores += super()._get_stores() + return stores @api.model def _get_blob_service_client(self): @@ -112,10 +112,8 @@ class IrAttachment(models.Model): @api.model def _get_container_name(self, db_name=None): - """ - Container naming rules: - https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names - """ + # Container naming rules: + # https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names # noqa: B950 running_env = os.environ.get("RUNNING_ENV", "dev") storage_name = os.environ.get("AZURE_STORAGE_NAME", r"{env}-{db}") storage_name = storage_name.format( @@ -135,7 +133,7 @@ class IrAttachment(models.Model): except exceptions.UserError: _logger.exception( "error accessing to storage '%s' please check credentials ", - container_name + container_name, ) return False container_client = blob_service_client.get_container_client(container_name) @@ -152,14 +150,14 @@ class IrAttachment(models.Model): def _store_file_read(self, fname, bin_size=False): if fname.startswith("azure://"): key = fname.replace("azure://", "", 1).lower() - if '/' in key: - container_name, key = key.split('/', 1) + if "/" in key: + container_name, key = key.split("/", 1) else: container_name = None container_client = self._get_azure_container(container_name) # if container cannot be retrived, abort reading from azure storage if not container_client: - return '' + return "" try: blob_client = container_client.get_blob_client(key) read = blob_client.download_blob().readall() @@ -199,13 +197,13 @@ class IrAttachment(models.Model): def _store_file_delete(self, fname): if fname.startswith("azure://"): key = fname.replace("azure://", "", 1).lower() - if '/' in key: - container_name, key = key.split('/', 1) + if "/" in key: + container_name, key = key.split("/", 1) else: container_name = None container_client = self._get_azure_container(container_name) if not container_client: - return '' + return "" # delete the file only if it is on the current configured container # otherwise, we might delete files used on a different environment try: diff --git a/attachment_s3/__init__.py b/attachment_s3/__init__.py index a9e3372..0650744 100644 --- a/attachment_s3/__init__.py +++ b/attachment_s3/__init__.py @@ -1,2 +1 @@ - from . import models diff --git a/attachment_s3/__manifest__.py b/attachment_s3/__manifest__.py index 35e1468..ad18148 100644 --- a/attachment_s3/__manifest__.py +++ b/attachment_s3/__manifest__.py @@ -1,18 +1,16 @@ # Copyright 2016-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - -{'name': 'Attachments on S3 storage', - 'summary': 'Store assets and attachments on a S3 compatible object storage', - 'version': "14.0.1.0.0", - 'author': 'Camptocamp,Odoo Community Association (OCA)', - 'license': 'AGPL-3', - 'category': 'Knowledge Management', - 'depends': ['base', 'base_attachment_object_storage'], - 'external_dependencies': { - 'python': ['boto3'], - }, - 'website': 'https://www.camptocamp.com', - 'data': [], - 'installable': True, - } +{ + "name": "Attachments on S3 storage", + "summary": "Store assets and attachments on a S3 compatible object storage", + "version": "14.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Knowledge Management", + "depends": ["base", "base_attachment_object_storage"], + "external_dependencies": {"python": ["boto3"]}, + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "data": [], + "installable": True, +} diff --git a/attachment_s3/migrations/13.0.0.0.1/post-migration.py b/attachment_s3/migrations/13.0.0.0.1/post-migration.py new file mode 100644 index 0000000..5ff079b --- /dev/null +++ b/attachment_s3/migrations/13.0.0.0.1/post-migration.py @@ -0,0 +1,69 @@ +# Copyright 2016-2019 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +import logging +import os +from contextlib import closing + +import odoo + +_logger = logging.getLogger(__name__) + + +def migrate(cr, version): + if not version: + return + cr.execute( + """ + SELECT value FROM ir_config_parameter + WHERE key = 'ir_attachment.location' + """ + ) + row = cr.fetchone() + bucket = os.environ.get("AWS_BUCKETNAME") + + if row[0] == "s3" and bucket: + uid = odoo.SUPERUSER_ID + registry = odoo.modules.registry.Registry(cr.dbname) + new_cr = registry.cursor() + with closing(new_cr): + with odoo.api.Environment.manage(): + env = odoo.api.Environment(new_cr, uid, {}) + store_local = env["ir.attachment"].search( + [ + ("store_fname", "=like", "s3://%"), + "|", + ("res_model", "=", "ir.ui.view"), + ( + "res_field", + "in", + ["image_small", "image_medium", "web_icon_data"], + ), + ], + ) + + _logger.info( + "Moving %d attachments from S3 to DB for fast access", + len(store_local), + ) + for attachment_id in store_local.ids: + # force re-storing the document, will move + # it from the object storage to the database + + # This is a trick to avoid having the 'datas' function + # fields computed for every attachment on each + # iteration of the loop. The former issue being that + # it reads the content of the file of ALL the + # attachments on each loop. + try: + env.clear() + attachment = env["ir.attachment"].browse(attachment_id) + _logger.info( + "Moving attachment %s (id: %s)", + attachment.name, + attachment.id, + ) + attachment.write({"datas": attachment.datas}) + new_cr.commit() + except Exception: + new_cr.rollback() diff --git a/attachment_s3/models/__init__.py b/attachment_s3/models/__init__.py index 3cf36e5..aaf38a1 100644 --- a/attachment_s3/models/__init__.py +++ b/attachment_s3/models/__init__.py @@ -1,2 +1 @@ - from . import ir_attachment diff --git a/attachment_s3/models/ir_attachment.py b/attachment_s3/models/ir_attachment.py index 85ab60d..3e8da47 100644 --- a/attachment_s3/models/ir_attachment.py +++ b/attachment_s3/models/ir_attachment.py @@ -2,12 +2,13 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +import io import logging import os -import io from urllib.parse import urlsplit from odoo import _, api, exceptions, models + from ..s3uri import S3Uri _logger = logging.getLogger(__name__) @@ -26,9 +27,9 @@ class IrAttachment(models.Model): _inherit = "ir.attachment" def _get_stores(self): - l = ['s3'] - l += super()._get_stores() - return l + stores = ["s3"] + stores += super()._get_stores() + return stores @api.model def _get_s3_bucket(self, name=None): @@ -45,46 +46,47 @@ class IrAttachment(models.Model): from the environment variable ``AWS_BUCKETNAME`` will be read. """ - host = os.environ.get('AWS_HOST') + host = os.environ.get("AWS_HOST") # Ensure host is prefixed with a scheme (use https as default) if host and not urlsplit(host).scheme: - host = 'https://%s' % host + host = "https://%s" % host - region_name = os.environ.get('AWS_REGION') - aws_use_irsa = os.environ.get('AWS_USE_IRSA') - access_key = os.environ.get('AWS_ACCESS_KEY_ID') - secret_key = os.environ.get('AWS_SECRET_ACCESS_KEY') - bucket_name = name or os.environ.get('AWS_BUCKETNAME') + region_name = os.environ.get("AWS_REGION") + aws_use_irsa = os.environ.get("AWS_USE_IRSA") + access_key = os.environ.get("AWS_ACCESS_KEY_ID") + secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY") + bucket_name = name or os.environ.get("AWS_BUCKETNAME") # replaces {db} by the database name to handle multi-tenancy bucket_name = bucket_name.format(db=self.env.cr.dbname) params = {} if not aws_use_irsa and access_key: - params['aws_access_key_id'] = access_key + params["aws_access_key_id"] = access_key if secret_key: - params['aws_secret_access_key'] = secret_key + params["aws_secret_access_key"] = secret_key if host: - params['endpoint_url'] = host + params["endpoint_url"] = host if region_name: - params['region_name'] = region_name + params["region_name"] = region_name if not (bucket_name and (access_key and secret_key or aws_use_irsa)): - msg = _('If you want to read from the %s S3 bucket, the following ' - 'environment variables must be set:\n' - '* AWS_ACCESS_KEY_ID\n' - '* AWS_SECRET_ACCESS_KEY\n' - 'If you want to write in the %s S3 bucket, this variable ' - 'must be set as well:\n' - '* AWS_BUCKETNAME\n' - 'if you want to user IRSA authentification method set' - '* AWS_USE_IRSA\n' - 'Optionally, the S3 host can be changed with:\n' - '* AWS_HOST\n' - ) % (bucket_name, bucket_name) + msg = _( + "If you want to read from the %s S3 bucket, the following " + "environment variables must be set:\n" + "* AWS_ACCESS_KEY_ID\n" + "* AWS_SECRET_ACCESS_KEY\n" + "If you want to write in the %s S3 bucket, this variable " + "must be set as well:\n" + "* AWS_BUCKETNAME\n" + "if you want to user IRSA authentification method set" + "* AWS_USE_IRSA\n" + "Optionally, the S3 host can be changed with:\n" + "* AWS_HOST\n" + ) % (bucket_name, bucket_name) raise exceptions.UserError(msg) # try: - s3 = boto3.resource('s3', **params) + s3 = boto3.resource("s3", **params) bucket = s3.Bucket(bucket_name) exists = True try: @@ -92,12 +94,12 @@ class IrAttachment(models.Model): except ClientError as e: # If a client error is thrown, then check that it was a 404 error. # If it was a 404 error, then the bucket does not exist. - error_code = e.response['Error']['Code'] - if error_code == '404': + error_code = e.response["Error"]["Code"] + if error_code == "404": exists = False except EndpointConnectionError as error: # log verbose error from s3, return short message for user - _logger.exception('Error during connection on S3') + _logger.exception("Error during connection on S3") raise exceptions.UserError(str(error)) if not exists: @@ -106,14 +108,13 @@ class IrAttachment(models.Model): else: bucket = s3.create_bucket( Bucket=bucket_name, - CreateBucketConfiguration={ - 'LocationConstraint': region_name - }) + CreateBucketConfiguration={"LocationConstraint": region_name}, + ) return bucket @api.model def _store_file_read(self, fname): - if fname.startswith('s3://'): + if fname.startswith("s3://"): s3uri = S3Uri(fname) try: bucket = self._get_s3_bucket(name=s3uri.bucket()) @@ -121,44 +122,38 @@ class IrAttachment(models.Model): _logger.exception( "error reading attachment '%s' from object storage", fname ) - return '' + return "" try: key = s3uri.item() - bucket.meta.client.head_object( - Bucket=bucket.name, Key=key - ) + bucket.meta.client.head_object(Bucket=bucket.name, Key=key) with io.BytesIO() as res: bucket.download_fileobj(key, res) res.seek(0) read = res.read() except ClientError: - read = '' - _logger.info( - "attachment '%s' missing on object storage", fname - ) + read = "" + _logger.info("attachment '%s' missing on object storage", fname) return read else: return super()._store_file_read(fname) @api.model def _store_file_write(self, key, bin_data): - location = self.env.context.get('storage_location') or self._storage() - if location == 's3': + location = self.env.context.get("storage_location") or self._storage() + if location == "s3": bucket = self._get_s3_bucket() obj = bucket.Object(key=key) with io.BytesIO() as file: file.write(bin_data) file.seek(0) - filename = 's3://%s/%s' % (bucket.name, key) + filename = "s3://%s/%s" % (bucket.name, key) try: obj.upload_fileobj(file) except ClientError as error: # log verbose error from s3, return short message for user - _logger.exception( - 'Error during storage of the file %s' % filename - ) + _logger.exception("Error during storage of the file %s" % filename) raise exceptions.UserError( - _('The file could not be stored: %s') % str(error) + _("The file could not be stored: %s") % str(error) ) else: _super = super() @@ -167,28 +162,22 @@ class IrAttachment(models.Model): @api.model def _store_file_delete(self, fname): - if fname.startswith('s3://'): + if fname.startswith("s3://"): s3uri = S3Uri(fname) bucket_name = s3uri.bucket() item_name = s3uri.item() # delete the file only if it is on the current configured bucket # otherwise, we might delete files used on a different environment - if bucket_name == os.environ.get('AWS_BUCKETNAME'): + if bucket_name == os.environ.get("AWS_BUCKETNAME"): bucket = self._get_s3_bucket() obj = bucket.Object(key=item_name) try: - bucket.meta.client.head_object( - Bucket=bucket.name, Key=item_name - ) + bucket.meta.client.head_object(Bucket=bucket.name, Key=item_name) obj.delete() - _logger.info( - 'file %s deleted on the object storage' % (fname,) - ) + _logger.info("file %s deleted on the object storage" % (fname,)) except ClientError: # log verbose error from s3, return short message for # user - _logger.exception( - 'Error during deletion of the file %s' % fname - ) + _logger.exception("Error during deletion of the file %s" % fname) else: super()._store_file_delete(fname) diff --git a/attachment_swift/__manifest__.py b/attachment_swift/__manifest__.py index d78ea82..7b30fa2 100644 --- a/attachment_swift/__manifest__.py +++ b/attachment_swift/__manifest__.py @@ -2,20 +2,18 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -{'name': 'Attachments on Swift storage', - 'summary': 'Store assets and attachments on a Swift compatible object store', - 'version': "14.0.1.0.0", - 'author': 'Camptocamp,Odoo Community Association (OCA)', - 'license': 'AGPL-3', - 'category': 'Knowledge Management', - 'depends': ['base_attachment_object_storage'], - 'external_dependencies': { - 'python': ['swiftclient', - 'keystoneclient', - 'keystoneauth1', - ], - }, - 'website': 'https://www.camptocamp.com', - 'data': [], - 'installable': True, - } +{ + "name": "Attachments on Swift storage", + "summary": "Store assets and attachments on a Swift compatible object store", + "version": "14.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Knowledge Management", + "depends": ["base_attachment_object_storage"], + "external_dependencies": { + "python": ["swiftclient", "keystoneclient", "keystoneauth1"], + }, + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "data": [], + "installable": True, +} diff --git a/attachment_swift/models/ir_attachment.py b/attachment_swift/models/ir_attachment.py index 7fd659e..2d973ce 100644 --- a/attachment_swift/models/ir_attachment.py +++ b/attachment_swift/models/ir_attachment.py @@ -4,17 +4,18 @@ import logging import os -from ..swift_uri import SwiftUri -from odoo import api, exceptions, models, _ +from odoo import _, api, exceptions, models + +from ..swift_uri import SwiftUri _logger = logging.getLogger(__name__) try: - import swiftclient import keystoneauth1 import keystoneauth1.identity import keystoneauth1.session + import swiftclient from swiftclient.exceptions import ClientException except ImportError: swiftclient = None @@ -48,8 +49,9 @@ class SwiftSessionStore(object): def _get_key(self, auth_url, username, password, project_name): return (auth_url, username, password, project_name) - def get_session(self, auth_url=None, username=None, password=None, - project_name=None): + def get_session( + self, auth_url=None, username=None, password=None, project_name=None + ): key = self._get_key(auth_url, username, password, project_name) session = self._sessions.get(key) if not session: @@ -58,8 +60,8 @@ class SwiftSessionStore(object): password=password, project_name=project_name, auth_url=auth_url, - project_domain_id='default', - user_domain_id='default', + project_domain_id="default", + user_domain_id="default", ) session = keystoneauth1.session.Session( auth=auth, @@ -73,36 +75,38 @@ swift_session_store = SwiftSessionStore() class IrAttachment(models.Model): - _inherit = 'ir.attachment' + _inherit = "ir.attachment" def _get_stores(self): - l = ['swift'] - l += super()._get_stores() - return l + stores = ["swift"] + stores += super()._get_stores() + return stores @api.model def _get_swift_connection(self): - """ Returns a connection object for the Swift object store """ - host = os.environ.get('SWIFT_AUTH_URL') - account = os.environ.get('SWIFT_ACCOUNT') - password = os.environ.get('SWIFT_PASSWORD') - project_name = os.environ.get('SWIFT_PROJECT_NAME') - if not project_name and os.environ.get('SWIFT_TENANT_NAME'): - project_name = os.environ['SWIFT_TENANT_NAME'] + """Returns a connection object for the Swift object store""" + host = os.environ.get("SWIFT_AUTH_URL") + account = os.environ.get("SWIFT_ACCOUNT") + password = os.environ.get("SWIFT_PASSWORD") + project_name = os.environ.get("SWIFT_PROJECT_NAME") + if not project_name and os.environ.get("SWIFT_TENANT_NAME"): + project_name = os.environ["SWIFT_TENANT_NAME"] _logger.warning( "SWIFT_TENANT_NAME is deprecated and " "must be replaced by SWIFT_PROJECT_NAME" ) - region = os.environ.get('SWIFT_REGION_NAME') + region = os.environ.get("SWIFT_REGION_NAME") os_options = {} if region: - os_options['region_name'] = region + os_options["region_name"] = region if not (host and account and password and project_name): - raise exceptions.UserError(_( - "Problem connecting to Swift store, are the env variables " - "(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, " - "SWIFT_TENANT_NAME) properly set?" - )) + raise exceptions.UserError( + _( + "Problem connecting to Swift store, are the env variables " + "(SWIFT_AUTH_URL, SWIFT_ACCOUNT, SWIFT_PASSWORD, " + "SWIFT_TENANT_NAME) properly set?" + ) + ) try: session = swift_session_store.get_session( username=account, @@ -115,13 +119,13 @@ class IrAttachment(models.Model): os_options=os_options, ) except ClientException: - _logger.exception('Error connecting to Swift object store') - raise exceptions.UserError(_('Error on Swift connection')) + _logger.exception("Error connecting to Swift object store") + raise exceptions.UserError(_("Error on Swift connection")) return conn @api.model def _store_file_read(self, fname): - if fname.startswith('swift://'): + if fname.startswith("swift://"): swifturi = SwiftUri(fname) try: conn = self._get_swift_connection() @@ -129,31 +133,27 @@ class IrAttachment(models.Model): _logger.exception( "error reading attachment '%s' from object storage", fname ) - return '' + return "" try: - resp, read = conn.get_object( - swifturi.container(), - swifturi.item() - ) + resp, read = conn.get_object(swifturi.container(), swifturi.item()) except ClientException: - read = '' - _logger.exception( - 'Error reading object from Swift object store') + read = "" + _logger.exception("Error reading object from Swift object store") return read else: return super()._store_file_read(fname) def _store_file_write(self, key, bin_data): - if self._storage() == 'swift': - container = os.environ.get('SWIFT_WRITE_CONTAINER') + if self._storage() == "swift": + container = os.environ.get("SWIFT_WRITE_CONTAINER") conn = self._get_swift_connection() conn.put_container(container) - filename = 'swift://{}/{}'.format(container, key) + filename = "swift://{}/{}".format(container, key) try: conn.put_object(container, key, bin_data) except ClientException: - _logger.exception('Error writing to Swift object store') - raise exceptions.UserError(_('Error writing to Swift')) + _logger.exception("Error writing to Swift object store") + raise exceptions.UserError(_("Error writing to Swift")) else: _super = super() filename = _super._store_file_write(key, bin_data) @@ -161,18 +161,17 @@ class IrAttachment(models.Model): @api.model def _store_file_delete(self, fname): - if fname.startswith('swift://'): + if fname.startswith("swift://"): swifturi = SwiftUri(fname) container = swifturi.container() # delete the file only if it is on the current configured bucket # otherwise, we might delete files used on a different environment - if container == os.environ.get('SWIFT_WRITE_CONTAINER'): + if container == os.environ.get("SWIFT_WRITE_CONTAINER"): conn = self._get_swift_connection() try: conn.delete_object(container, swifturi.item()) except ClientException: - _logger.exception( - _('Error deleting an object on the Swift store')) + _logger.exception(_("Error deleting an object on the Swift store")) # we ignore the error, file will stay on the object # storage but won't disrupt the process else: diff --git a/attachment_swift/swift_uri.py b/attachment_swift/swift_uri.py index fdd7ef4..f12c841 100644 --- a/attachment_swift/swift_uri.py +++ b/attachment_swift/swift_uri.py @@ -6,8 +6,7 @@ import re class SwiftUri(object): - _url_re = re.compile("^swift:///*([^/]*)/?(.*)", - re.IGNORECASE | re.UNICODE) + _url_re = re.compile("^swift:///*([^/]*)/?(.*)", re.IGNORECASE | re.UNICODE) def __init__(self, uri): match = self._url_re.match(uri) diff --git a/attachment_swift/tests/__init__.py b/attachment_swift/tests/__init__.py index 506e6b2..8a60b37 100644 --- a/attachment_swift/tests/__init__.py +++ b/attachment_swift/tests/__init__.py @@ -1,2 +1 @@ - from . import test_mock_swift_api diff --git a/attachment_swift/tests/test_mock_swift_api.py b/attachment_swift/tests/test_mock_swift_api.py index 11a5421..dc6aaa6 100644 --- a/attachment_swift/tests/test_mock_swift_api.py +++ b/attachment_swift/tests/test_mock_swift_api.py @@ -2,30 +2,27 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) import base64 -import mock import os +import keystoneauth1 +import mock from mock import patch -import keystoneauth1 - -from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment from odoo.addons.attachment_swift.models.ir_attachment import SwiftSessionStore from odoo.addons.attachment_swift.swift_uri import SwiftUri +from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment class TestAttachmentSwift(TestIrAttachment): - def setup(self): super().setUp() - self.env['ir.config_parameter'].set_param('ir_attachment.location', - 'swift') + self.env["ir.config_parameter"].set_param("ir_attachment.location", "swift") def test_session_store_get_session(self): - auth_url = 'auth_url' - username = 'username' - password = 'password' - project_name = 'project_name' + auth_url = "auth_url" + username = "username" + password = "password" + project_name = "project_name" store = SwiftSessionStore() session = store.get_session( auth_url=auth_url, @@ -34,10 +31,12 @@ class TestAttachmentSwift(TestIrAttachment): project_name=project_name, ) self.assertEqual(session.auth.auth_url, auth_url) - self.assertEqual(session.auth.get_cache_id_elements().get( - 'password_username'), username) - self.assertEqual(session.auth.get_cache_id_elements().get( - 'password_password'), password) + self.assertEqual( + session.auth.get_cache_id_elements().get("password_username"), username + ) + self.assertEqual( + session.auth.get_cache_id_elements().get("password_password"), password + ) self.assertEqual(session.auth.project_name, project_name) # get the same session on a second call @@ -48,73 +47,73 @@ class TestAttachmentSwift(TestIrAttachment): password=password, project_name=project_name, ), - session + session, ) - @patch('swiftclient.client') + @patch("swiftclient.client") def test_connection(self, mock_swift_client): - """ Test the connection to the store""" - os.environ['SWIFT_AUTH_URL'] = 'auth_url' - os.environ['SWIFT_ACCOUNT'] = 'account' - os.environ['SWIFT_PASSWORD'] = 'password' - os.environ['SWIFT_PROJECT_NAME'] = 'project_name' - os.environ['SWIFT_REGION_NAME'] = 'NOWHERE' + """Test the connection to the store""" + os.environ["SWIFT_AUTH_URL"] = "auth_url" + os.environ["SWIFT_ACCOUNT"] = "account" + os.environ["SWIFT_PASSWORD"] = "password" + os.environ["SWIFT_PROJECT_NAME"] = "project_name" + os.environ["SWIFT_REGION_NAME"] = "NOWHERE" attachment = self.Attachment attachment._get_swift_connection() mock_swift_client.Connection.assert_called_once_with( session=mock.ANY, - os_options={'region_name': os.environ.get('SWIFT_REGION_NAME')}, + os_options={"region_name": os.environ.get("SWIFT_REGION_NAME")}, ) __, kwargs = mock_swift_client.Connection.call_args - session = kwargs['session'] + session = kwargs["session"] self.assertTrue(isinstance(session, keystoneauth1.session.Session)) - self.assertEqual(session.auth.auth_url, os.environ['SWIFT_AUTH_URL']) - self.assertEqual(session.auth.get_cache_id_elements().get( - 'password_username'), os.environ['SWIFT_ACCOUNT']) - self.assertEqual(session.auth.get_cache_id_elements().get( - 'password_password'), os.environ['SWIFT_PASSWORD']) - self.assertEqual(session.auth.project_name, - os.environ['SWIFT_PROJECT_NAME']) + self.assertEqual(session.auth.auth_url, os.environ["SWIFT_AUTH_URL"]) + self.assertEqual( + session.auth.get_cache_id_elements().get("password_username"), + os.environ["SWIFT_ACCOUNT"], + ) + self.assertEqual( + session.auth.get_cache_id_elements().get("password_password"), + os.environ["SWIFT_PASSWORD"], + ) + self.assertEqual(session.auth.project_name, os.environ["SWIFT_PROJECT_NAME"]) def test_store_file_on_swift(self): """ - Test writing a file + Test writing a file """ - (self.env['ir.config_parameter']. - set_param('ir_attachment.location', 'swift')) - os.environ['SWIFT_AUTH_URL'] = 'auth_url' - os.environ['SWIFT_ACCOUNT'] = 'account' - os.environ['SWIFT_PASSWORD'] = 'password' - os.environ['SWIFT_PROJECT_NAME'] = 'project_name' - os.environ['SWIFT_WRITE_CONTAINER'] = 'my_container' - container = os.environ.get('SWIFT_WRITE_CONTAINER') + (self.env["ir.config_parameter"].set_param("ir_attachment.location", "swift")) + os.environ["SWIFT_AUTH_URL"] = "auth_url" + os.environ["SWIFT_ACCOUNT"] = "account" + os.environ["SWIFT_PASSWORD"] = "password" + os.environ["SWIFT_PROJECT_NAME"] = "project_name" + os.environ["SWIFT_WRITE_CONTAINER"] = "my_container" + container = os.environ.get("SWIFT_WRITE_CONTAINER") attachment = self.Attachment bin_data = base64.b64decode(self.blob1_b64) - with patch('swiftclient.client.Connection') as MockConnection: + with patch("swiftclient.client.Connection") as MockConnection: conn = MockConnection.return_value - attachment.create({'name': 'a5', 'datas': self.blob1_b64}) + attachment.create({"name": "a5", "datas": self.blob1_b64}) conn.put_object.assert_called_with( - container, - attachment._compute_checksum(bin_data), - bin_data) + container, attachment._compute_checksum(bin_data), bin_data + ) def test_delete_file_on_swift(self): """ - Test deleting a file + Test deleting a file """ - (self.env['ir.config_parameter']. - set_param('ir_attachment.location', 'swift')) - os.environ['SWIFT_AUTH_URL'] = 'auth_url' - os.environ['SWIFT_ACCOUNT'] = 'account' - os.environ['SWIFT_PASSWORD'] = 'password' - os.environ['SWIFT_PROJECT_NAME'] = 'project_name' - os.environ['SWIFT_WRITE_CONTAINER'] = 'my_container' + (self.env["ir.config_parameter"].set_param("ir_attachment.location", "swift")) + os.environ["SWIFT_AUTH_URL"] = "auth_url" + os.environ["SWIFT_ACCOUNT"] = "account" + os.environ["SWIFT_PASSWORD"] = "password" + os.environ["SWIFT_PROJECT_NAME"] = "project_name" + os.environ["SWIFT_WRITE_CONTAINER"] = "my_container" attachment = self.Attachment - container = os.environ.get('SWIFT_WRITE_CONTAINER') - with patch('swiftclient.client.Connection') as MockConnection: + container = os.environ.get("SWIFT_WRITE_CONTAINER") + with patch("swiftclient.client.Connection") as MockConnection: conn = MockConnection.return_value - a5 = attachment.create({'name': 'a5', 'datas': self.blob1_b64}) + a5 = attachment.create({"name": "a5", "datas": self.blob1_b64}) uri = SwiftUri(a5.store_fname) a5.unlink() conn.delete_object.assert_called_with(container, uri.item()) diff --git a/attachment_swift/tests/test_with_swift_store.py b/attachment_swift/tests/test_with_swift_store.py index e46ba41..fb60155 100644 --- a/attachment_swift/tests/test_with_swift_store.py +++ b/attachment_swift/tests/test_with_swift_store.py @@ -1,10 +1,12 @@ # Copyright 2017-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment -from ..swift_uri import SwiftUri from swiftclient.exceptions import ClientException +from odoo.addons.base.tests.test_ir_attachment import TestIrAttachment + +from ..swift_uri import SwiftUri + class TestAttachmentSwift(TestIrAttachment): """ @@ -13,27 +15,24 @@ class TestAttachmentSwift(TestIrAttachment): def setup(self): super().setUp() - self.env['ir.config_parameter'].set_param('ir_attachment.location', - 'swift') + self.env["ir.config_parameter"].set_param("ir_attachment.location", "swift") def test_connection(self): - """ Test the connection to the Swift object store """ + """Test the connection to the Swift object store""" conn = self.Attachment._get_swift_connection() self.assertNotEqual(conn, False) def test_store_file_on_swift(self): - """ Test writing a file and then reading it """ - (self.env['ir.config_parameter']. - set_param('ir_attachment.location', 'swift')) - a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64}) + """Test writing a file and then reading it""" + (self.env["ir.config_parameter"].set_param("ir_attachment.location", "swift")) + a5 = self.Attachment.create({"name": "a5", "datas": self.blob1_b64}) a5bis = self.Attachment.browse(a5.id)[0] self.assertEqual(a5.datas, a5bis.datas) def test_delete_file_on_swift(self): - """ Create a file and then test the deletion """ - (self.env['ir.config_parameter']. - set_param('ir_attachment.location', 'swift')) - a5 = self.Attachment.create({'name': 'a5', 'datas': self.blob1_b64}) + """Create a file and then test the deletion""" + (self.env["ir.config_parameter"].set_param("ir_attachment.location", "swift")) + a5 = self.Attachment.create({"name": "a5", "datas": self.blob1_b64}) uri = SwiftUri(a5.store_fname) con = self.Attachment._get_swift_connection() con.get_object(uri.container(), uri.item()) diff --git a/base_attachment_object_storage/__manifest__.py b/base_attachment_object_storage/__manifest__.py index e410519..f129fc6 100644 --- a/base_attachment_object_storage/__manifest__.py +++ b/base_attachment_object_storage/__manifest__.py @@ -1,16 +1,16 @@ # Copyright 2017-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - -{'name': 'Base Attachment Object Store', - 'summary': 'Base module for the implementation of external object store.', - 'version': "14.0.1.0.0", - 'author': 'Camptocamp,Odoo Community Association (OCA)', - 'license': 'AGPL-3', - 'category': 'Knowledge Management', - 'depends': ['base'], - 'website': 'http://www.camptocamp.com', - 'data': ['data/res_config_settings_data.xml'], - 'installable': True, - 'auto_install': True, - } +{ + "name": "Base Attachment Object Store", + "summary": "Base module for the implementation of external object store.", + "version": "14.0.1.1.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Knowledge Management", + "depends": ["base"], + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "data": ["data/res_config_settings_data.xml"], + "installable": True, + "auto_install": True, +} diff --git a/base_attachment_object_storage/data/res_config_settings_data.xml b/base_attachment_object_storage/data/res_config_settings_data.xml index 76c6961..4a1b8d4 100644 --- a/base_attachment_object_storage/data/res_config_settings_data.xml +++ b/base_attachment_object_storage/data/res_config_settings_data.xml @@ -1,9 +1,11 @@ - + ir_attachment.storage.force.database - {"image/": 51200, "application/javascript": 0, "text/css": 0} + {"image/": 51200, "application/javascript": 0, "text/css": 0} diff --git a/base_attachment_object_storage/models/ir_attachment.py b/base_attachment_object_storage/models/ir_attachment.py index ed43c69..8586ebf 100644 --- a/base_attachment_object_storage/models/ir_attachment.py +++ b/base_attachment_object_storage/models/ir_attachment.py @@ -5,53 +5,50 @@ import inspect import logging import os import time +from contextlib import closing, contextmanager from distutils.util import strtobool import psycopg2 -import odoo -from contextlib import closing, contextmanager -from odoo import api, exceptions, models, _ +import odoo +from odoo import _, api, exceptions, models from odoo.osv.expression import AND, OR, normalize_domain from odoo.tools.safe_eval import const_eval - _logger = logging.getLogger(__name__) def is_true(strval): - return bool(strtobool(strval or '0')) + return bool(strtobool(strval or "0")) def clean_fs(files): - _logger.info('cleaning old files from filestore') + _logger.info("cleaning old files from filestore") for full_path in files: if os.path.exists(full_path): try: os.unlink(full_path) except OSError: _logger.info( - "_file_delete could not unlink %s", - full_path, exc_info=True + "_file_delete could not unlink %s", full_path, exc_info=True ) except IOError: # Harmless and needed for race conditions _logger.info( - "_file_delete could not unlink %s", - full_path, exc_info=True + "_file_delete could not unlink %s", full_path, exc_info=True ) class IrAttachment(models.Model): - _inherit = 'ir.attachment' + _inherit = "ir.attachment" @staticmethod def is_storage_disabled(storage=None, log=True): msg = _("Storages are disabled (see environment configuration).") if storage: - msg = _( - "Storage '%s' is disabled (see environment configuration)." - ) % (storage,) + msg = _("Storage '%s' is disabled (see environment configuration).") % ( + storage, + ) is_disabled = is_true(os.environ.get("DISABLE_ATTACHMENT_STORAGE")) if is_disabled and log: _logger.warning(msg) @@ -59,7 +56,7 @@ class IrAttachment(models.Model): def _register_hook(self): super()._register_hook() - location = self.env.context.get('storage_location') or self._storage() + location = self.env.context.get("storage_location") or self._storage() # ignore if we are not using an object storage if location not in self._get_stores(): return @@ -73,7 +70,7 @@ class IrAttachment(models.Model): # done during the initialization. We need to move the attachments that # could have been created or updated in other addons before this addon # was loaded - update_module = load_modules_frame.f_locals.get('update_module') + update_module = load_modules_frame.f_locals.get("update_module") # We need to call the migration on the loading of the model because # when we are upgrading addons, some of them might add attachments. @@ -82,15 +79,19 @@ class IrAttachment(models.Model): # Typical example is images of ir.ui.menu which are updated in # ir.attachment at every upgrade of the addons if update_module: - self.env['ir.attachment'].sudo()._force_storage_to_object_storage() + self.env["ir.attachment"].sudo()._force_storage_to_object_storage() @property def _object_storage_default_force_db_config(self): return {"image/": 51200, "application/javascript": 0, "text/css": 0} def _get_storage_force_db_config(self): - param = self.env['ir.config_parameter'].sudo().get_param( - 'ir_attachment.storage.force.database', + param = ( + self.env["ir.config_parameter"] + .sudo() + .get_param( + "ir_attachment.storage.force.database", + ) ) storage_config = None if param: @@ -100,7 +101,8 @@ class IrAttachment(models.Model): _logger.exception( "Could not parse system parameter" " 'ir_attachment.storage.force.database', reverting to the" - " default configuration.") + " default configuration." + ) if not storage_config: storage_config = self._object_storage_default_force_db_config @@ -128,7 +130,7 @@ class IrAttachment(models.Model): return domain def _store_in_db_instead_of_object_storage(self, data, mimetype): - """ Return whether an attachment must be stored in db + """Return whether an attachment must be stored in db When we are using an Object Storage. This is sometimes required because the object storage is slower than the database/filesystem. @@ -180,17 +182,17 @@ class IrAttachment(models.Model): return False def _get_datas_related_values(self, data, mimetype): - storage = self.env.context.get('storage_location') or self._storage() + storage = self.env.context.get("storage_location") or self._storage() if data and storage in self._get_stores(): if self._store_in_db_instead_of_object_storage(data, mimetype): # compute the fields that depend on datas bin_data = data values = { - 'file_size': len(bin_data), - 'checksum': self._compute_checksum(bin_data), - 'index_content': self._index(bin_data, mimetype), - 'store_fname': False, - 'db_datas': data, + "file_size": len(bin_data), + "checksum": self._compute_checksum(bin_data), + "index_content": self._index(bin_data, mimetype), + "store_fname": False, + "db_datas": data, } return values return super()._get_datas_related_values(data, mimetype) @@ -203,28 +205,22 @@ class IrAttachment(models.Model): return super()._file_read(fname) def _store_file_read(self, fname): - storage = fname.partition('://')[0] - raise NotImplementedError( - 'No implementation for %s' % (storage,) - ) + storage = fname.partition("://")[0] + raise NotImplementedError("No implementation for %s" % (storage,)) def _store_file_write(self, key, bin_data): storage = self.storage() - raise NotImplementedError( - 'No implementation for %s' % (storage,) - ) + raise NotImplementedError("No implementation for %s" % (storage,)) def _store_file_delete(self, fname): - storage = fname.partition('://')[0] - raise NotImplementedError( - 'No implementation for %s' % (storage,) - ) + storage = fname.partition("://")[0] + raise NotImplementedError("No implementation for %s" % (storage,)) @api.model def _file_write(self, bin_data, checksum): - location = self.env.context.get('storage_location') or self._storage() + location = self.env.context.get("storage_location") or self._storage() if location in self._get_stores(): - key = self.env.context.get('force_storage_key') + key = self.env.context.get("force_storage_key") if not key: key = self._compute_checksum(bin_data) filename = self._store_file_write(key, bin_data) @@ -238,8 +234,9 @@ class IrAttachment(models.Model): cr = self.env.cr # using SQL to include files hidden through unlink or due to record # rules - cr.execute("SELECT COUNT(*) FROM ir_attachment " - "WHERE store_fname = %s", (fname,)) + cr.execute( + "SELECT COUNT(*) FROM ir_attachment WHERE store_fname = %s", (fname,) + ) count = cr.fetchone()[0] if not count: self._store_file_delete(fname) @@ -251,22 +248,20 @@ class IrAttachment(models.Model): for store_name in self._get_stores(): if self.is_storage_disabled(store_name): continue - uri = '{}://'.format(store_name) + uri = "{}://".format(store_name) if fname.startswith(uri): return True return False @contextmanager def do_in_new_env(self, new_cr=False): - """ Context manager that yields a new environment + """Context manager that yields a new environment Using a new Odoo Environment thus a new PG transaction. """ with api.Environment.manage(): if new_cr: - registry = odoo.modules.registry.Registry.new( - self.env.cr.dbname - ) + registry = odoo.modules.registry.Registry.new(self.env.cr.dbname) with closing(registry.cursor()) as cr: try: yield self.env(cr=cr) @@ -283,33 +278,38 @@ class IrAttachment(models.Model): def _move_attachment_to_store(self): self.ensure_one() - _logger.info('inspecting attachment %s (%d)', self.name, self.id) + _logger.info("inspecting attachment %s (%d)", self.name, self.id) fname = self.store_fname - storage = fname.partition('://')[0] + storage = fname.partition("://")[0] if self.is_storage_disabled(storage): fname = False if fname: # migrating from filesystem filestore # or from the old 'store_fname' without the bucket name - _logger.info('moving %s on the object storage', fname) - self.write({'datas': self.datas, - # this is required otherwise the - # mimetype gets overriden with - # 'application/octet-stream' - # on assets - 'mimetype': self.mimetype}) - _logger.info('moved %s on the object storage', fname) + _logger.info("moving %s on the object storage", fname) + self.write( + { + "datas": self.datas, + # this is required otherwise the + # mimetype gets overriden with + # 'application/octet-stream' + # on assets + "mimetype": self.mimetype, + } + ) + _logger.info("moved %s on the object storage", fname) return self._full_path(fname) elif self.db_datas: - _logger.info('moving on the object storage from database') - self.write({'datas': self.datas}) + _logger.info("moving on the object storage from database") + self.write({"datas": self.datas}) @api.model def force_storage(self): - if not self.env['res.users'].browse(self.env.uid)._is_admin(): + if not self.env["res.users"].browse(self.env.uid)._is_admin(): raise exceptions.AccessError( - _('Only administrators can execute this action.')) - location = self.env.context.get('storage_location') or self._storage() + _("Only administrators can execute this action.") + ) + location = self.env.context.get("storage_location") or self._storage() if location not in self._get_stores(): return super().force_storage() self._force_storage_to_object_storage() @@ -335,30 +335,32 @@ class IrAttachment(models.Model): if storage not in self._get_stores(): return - domain = AND(( - normalize_domain( - [('store_fname', '=like', '{}://%'.format(storage)), - # for res_field, see comment in - # _force_storage_to_object_storage - '|', - ('res_field', '=', False), - ('res_field', '!=', False), - ] - ), - normalize_domain(self._store_in_db_instead_of_object_storage_domain()) - )) + domain = AND( + ( + normalize_domain( + [ + ("store_fname", "=like", "{}://%".format(storage)), + # for res_field, see comment in + # _force_storage_to_object_storage + "|", + ("res_field", "=", False), + ("res_field", "!=", False), + ] + ), + normalize_domain(self._store_in_db_instead_of_object_storage_domain()), + ) + ) with self.do_in_new_env(new_cr=new_cr) as new_env: - model_env = new_env['ir.attachment'].with_context( - prefetch_fields=False - ) + model_env = new_env["ir.attachment"].with_context(prefetch_fields=False) attachment_ids = model_env.search(domain).ids if not attachment_ids: return total = len(attachment_ids) start_time = time.time() - _logger.info('Moving %d attachments from %s to' - ' DB for fast access', total, storage) + _logger.info( + "Moving %d attachments from %s to" " DB for fast access", total, storage + ) current = 0 for attachment_id in attachment_ids: current += 1 @@ -370,38 +372,42 @@ class IrAttachment(models.Model): # this write will read the datas from the Object Storage and # write them back in the DB (the logic for location to write is # in the 'datas' inverse computed field) - attachment.write({'datas': attachment.datas}) + attachment.write({"datas": attachment.datas}) # as the file will potentially be dropped on the bucket, # we should commit the changes here new_env.cr.commit() if current % 100 == 0 or total - current == 0: _logger.info( - 'attachment %s/%s after %.2fs', - current, total, - time.time() - start_time + "attachment %s/%s after %.2fs", + current, + total, + time.time() - start_time, ) @api.model def _force_storage_to_object_storage(self, new_cr=False): - _logger.info('migrating files to the object storage') - storage = self.env.context.get('storage_location') or self._storage() + _logger.info("migrating files to the object storage") + storage = self.env.context.get("storage_location") or self._storage() if self.is_storage_disabled(storage): return # The weird "res_field = False OR res_field != False" domain # is required! It's because of an override of _search in ir.attachment # which adds ('res_field', '=', False) when the domain does not # contain 'res_field'. - # https://github.com/odoo/odoo/blob/9032617120138848c63b3cfa5d1913c5e5ad76db/odoo/addons/base/ir/ir_attachment.py#L344-L347 - domain = ['!', ('store_fname', '=like', '{}://%'.format(storage)), - '|', - ('res_field', '=', False), - ('res_field', '!=', False)] + # https://github.com/odoo/odoo/blob/9032617120138848c63b3cfa5d1913c5e5ad76db/odoo/addons/base/ir/ir_attachment.py#L344-L347 # noqa: B950 + domain = [ + "!", + ("store_fname", "=like", "{}://%".format(storage)), + "|", + ("res_field", "=", False), + ("res_field", "!=", False), + ] # We do a copy of the environment so we can workaround the cache issue # below. We do not create a new cursor by default because it causes # serialization issues due to concurrent updates on attachments during # the installation with self.do_in_new_env(new_cr=new_cr) as new_env: - model_env = new_env['ir.attachment'] + model_env = new_env["ir.attachment"] ids = model_env.search(domain).ids files_to_clean = [] for attachment_id in ids: @@ -410,12 +416,14 @@ class IrAttachment(models.Model): # check that no other transaction has # locked the row, don't send a file to storage # in that case - self.env.cr.execute("SELECT id " - "FROM ir_attachment " - "WHERE id = %s " - "FOR UPDATE NOWAIT", - (attachment_id,), - log_exceptions=False) + self.env.cr.execute( + "SELECT id " + "FROM ir_attachment " + "WHERE id = %s " + "FOR UPDATE NOWAIT", + (attachment_id,), + log_exceptions=False, + ) # This is a trick to avoid having the 'datas' # function fields computed for every attachment on @@ -428,8 +436,9 @@ class IrAttachment(models.Model): if path: files_to_clean.append(path) except psycopg2.OperationalError: - _logger.error('Could not migrate attachment %s to S3', - attachment_id) + _logger.error( + "Could not migrate attachment %s to S3", attachment_id + ) def clean(): clean_fs(files_to_clean) @@ -437,8 +446,8 @@ class IrAttachment(models.Model): # delete the files from the filesystem once we know the changes # have been committed in ir.attachment if files_to_clean: - new_env.cr.after('commit', clean) + new_env.cr.after("commit", clean) def _get_stores(self): - """ To get the list of stores activated in the system """ + """To get the list of stores activated in the system""" return [] diff --git a/base_fileurl_field/__init__.py b/base_fileurl_field/__init__.py index 08405c5..7354438 100644 --- a/base_fileurl_field/__init__.py +++ b/base_fileurl_field/__init__.py @@ -1,2 +1 @@ from . import fields - diff --git a/base_fileurl_field/__manifest__.py b/base_fileurl_field/__manifest__.py index 2fe939e..247abba 100644 --- a/base_fileurl_field/__manifest__.py +++ b/base_fileurl_field/__manifest__.py @@ -5,11 +5,10 @@ "summary": "Implementation of FileURL type fields", "version": "14.0.1.0.0", "category": "Technical Settings", - 'author': 'Camptocamp, Odoo Community Association (OCA)', - 'license': 'AGPL-3', - "depends": [ - "base_attachment_object_storage", - ], + "author": "Camptocamp, Odoo Community Association (OCA)", + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "license": "AGPL-3", + "depends": ["base_attachment_object_storage"], "auto_install": False, "installable": True, } diff --git a/base_fileurl_field/fields.py b/base_fileurl_field/fields.py index 133082a..f87b2d2 100644 --- a/base_fileurl_field/fields.py +++ b/base_fileurl_field/fields.py @@ -4,7 +4,6 @@ import unicodedata from odoo import fields - fields.Field.__doc__ += """ .. _field-fileurl: @@ -29,10 +28,10 @@ fields.Field.__doc__ += """ class FileURL(fields.Binary): _slots = { - 'attachment': True, # Override default with True - 'storage_location': '', # External storage activated on the system (cf base_attachment_storage) # noqa - 'storage_path': '', # Path to be used as storage key (prefix of filename) # noqa - 'filename': '', # Field to use to store the filename on ir.attachment + "attachment": True, # Override default with True + "storage_location": "", # External storage activated on the system (cf base_attachment_storage) # noqa + "storage_path": "", # Path to be used as storage key (prefix of filename) # noqa + "filename": "", # Field to use to store the filename on ir.attachment } # pylint: disable=method-required-super @@ -47,26 +46,27 @@ class FileURL(fields.Binary): if not value: continue vals = { - 'name': self.name, - 'res_model': self.model_name, - 'res_field': self.name, - 'res_id': record.id, - 'type': 'binary', - 'datas': value, + "name": self.name, + "res_model": self.model_name, + "res_field": self.name, + "res_id": record.id, + "type": "binary", + "datas": value, } fname = False if self.filename: fname = record[self.filename] - vals['datas_fname'] = fname + vals["datas_fname"] = fname if fname and self.storage_path: storage_key = self._build_storage_key(fname) if not fname: storage_key = False - env['ir.attachment'].sudo().with_context( + env["ir.attachment"].sudo().with_context( binary_field_real_user=env.user, storage_location=self.storage_location, force_storage_key=storage_key, ).create(vals) + return super().create(record_values) def write(self, records, value): for record in records: @@ -80,21 +80,21 @@ class FileURL(fields.Binary): storage_location=self.storage_location, force_storage_key=storage_key, ), - value + value, ) return True def _setup_regular_base(self, model): super()._setup_regular_base(model) if self.storage_path: - assert self.filename is not None, \ + assert self.filename is not None, ( "Field %s defines storage_path without filename" % self + ) def _build_storage_key(self, filename): - return '/'.join([ - self.storage_path.rstrip('/'), - unicodedata.normalize('NFKC', filename) - ]) + return "/".join( + [self.storage_path.rstrip("/"), unicodedata.normalize("NFKC", filename)] + ) fields.FileURL = FileURL diff --git a/cloud_platform/__init__.py b/cloud_platform/__init__.py index a9e3372..0650744 100644 --- a/cloud_platform/__init__.py +++ b/cloud_platform/__init__.py @@ -1,2 +1 @@ - from . import models diff --git a/cloud_platform/__manifest__.py b/cloud_platform/__manifest__.py index ed35c60..19cf0e0 100644 --- a/cloud_platform/__manifest__.py +++ b/cloud_platform/__manifest__.py @@ -2,19 +2,20 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -{'name': 'Cloud Platform', - 'summary': 'Addons required for the Camptocamp Cloud Platform', - 'version': "14.0.2.0.0", - 'author': 'Camptocamp,Odoo Community Association (OCA)', - 'license': 'AGPL-3', - 'category': 'Extra Tools', - 'depends': [ - 'session_redis', - 'monitoring_status', - 'logging_json', - 'server_environment', # OCA/server-tools - ], - 'website': 'https://www.camptocamp.com', - 'data': [], - 'installable': True, - } +{ + "name": "Cloud Platform", + "summary": "Addons required for the Camptocamp Cloud Platform", + "version": "14.0.2.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Extra Tools", + "depends": [ + "session_redis", + "monitoring_status", + "logging_json", + "server_environment", # OCA/server-tools + ], + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "data": [], + "installable": True, +} diff --git a/cloud_platform/models/__init__.py b/cloud_platform/models/__init__.py index fe15395..5d08f36 100644 --- a/cloud_platform/models/__init__.py +++ b/cloud_platform/models/__init__.py @@ -1,2 +1 @@ - from . import cloud_platform diff --git a/cloud_platform/models/cloud_platform.py b/cloud_platform/models/cloud_platform.py index 750f4bd..4d8c499 100644 --- a/cloud_platform/models/cloud_platform.py +++ b/cloud_platform/models/cloud_platform.py @@ -4,46 +4,38 @@ import logging import os import re - from collections import namedtuple from distutils.util import strtobool from odoo import api, models from odoo.tools.config import config - _logger = logging.getLogger(__name__) def is_true(strval): - return bool(strtobool(strval or '0')) + return bool(strtobool(strval or "0")) -PlatformConfig = namedtuple( - 'PlatformConfig', - 'filestore' -) +PlatformConfig = namedtuple("PlatformConfig", "filestore") -FilestoreKind = namedtuple( - 'FilestoreKind', - ['name', 'location'] -) +FilestoreKind = namedtuple("FilestoreKind", ["name", "location"]) class CloudPlatform(models.AbstractModel): - _name = 'cloud.platform' - _description = 'cloud.platform' + _name = "cloud.platform" + _description = "cloud.platform" @api.model def _default_config(self): - return PlatformConfig(self._filestore_kinds()['db']) + return PlatformConfig(self._filestore_kinds()["db"]) @api.model def _filestore_kinds(self): return { - 'db': FilestoreKind('db', 'local'), - 'file': FilestoreKind('file', 'local'), + "db": FilestoreKind("db", "local"), + "file": FilestoreKind("file", "local"), } @api.model @@ -53,33 +45,31 @@ class CloudPlatform(models.AbstractModel): @api.model def _config_by_server_env(self, platform_kind, environment): configs_getter = getattr( - self, - '_config_by_server_env_for_%s' % platform_kind, - None + self, "_config_by_server_env_for_%s" % platform_kind, None ) configs = configs_getter() if configs_getter else {} return configs.get(environment) or self._default_config() def _get_running_env(self): - environment_name = config['running_env'] - if environment_name.startswith('labs'): + environment_name = config["running_env"] + if environment_name.startswith("labs"): # We allow to have environments such as 'labs-logistics' # or 'labs-finance', in order to have the matching ribbon. - environment_name = 'labs' + environment_name = "labs" return environment_name @api.model def _install(self, platform_kind): assert platform_kind in self._platform_kinds() - params = self.env['ir.config_parameter'].sudo() - params.set_param('cloud.platform.kind', platform_kind) + params = self.env["ir.config_parameter"].sudo() + params.set_param("cloud.platform.kind", platform_kind) environment_name = self._get_running_env() configs = self._config_by_server_env(platform_kind, environment_name) - params.set_param('ir_attachment.location', configs.filestore.name) + params.set_param("ir_attachment.location", configs.filestore.name) self.check() - if configs.filestore.location == 'remote': - self.env['ir.attachment'].sudo().force_storage() - _logger.info('cloud platform configured for {}'.format(platform_kind)) + if configs.filestore.location == "remote": + self.env["ir.attachment"].sudo().force_storage() + _logger.info("cloud platform configured for {}".format(platform_kind)) @api.model def install(self): @@ -91,39 +81,39 @@ class CloudPlatform(models.AbstractModel): @api.model def _check_redis(self, environment_name): - if environment_name in ('prod', 'integration', 'labs', 'test'): - assert is_true(os.environ.get('ODOO_SESSION_REDIS')), ( + if environment_name in ("prod", "integration", "labs", "test"): + assert is_true(os.environ.get("ODOO_SESSION_REDIS")), ( "Redis must be activated on prod, integration, labs," " test instances. This is done by setting ODOO_SESSION_REDIS=1." ) - assert (os.environ.get('ODOO_SESSION_REDIS_HOST') or - os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST') or - os.environ.get('ODOO_SESSION_REDIS_URL')), ( + assert ( + os.environ.get("ODOO_SESSION_REDIS_HOST") + or os.environ.get("ODOO_SESSION_REDIS_SENTINEL_HOST") + or os.environ.get("ODOO_SESSION_REDIS_URL") + ), ( "ODOO_SESSION_REDIS_HOST or " "ODOO_SESSION_REDIS_SENTINEL_HOST or " "ODOO_SESSION_REDIS_URL " "environment variable is required to connect on Redis" ) - assert os.environ.get('ODOO_SESSION_REDIS_PREFIX'), ( + assert os.environ.get("ODOO_SESSION_REDIS_PREFIX"), ( "ODOO_SESSION_REDIS_PREFIX environment variable is required " "to store sessions on Redis" ) - prefix = os.environ['ODOO_SESSION_REDIS_PREFIX'] - assert re.match(r'^[a-z-0-9]+-odoo-[a-z-0-9]+$', prefix), ( + prefix = os.environ["ODOO_SESSION_REDIS_PREFIX"] + assert re.match(r"^[a-z-0-9]+-odoo-[a-z-0-9]+$", prefix), ( "ODOO_SESSION_REDIS_PREFIX must match '-odoo-'" ", we got: '%s'" % (prefix,) ) @api.model def check(self): - if is_true(os.environ.get('ODOO_CLOUD_PLATFORM_UNSAFE')): - _logger.warning( - "cloud platform checks disabled, this is not safe" - ) + if is_true(os.environ.get("ODOO_CLOUD_PLATFORM_UNSAFE")): + _logger.warning("cloud platform checks disabled, this is not safe") return - params = self.env['ir.config_parameter'].sudo() - kind = params.get_param('cloud.platform.kind') + params = self.env["ir.config_parameter"].sudo() + kind = params.get_param("cloud.platform.kind") if not kind: _logger.warning( "cloud platform not configured, you should " diff --git a/cloud_platform/songs.py b/cloud_platform/songs.py index 043fc7b..1a0f788 100644 --- a/cloud_platform/songs.py +++ b/cloud_platform/songs.py @@ -1,3 +1,2 @@ - def install(ctx): - ctx.env['cloud.platform'].install() + ctx.env["cloud.platform"].install() diff --git a/cloud_platform_azure/README.md b/cloud_platform_azure/README.md index 1f7bd5d..449ab29 100644 --- a/cloud_platform_azure/README.md +++ b/cloud_platform_azure/README.md @@ -1,5 +1,4 @@ -Cloud Platform Azure -==================== +# Cloud Platform Azure Install addons specific to the Azure setup. diff --git a/cloud_platform_azure/__manifest__.py b/cloud_platform_azure/__manifest__.py index 865e000..937dcdc 100644 --- a/cloud_platform_azure/__manifest__.py +++ b/cloud_platform_azure/__manifest__.py @@ -9,16 +9,9 @@ "author": "Camptocamp,Odoo Community Association (OCA)", "license": "AGPL-3", "category": "Extra Tools", - "depends": [ - "cloud_platform", - "attachment_azure", - "monitoring_prometheus", - ], - "excludes": [ - "cloud_platform_ovh", - "cloud_platform_exoscale", - ], - "website": "https://www.camptocamp.com", + "depends": ["cloud_platform", "attachment_azure", "monitoring_prometheus"], + "excludes": ["cloud_platform_ovh", "cloud_platform_exoscale"], + "website": "https://github.com/camptocamp/odoo-cloud-platform", "data": [], "installable": True, } diff --git a/cloud_platform_azure/models/cloud_platform.py b/cloud_platform_azure/models/cloud_platform.py index e93e706..b72d229 100644 --- a/cloud_platform_azure/models/cloud_platform.py +++ b/cloud_platform_azure/models/cloud_platform.py @@ -1,13 +1,15 @@ # Copyright 2016-2021 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -import re import os +import re -from odoo import models, api -from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind -from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig +from odoo import api, models +from odoo.addons.cloud_platform.models.cloud_platform import ( + FilestoreKind, + PlatformConfig, +) AZURE_STORE_KIND = FilestoreKind("azure", "remote") @@ -42,8 +44,7 @@ class CloudPlatform(models.AbstractModel): @api.model def _check_filestore(self, environment_name): params = self.env["ir.config_parameter"].sudo() - use_azure = (params.get_param("ir_attachment.location") == - AZURE_STORE_KIND.name) + use_azure = params.get_param("ir_attachment.location") == AZURE_STORE_KIND.name if environment_name in ("prod", "integration"): # Labs instances use azure by default, but we don't want # to enforce it in case we want to test something with a different diff --git a/cloud_platform_exoscale/README.md b/cloud_platform_exoscale/README.md index f2931ea..3c815ed 100644 --- a/cloud_platform_exoscale/README.md +++ b/cloud_platform_exoscale/README.md @@ -1,5 +1,4 @@ -Cloud Platform Exoscale -======================= +# Cloud Platform Exoscale Install addons specific to the Exoscale setup. diff --git a/cloud_platform_exoscale/__manifest__.py b/cloud_platform_exoscale/__manifest__.py index df8d203..44db0c0 100644 --- a/cloud_platform_exoscale/__manifest__.py +++ b/cloud_platform_exoscale/__manifest__.py @@ -9,15 +9,9 @@ "author": "Camptocamp,Odoo Community Association (OCA)", "license": "AGPL-3", "category": "Extra Tools", - "depends": [ - "cloud_platform", - "attachment_s3", - "monitoring_statsd", - ], - "excludes": [ - "cloud_platform_ovh", - ], - "website": "https://www.camptocamp.com", + "depends": ["cloud_platform", "attachment_s3", "monitoring_statsd"], + "excludes": ["cloud_platform_ovh", "cloud_platform_azure"], + "website": "https://github.com/camptocamp/odoo-cloud-platform", "data": [], "installable": True, } diff --git a/cloud_platform_exoscale/models/cloud_platform.py b/cloud_platform_exoscale/models/cloud_platform.py index 3a29942..66702fd 100644 --- a/cloud_platform_exoscale/models/cloud_platform.py +++ b/cloud_platform_exoscale/models/cloud_platform.py @@ -1,50 +1,51 @@ # Copyright 2016-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -import re import os +import re -from odoo import models, api -from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind -from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig +from odoo import api, models +from odoo.addons.cloud_platform.models.cloud_platform import ( + FilestoreKind, + PlatformConfig, +) -S3_STORE_KIND = FilestoreKind('s3', 'remote') +S3_STORE_KIND = FilestoreKind("s3", "remote") class CloudPlatform(models.AbstractModel): - _inherit = 'cloud.platform' + _inherit = "cloud.platform" @api.model def _filestore_kinds(self): kinds = super(CloudPlatform, self)._filestore_kinds() - kinds['s3'] = S3_STORE_KIND + kinds["s3"] = S3_STORE_KIND return kinds @api.model def _platform_kinds(self): kinds = super(CloudPlatform, self)._platform_kinds() - kinds.append('exoscale') + kinds.append("exoscale") return kinds @api.model def _config_by_server_env_for_exoscale(self): fs_kinds = self._filestore_kinds() configs = { - 'prod': PlatformConfig(filestore=fs_kinds['s3']), - 'integration': PlatformConfig(filestore=fs_kinds['s3']), - 'labs': PlatformConfig(filestore=fs_kinds['s3']), - 'test': PlatformConfig(filestore=fs_kinds['db']), - 'dev': PlatformConfig(filestore=fs_kinds['db']), + "prod": PlatformConfig(filestore=fs_kinds["s3"]), + "integration": PlatformConfig(filestore=fs_kinds["s3"]), + "labs": PlatformConfig(filestore=fs_kinds["s3"]), + "test": PlatformConfig(filestore=fs_kinds["db"]), + "dev": PlatformConfig(filestore=fs_kinds["db"]), } return configs @api.model def _check_filestore(self, environment_name): - params = self.env['ir.config_parameter'].sudo() - use_s3 = (params.get_param('ir_attachment.location') == - S3_STORE_KIND.name) - if environment_name in ('prod', 'integration'): + params = self.env["ir.config_parameter"].sudo() + use_s3 = params.get_param("ir_attachment.location") == S3_STORE_KIND.name + if environment_name in ("prod", "integration"): # Labs instances use s3 by default, but we don't want # to enforce it in case we want to test something with a different # storage. At your own risks! @@ -55,16 +56,16 @@ class CloudPlatform(models.AbstractModel): "automatically." ) if use_s3: - assert os.environ.get('AWS_ACCESS_KEY_ID'), ( + assert os.environ.get("AWS_ACCESS_KEY_ID"), ( "AWS_ACCESS_KEY_ID environment variable is required when " "ir_attachment.location is 's3'." ) - assert os.environ.get('AWS_SECRET_ACCESS_KEY'), ( + assert os.environ.get("AWS_SECRET_ACCESS_KEY"), ( "AWS_SECRET_ACCESS_KEY environment variable is required when " "ir_attachment.location is 's3'." ) - bucket_name = os.environ.get('AWS_BUCKETNAME', '') - if environment_name in ('prod', 'integration', 'labs'): + bucket_name = os.environ.get("AWS_BUCKETNAME", "") + if environment_name in ("prod", "integration", "labs"): assert bucket_name, ( "AWS_BUCKETNAME environment variable is required when " "ir_attachment.location is 's3'.\n" @@ -80,10 +81,10 @@ class CloudPlatform(models.AbstractModel): # # Use AWS_BUCKETNAME_UNSTRUCTURED to by-pass check on bucket name # structure - if os.environ.get('AWS_BUCKETNAME_UNSTRUCTURED'): + if os.environ.get("AWS_BUCKETNAME_UNSTRUCTURED"): return - prod_bucket = bool(re.match(r'[a-z-0-9]+-odoo-prod', bucket_name)) - if environment_name == 'prod': + prod_bucket = bool(re.match(r"[a-z-0-9]+-odoo-prod", bucket_name)) + if environment_name == "prod": assert prod_bucket, ( "AWS_BUCKETNAME should match '-odoo-prod', " "we got: '%s'" % (bucket_name,) @@ -96,9 +97,9 @@ class CloudPlatform(models.AbstractModel): "we got: '%s'" % (bucket_name,) ) - elif environment_name == 'test': + elif environment_name == "test": # store in DB so we don't have files local to the host - assert params.get_param('ir_attachment.location') == 'db', ( + assert params.get_param("ir_attachment.location") == "db", ( "In test instances, files must be stored in the database with " "'ir_attachment.location' set to 'db'. This is " "automatically set by the function 'install()'." @@ -106,4 +107,4 @@ class CloudPlatform(models.AbstractModel): @api.model def install(self): - self._install('exoscale') + self._install("exoscale") diff --git a/cloud_platform_ovh/README.md b/cloud_platform_ovh/README.md index c350eba..f82fe48 100644 --- a/cloud_platform_ovh/README.md +++ b/cloud_platform_ovh/README.md @@ -1,7 +1,5 @@ -Cloud Platform OVH -================== +# Cloud Platform OVH Install addons specific to the OVH setup. * The object storage is Swift - diff --git a/cloud_platform_ovh/__manifest__.py b/cloud_platform_ovh/__manifest__.py index b43ad30..ab88843 100644 --- a/cloud_platform_ovh/__manifest__.py +++ b/cloud_platform_ovh/__manifest__.py @@ -1,7 +1,6 @@ # Copyright 2017-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - { "name": "Cloud Platform OVH", "summary": "Addons required for the Camptocamp Cloud Platform on OVH", @@ -9,15 +8,9 @@ "author": "Camptocamp,Odoo Community Association (OCA)", "license": "AGPL-3", "category": "Extra Tools", - "depends": [ - "cloud_platform", - "attachment_swift", - "monitoring_statsd", - ], - "excludes": [ - "cloud_platform_exoscale", - ], - "website": "https://www.camptocamp.com", + "depends": ["cloud_platform", "attachment_swift", "monitoring_statsd"], + "excludes": ["cloud_platform_exoscale", "cloud_platform_azure"], + "website": "https://github.com/camptocamp/odoo-cloud-platform", "data": [], "installable": True, } diff --git a/cloud_platform_ovh/models/cloud_platform.py b/cloud_platform_ovh/models/cloud_platform.py index c4e0216..bed4db9 100644 --- a/cloud_platform_ovh/models/cloud_platform.py +++ b/cloud_platform_ovh/models/cloud_platform.py @@ -1,51 +1,51 @@ # Copyright 2017-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -import re import os +import re from odoo import api, models -from odoo.addons.cloud_platform.models.cloud_platform import FilestoreKind -from odoo.addons.cloud_platform.models.cloud_platform import PlatformConfig +from odoo.addons.cloud_platform.models.cloud_platform import ( + FilestoreKind, + PlatformConfig, +) - -SWIFT_STORE_KIND = FilestoreKind('swift', 'remote') +SWIFT_STORE_KIND = FilestoreKind("swift", "remote") class CloudPlatform(models.AbstractModel): - _inherit = 'cloud.platform' + _inherit = "cloud.platform" @api.model def _filestore_kinds(self): kinds = super(CloudPlatform, self)._filestore_kinds() - kinds['swift'] = SWIFT_STORE_KIND + kinds["swift"] = SWIFT_STORE_KIND return kinds @api.model def _platform_kinds(self): kinds = super()._platform_kinds() - kinds.append('ovh') + kinds.append("ovh") return kinds @api.model def _config_by_server_env_for_ovh(self): fs_kinds = self._filestore_kinds() configs = { - 'prod': PlatformConfig(filestore=fs_kinds['swift']), - 'integration': PlatformConfig(filestore=fs_kinds['swift']), - 'labs': PlatformConfig(filestore=fs_kinds['swift']), - 'test': PlatformConfig(filestore=fs_kinds['db']), - 'dev': PlatformConfig(filestore=fs_kinds['db']), + "prod": PlatformConfig(filestore=fs_kinds["swift"]), + "integration": PlatformConfig(filestore=fs_kinds["swift"]), + "labs": PlatformConfig(filestore=fs_kinds["swift"]), + "test": PlatformConfig(filestore=fs_kinds["db"]), + "dev": PlatformConfig(filestore=fs_kinds["db"]), } return configs @api.model def _check_filestore(self, environment_name): - params = self.env['ir.config_parameter'].sudo() - use_swift = (params.get_param('ir_attachment.location') == - SWIFT_STORE_KIND.name) - if environment_name in ('prod', 'integration'): + params = self.env["ir.config_parameter"].sudo() + use_swift = params.get_param("ir_attachment.location") == SWIFT_STORE_KIND.name + if environment_name in ("prod", "integration"): # Labs instances use swift by default, but we don't want # to enforce it in case we want to test something with a different # storage. At your own risks! @@ -56,20 +56,20 @@ class CloudPlatform(models.AbstractModel): "automatically." ) if use_swift: - assert os.environ.get('SWIFT_AUTH_URL'), ( + assert os.environ.get("SWIFT_AUTH_URL"), ( "SWIFT_AUTH_URL environment variable is required when " "ir_attachment.location is 'swift'." ) - assert os.environ.get('SWIFT_ACCOUNT'), ( + assert os.environ.get("SWIFT_ACCOUNT"), ( "SWIFT_ACCOUNT environment variable is required when " "ir_attachment.location is 'swift'." ) - assert os.environ.get('SWIFT_PASSWORD'), ( + assert os.environ.get("SWIFT_PASSWORD"), ( "SWIFT_PASSWORD environment variable is required when " "ir_attachment.location is 'swift'." ) - container_name = os.environ.get('SWIFT_WRITE_CONTAINER', '') - if environment_name in ('prod', 'integration', 'labs'): + container_name = os.environ.get("SWIFT_WRITE_CONTAINER", "") + if environment_name in ("prod", "integration", "labs"): assert container_name, ( "SWIFT_WRITE_CONTAINER environment variable is required when " "ir_attachment.location is 'swift'.\n" @@ -80,16 +80,15 @@ class CloudPlatform(models.AbstractModel): "If you don't actually need a bucket, change the" " 'ir_attachment.location' parameter." ) - prod_container = bool(re.match(r'[a-z0-9-]+-odoo-prod', - container_name)) + prod_container = bool(re.match(r"[a-z0-9-]+-odoo-prod", container_name)) # A bucket name is defined under the following format # -odoo- # # Use SWIFT_WRITE_CONTAINER_UNSTRUCTURED to by-pass check on bucket name # structure - if os.environ.get('SWIFT_WRITE_CONTAINER_UNSTRUCTURED'): + if os.environ.get("SWIFT_WRITE_CONTAINER_UNSTRUCTURED"): return - if environment_name == 'prod': + if environment_name == "prod": assert prod_container, ( "SWIFT_WRITE_CONTAINER should match '-odoo-prod', " "we got: '%s'" % (container_name,) @@ -101,9 +100,9 @@ class CloudPlatform(models.AbstractModel): "SWIFT_WRITE_CONTAINER should not match " "'-odoo-prod', we got: '%s'" % (container_name,) ) - elif environment_name == 'test': + elif environment_name == "test": # store in DB so we don't have files local to the host - assert params.get_param('ir_attachment.location') == 'db', ( + assert params.get_param("ir_attachment.location") == "db", ( "In test instances, files must be stored in the database with " "'ir_attachment.location' set to 'db'. This is " "automatically set by the function 'install()'." @@ -111,4 +110,4 @@ class CloudPlatform(models.AbstractModel): @api.model def install(self): - self._install('ovh') + self._install("ovh") diff --git a/logging_json/__init__.py b/logging_json/__init__.py index 1f1aa38..9f2ed66 100644 --- a/logging_json/__init__.py +++ b/logging_json/__init__.py @@ -1,2 +1 @@ - from . import json_log diff --git a/logging_json/__manifest__.py b/logging_json/__manifest__.py index b6db7d0..c456541 100644 --- a/logging_json/__manifest__.py +++ b/logging_json/__manifest__.py @@ -1,17 +1,15 @@ # Copyright 2016-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -{'name': 'JSON Logging', - 'version': "14.0.1.0.0", - 'author': 'Camptocamp,Odoo Community Association (OCA)', - 'license': 'AGPL-3', - 'category': 'Extra Tools', - 'depends': ['base', - ], - 'external_dependencies': { - 'python': ['python-json-logger'], - }, - 'website': 'http://www.camptocamp.com', - 'data': [], - 'installable': True, - } +{ + "name": "JSON Logging", + "version": "14.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Extra Tools", + "depends": ["base"], + "external_dependencies": {"python": ["python-json-logger"]}, + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "data": [], + "installable": True, +} diff --git a/logging_json/json_log.py b/logging_json/json_log.py index 8215df4..09d3cbc 100644 --- a/logging_json/json_log.py +++ b/logging_json/json_log.py @@ -5,7 +5,6 @@ import logging import os import threading import uuid - from distutils.util import strtobool from odoo import http @@ -20,23 +19,22 @@ except ImportError: def is_true(strval): - return bool(strtobool(strval or '0'.lower())) + return bool(strtobool(strval or "0".lower())) class OdooJsonFormatter(jsonlogger.JsonFormatter): - def add_fields(self, log_record, record, message_dict): record.pid = os.getpid() - record.dbname = getattr(threading.currentThread(), 'dbname', '?') - record.request_id = getattr(threading.current_thread(), 'request_uuid', None) - record.uid = getattr(threading.current_thread(), 'uid', None) + record.dbname = getattr(threading.currentThread(), "dbname", "?") + record.request_id = getattr(threading.current_thread(), "request_uuid", None) + record.uid = getattr(threading.current_thread(), "uid", None) _super = super(OdooJsonFormatter, self) return _super.add_fields(log_record, record, message_dict) -if is_true(os.environ.get('ODOO_LOGGING_JSON')): +if is_true(os.environ.get("ODOO_LOGGING_JSON")): formatted_message = ( - '%(asctime)s %(pid)s %(levelname)s %(dbname)s %(name)s: %(message)s' + "%(asctime)s %(pid)s %(levelname)s %(dbname)s %(name)s: %(message)s" ) formatter = OdooJsonFormatter(formatted_message) logging.getLogger().handlers[0].formatter = formatter diff --git a/monitoring_log_requests/__init__.py b/monitoring_log_requests/__init__.py index a9e3372..0650744 100644 --- a/monitoring_log_requests/__init__.py +++ b/monitoring_log_requests/__init__.py @@ -1,2 +1 @@ - from . import models diff --git a/monitoring_log_requests/__manifest__.py b/monitoring_log_requests/__manifest__.py index 9364c46..84a7636 100644 --- a/monitoring_log_requests/__manifest__.py +++ b/monitoring_log_requests/__manifest__.py @@ -1,14 +1,14 @@ # Copyright 2016-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - -{'name': 'Monitoring: Requests Logging', - 'version': "14.0.1.0.0", - 'author': 'Camptocamp,Numigi,Odoo Community Association (OCA)', - 'license': 'AGPL-3', - 'category': 'category', - 'depends': ['base', 'web'], - 'website': 'http://www.camptocamp.com', - 'data': [], - 'installable': True, - } +{ + "name": "Monitoring: Requests Logging", + "version": "14.0.1.0.0", + "author": "Camptocamp,Numigi,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "category", + "depends": ["base", "web"], + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "data": [], + "installable": True, +} diff --git a/monitoring_log_requests/models/__init__.py b/monitoring_log_requests/models/__init__.py index 0d643ba..9a5eb71 100644 --- a/monitoring_log_requests/models/__init__.py +++ b/monitoring_log_requests/models/__init__.py @@ -1,2 +1 @@ - from . import ir_http diff --git a/monitoring_log_requests/models/ir_http.py b/monitoring_log_requests/models/ir_http.py index abaf46b..505e71f 100644 --- a/monitoring_log_requests/models/ir_http.py +++ b/monitoring_log_requests/models/ir_http.py @@ -9,28 +9,28 @@ from odoo import models from odoo.http import request as http_request from odoo.tools.config import config - -_logger = logging.getLogger('monitoring.http.requests') +_logger = logging.getLogger("monitoring.http.requests") class IrHttp(models.AbstractModel): - _inherit = 'ir.http' + _inherit = "ir.http" @classmethod def _dispatch(cls): begin = time.time() response = super()._dispatch() end = time.time() - if (not cls._monitoring_blacklist(http_request) and - cls._monitoring_filter(http_request)): + if not cls._monitoring_blacklist(http_request) and cls._monitoring_filter( + http_request + ): info = cls._monitoring_info(http_request, response, begin, end) cls._monitoring_log(info) return response @classmethod def _monitoring_blacklist(cls, request): - path_info = request.httprequest.environ.get('PATH_INFO') - if path_info.startswith('/longpolling/'): + path_info = request.httprequest.environ.get("PATH_INFO") + if path_info.startswith("/longpolling/"): return True return False @@ -40,42 +40,45 @@ class IrHttp(models.AbstractModel): @classmethod def _monitoring_info(cls, request, response, begin, end): - path = request.httprequest.environ.get('PATH_INFO') + path = request.httprequest.environ.get("PATH_INFO") info = { # timing - 'start_time': time.strftime("%Y-%m-%d %H:%M:%S", - time.gmtime(begin)), - 'duration': end - begin, + "start_time": time.strftime("%Y-%m-%d %H:%M:%S", time.gmtime(begin)), + "duration": end - begin, # HTTP things - 'method': request.httprequest.method, - 'url': request.httprequest.url, - 'path': path, - 'content_type': request.httprequest.environ.get('CONTENT_TYPE'), - 'user_agent': request.httprequest.environ.get('HTTP_USER_AGENT'), + "method": request.httprequest.method, + "url": request.httprequest.url, + "path": path, + "content_type": request.httprequest.environ.get("CONTENT_TYPE"), + "user_agent": request.httprequest.environ.get("HTTP_USER_AGENT"), # Odoo things - 'db': None, - 'uid': request.uid, - 'login': None, - 'server_environment': config.get('running_env'), - 'model': None, - 'model_method': None, - 'workflow_signal': None, + "db": None, + "uid": request.uid, + "login": None, + "server_environment": config.get("running_env"), + "model": None, + "model_method": None, + "workflow_signal": None, # response things - 'response_status_code': None, + "response_status_code": None, } - if hasattr(request, 'status_code'): - info['status_code'] = response.status_code - if hasattr(request, 'session'): - info.update({ - 'login': request.session.get('login'), - 'db': request.session.get('db'), - }) - if hasattr(request, 'params'): - info.update({ - 'model': request.params.get('model'), - 'model_method': request.params.get('method'), - 'workflow_signal': request.params.get('signal'), - }) + if hasattr(request, "status_code"): + info["status_code"] = response.status_code + if hasattr(request, "session"): + info.update( + { + "login": request.session.get("login"), + "db": request.session.get("db"), + } + ) + if hasattr(request, "params"): + info.update( + { + "model": request.params.get("model"), + "model_method": request.params.get("method"), + "workflow_signal": request.params.get("signal"), + } + ) return info @classmethod diff --git a/monitoring_log_requests/utils.py b/monitoring_log_requests/utils.py new file mode 100644 index 0000000..98cc0e0 --- /dev/null +++ b/monitoring_log_requests/utils.py @@ -0,0 +1,10 @@ +# Copyright 2020 Camptocamp SA +# License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) + +from distutils.util import strtobool +from os import environ + + +def is_enabled(): + env_val = environ.get("ODOO_REQUESTS_LOGGING") + return bool(strtobool(env_val or "0".lower())) diff --git a/monitoring_prometheus/__manifest__.py b/monitoring_prometheus/__manifest__.py index f2d7fda..21eca90 100644 --- a/monitoring_prometheus/__manifest__.py +++ b/monitoring_prometheus/__manifest__.py @@ -8,15 +8,9 @@ "author": "Camptocamp,Odoo Community Association (OCA)", "license": "AGPL-3", "category": "category", - "depends": [ - "base", - "web", - "server_environment", - ], - "website": "http://www.camptocamp.com", + "depends": ["base", "web", "server_environment"], + "website": "https://github.com/camptocamp/odoo-cloud-platform", "data": [], - "external_dependencies": { - "python": ["prometheus_client"], - }, + "external_dependencies": {"python": ["prometheus_client"]}, "installable": True, } diff --git a/monitoring_prometheus/controllers/prometheus_metrics.py b/monitoring_prometheus/controllers/prometheus_metrics.py index 411a2ac..4aa4996 100644 --- a/monitoring_prometheus/controllers/prometheus_metrics.py +++ b/monitoring_prometheus/controllers/prometheus_metrics.py @@ -1,11 +1,12 @@ # Copyright 2016-2021 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -from odoo.http import Controller, route from prometheus_client import generate_latest +from odoo.http import Controller, route + class PrometheusController(Controller): - @route('/metrics', auth='public') + @route("/metrics", auth="public") def metrics(self): return generate_latest() diff --git a/monitoring_prometheus/models/ir_http.py b/monitoring_prometheus/models/ir_http.py index 0c026d6..26375af 100644 --- a/monitoring_prometheus/models/ir_http.py +++ b/monitoring_prometheus/models/ir_http.py @@ -1,10 +1,10 @@ # Copyright 2016-2021 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) +from prometheus_client import Counter, Summary + from odoo import models from odoo.http import request -from prometheus_client import Summary, Counter - REQUEST_TIME = Summary( "request_latency_sec", "Request response time in sec", ["query_type"] diff --git a/monitoring_statsd/__init__.py b/monitoring_statsd/__init__.py index a9e3372..0650744 100644 --- a/monitoring_statsd/__init__.py +++ b/monitoring_statsd/__init__.py @@ -1,2 +1 @@ - from . import models diff --git a/monitoring_statsd/__manifest__.py b/monitoring_statsd/__manifest__.py index 381aa8b..4665ec5 100644 --- a/monitoring_statsd/__manifest__.py +++ b/monitoring_statsd/__manifest__.py @@ -1,20 +1,15 @@ # Copyright 2016-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - -{'name': 'Monitoring: Statsd Metrics', - 'version': "14.0.1.0.0", - 'author': 'Camptocamp,Odoo Community Association (OCA)', - 'license': 'AGPL-3', - 'category': 'category', - 'depends': ['base', - 'web', - 'server_environment', - ], - 'website': 'http://www.camptocamp.com', - 'data': [], - 'external_dependencies': { - 'python': ['statsd'], - }, - 'installable': True, - } +{ + "name": "Monitoring: Statsd Metrics", + "version": "14.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "category", + "depends": ["base", "web", "server_environment"], + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "data": [], + "external_dependencies": {"python": ["statsd"]}, + "installable": True, +} diff --git a/monitoring_statsd/models/__init__.py b/monitoring_statsd/models/__init__.py index 0d643ba..9a5eb71 100644 --- a/monitoring_statsd/models/__init__.py +++ b/monitoring_statsd/models/__init__.py @@ -1,2 +1 @@ - from . import ir_http diff --git a/monitoring_statsd/models/ir_http.py b/monitoring_statsd/models/ir_http.py index ba4f538..e49120c 100644 --- a/monitoring_statsd/models/ir_http.py +++ b/monitoring_statsd/models/ir_http.py @@ -4,38 +4,46 @@ from odoo import models from odoo.http import request -from ..statsd_client import statsd, customer, environment +from ..statsd_client import customer, environment, statsd class IrHttp(models.AbstractModel): - _inherit = 'ir.http' + _inherit = "ir.http" @classmethod def _dispatch(cls): if not statsd: return super()._dispatch() - path_info = request.httprequest.environ.get('PATH_INFO') - if path_info.startswith('/longpolling/'): + path_info = request.httprequest.environ.get("PATH_INFO") + if path_info.startswith("/longpolling/"): return super()._dispatch() - parts = ['http', ] - if path_info.startswith('/web/dataset/call_button'): - parts += ['button', - customer, environment, - request.params['model'].replace('.', '_'), - request.params['method'], - ] - elif path_info.startswith('/web/dataset/exec_workflow'): - parts += ['workflow', - customer, environment, - request.params['model'].replace('.', '_'), - request.params['signal'], - ] + parts = [ + "http", + ] + if path_info.startswith("/web/dataset/call_button"): + parts += [ + "button", + customer, + environment, + request.params["model"].replace(".", "_"), + request.params["method"], + ] + elif path_info.startswith("/web/dataset/exec_workflow"): + parts += [ + "workflow", + customer, + environment, + request.params["model"].replace(".", "_"), + request.params["signal"], + ] else: - parts += ['request', - customer, environment, - ] + parts += [ + "request", + customer, + environment, + ] - with statsd.timer('.'.join(parts)): + with statsd.timer(".".join(parts)): return super()._dispatch() diff --git a/monitoring_statsd/statsd_client.py b/monitoring_statsd/statsd_client.py index 1ab1bba..9231fe9 100644 --- a/monitoring_statsd/statsd_client.py +++ b/monitoring_statsd/statsd_client.py @@ -3,7 +3,6 @@ import logging import os - from distutils.util import strtobool from odoo.tools.config import config @@ -14,40 +13,39 @@ try: from statsd import defaults from statsd.client import StatsClient except ImportError: - _logger.warning('statds must be installed') + _logger.warning("statds must be installed") defaults = None # noqa StatsClient = None # noqa def is_true(strval): - return bool(strtobool(strval or '0'.lower())) + return bool(strtobool(strval or "0".lower())) -statsd_active = is_true(os.environ.get('ODOO_STATSD')) +statsd_active = is_true(os.environ.get("ODOO_STATSD")) statsd = None customer = None environment = None if statsd_active and statsd is None and StatsClient is not None: - if not os.environ.get('STATSD_CUSTOMER'): - raise Exception( - 'STATSD_CUSTOMER must contain the name of the customer' - ) - customer = os.environ.get('STATSD_CUSTOMER') - if os.environ.get('STATSD_ENVIRONMENT'): - environment = os.environ['STATSD_ENVIRONMENT'] - elif config.get('running_env'): - environment = config['running_env'] + if not os.environ.get("STATSD_CUSTOMER"): + raise Exception("STATSD_CUSTOMER must contain the name of the customer") + customer = os.environ.get("STATSD_CUSTOMER") + if os.environ.get("STATSD_ENVIRONMENT"): + environment = os.environ["STATSD_ENVIRONMENT"] + elif config.get("running_env"): + environment = config["running_env"] else: raise Exception( - 'Either STATSD_ENVIRONMENT or configuration option running_env ' - 'must contain the environment (prod, integration, ...)' + "Either STATSD_ENVIRONMENT or configuration option running_env " + "must contain the environment (prod, integration, ...)" ) - host = os.getenv('STATSD_HOST', defaults.HOST) - port = int(os.getenv('STATSD_PORT', defaults.PORT)) - prefix = os.getenv('STATSD_PREFIX', defaults.PREFIX) - maxudpsize = int(os.getenv('STATSD_MAXUDPSIZE', defaults.MAXUDPSIZE)) - ipv6 = bool(int(os.getenv('STATSD_IPV6', defaults.IPV6))) - statsd = StatsClient(host=host, port=port, prefix='odoo', - maxudpsize=maxudpsize, ipv6=ipv6) + host = os.getenv("STATSD_HOST", defaults.HOST) + port = int(os.getenv("STATSD_PORT", defaults.PORT)) + prefix = os.getenv("STATSD_PREFIX", defaults.PREFIX) + maxudpsize = int(os.getenv("STATSD_MAXUDPSIZE", defaults.MAXUDPSIZE)) + ipv6 = bool(int(os.getenv("STATSD_IPV6", defaults.IPV6))) + statsd = StatsClient( + host=host, port=port, prefix="odoo", maxudpsize=maxudpsize, ipv6=ipv6 + ) diff --git a/monitoring_status/__manifest__.py b/monitoring_status/__manifest__.py index 91ea646..540f481 100644 --- a/monitoring_status/__manifest__.py +++ b/monitoring_status/__manifest__.py @@ -1,14 +1,14 @@ # Copyright 2016-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - -{'name': 'Monitoring: Status', - 'version': "14.0.1.0.0", - 'author': 'Camptocamp,Odoo Community Association (OCA)', - 'license': 'AGPL-3', - 'category': 'category', - 'depends': ['base', 'web'], - 'website': 'http://www.camptocamp.com', - 'data': [], - 'installable': True, - } +{ + "name": "Monitoring: Status", + "version": "14.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "category", + "depends": ["base", "web"], + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "data": [], + "installable": True, +} diff --git a/monitoring_status/controllers/main.py b/monitoring_status/controllers/main.py index c2463eb..21b6893 100644 --- a/monitoring_status/controllers/main.py +++ b/monitoring_status/controllers/main.py @@ -1,18 +1,18 @@ # Copyright 2016-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) -import logging import json +import logging import werkzeug from odoo import http + from odoo.addons.web.controllers.main import ensure_db class HealthCheckFilter(logging.Filter): - - def __init__(self, path, name=''): + def __init__(self, path, name=""): super().__init__(name) self.path = path @@ -20,20 +20,19 @@ class HealthCheckFilter(logging.Filter): return self.path not in record.getMessage() -logging.getLogger('werkzeug').addFilter( - HealthCheckFilter('GET /monitoring/status HTTP') +logging.getLogger("werkzeug").addFilter( + HealthCheckFilter("GET /monitoring/status HTTP") ) class Monitoring(http.Controller): - - @http.route('/monitoring/status', type='http', auth='none') + @http.route("/monitoring/status", type="http", auth="none") def status(self): ensure_db() # TODO: add 'sub-systems' status and infos: # queue job, cron, database, ... - headers = {'Content-Type': 'application/json'} - info = {'status': 1} + headers = {"Content-Type": "application/json"} + info = {"status": 1} session = http.request.session # We set a custom expiration of 1 second for this request, as we do a # lot of health checks, we don't want those anonymous sessions to be diff --git a/session_redis/__init__.py b/session_redis/__init__.py index 8f2ee9e..a64bcb4 100644 --- a/session_redis/__init__.py +++ b/session_redis/__init__.py @@ -1,3 +1,2 @@ - from . import http from . import session diff --git a/session_redis/__manifest__.py b/session_redis/__manifest__.py index 5b7015b..4672c48 100644 --- a/session_redis/__manifest__.py +++ b/session_redis/__manifest__.py @@ -1,18 +1,16 @@ # Copyright 2016-2019 Camptocamp SA # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) - -{'name': 'Sessions in Redis', - 'summary': 'Store web sessions in Redis', - 'version': "14.0.1.0.0", - 'author': 'Camptocamp,Odoo Community Association (OCA)', - 'license': 'AGPL-3', - 'category': 'Extra Tools', - 'depends': ['base'], - 'external_dependencies': { - 'python': ['redis'], - }, - 'website': 'http://www.camptocamp.com', - 'data': [], - 'installable': True, - } +{ + "name": "Sessions in Redis", + "summary": "Store web sessions in Redis", + "version": "14.0.1.0.0", + "author": "Camptocamp,Odoo Community Association (OCA)", + "license": "AGPL-3", + "category": "Extra Tools", + "depends": ["base"], + "external_dependencies": {"python": ["redis"]}, + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "data": [], + "installable": True, +} diff --git a/session_redis/http.py b/session_redis/http.py index 48902af..76e5ed2 100644 --- a/session_redis/http.py +++ b/session_redis/http.py @@ -3,7 +3,6 @@ import logging import os - from distutils.util import strtobool import odoo @@ -23,46 +22,46 @@ except ImportError: def is_true(strval): - return bool(strtobool(strval or '0'.lower())) + return bool(strtobool(strval or "0".lower())) -sentinel_host = os.environ.get('ODOO_SESSION_REDIS_SENTINEL_HOST') -sentinel_master_name = os.environ.get( - 'ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME' -) +sentinel_host = os.environ.get("ODOO_SESSION_REDIS_SENTINEL_HOST") +sentinel_master_name = os.environ.get("ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME") if sentinel_host and not sentinel_master_name: raise Exception( "ODOO_SESSION_REDIS_SENTINEL_MASTER_NAME must be defined " "when using session_redis" ) -sentinel_port = int(os.environ.get('ODOO_SESSION_REDIS_SENTINEL_PORT', 26379)) -host = os.environ.get('ODOO_SESSION_REDIS_HOST', 'localhost') -port = int(os.environ.get('ODOO_SESSION_REDIS_PORT', 6379)) -prefix = os.environ.get('ODOO_SESSION_REDIS_PREFIX') -url = os.environ.get('ODOO_SESSION_REDIS_URL') -password = os.environ.get('ODOO_SESSION_REDIS_PASSWORD') -expiration = os.environ.get('ODOO_SESSION_REDIS_EXPIRATION') -anon_expiration = os.environ.get('ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS') +sentinel_port = int(os.environ.get("ODOO_SESSION_REDIS_SENTINEL_PORT", 26379)) +host = os.environ.get("ODOO_SESSION_REDIS_HOST", "localhost") +port = int(os.environ.get("ODOO_SESSION_REDIS_PORT", 6379)) +prefix = os.environ.get("ODOO_SESSION_REDIS_PREFIX") +url = os.environ.get("ODOO_SESSION_REDIS_URL") +password = os.environ.get("ODOO_SESSION_REDIS_PASSWORD") +expiration = os.environ.get("ODOO_SESSION_REDIS_EXPIRATION") +anon_expiration = os.environ.get("ODOO_SESSION_REDIS_EXPIRATION_ANONYMOUS") @lazy_property def session_store(self): if sentinel_host: - sentinel = Sentinel([(sentinel_host, sentinel_port)], - password=password) + sentinel = Sentinel([(sentinel_host, sentinel_port)], password=password) redis_client = sentinel.master_for(sentinel_master_name) elif url: redis_client = redis.from_url(url) else: redis_client = redis.Redis(host=host, port=port, password=password) - return RedisSessionStore(redis=redis_client, prefix=prefix, - expiration=expiration, - anon_expiration=anon_expiration, - session_class=http.OpenERPSession) + return RedisSessionStore( + redis=redis_client, + prefix=prefix, + expiration=expiration, + anon_expiration=anon_expiration, + session_class=http.OpenERPSession, + ) def session_gc(session_store): - """ Do not garbage collect the sessions + """Do not garbage collect the sessions Redis keys are automatically cleaned at the end of their expiration. @@ -79,14 +78,22 @@ def purge_fs_sessions(path): pass -if is_true(os.environ.get('ODOO_SESSION_REDIS')): +if is_true(os.environ.get("ODOO_SESSION_REDIS")): if sentinel_host: - _logger.debug("HTTP sessions stored in Redis with prefix '%s'. " - "Using Sentinel on %s:%s", - prefix or '', sentinel_host, sentinel_port) + _logger.debug( + "HTTP sessions stored in Redis with prefix '%s'. " + "Using Sentinel on %s:%s", + prefix or "", + sentinel_host, + sentinel_port, + ) else: - _logger.debug("HTTP sessions stored in Redis with prefix '%s' on " - "%s:%s", prefix or '', host, port) + _logger.debug( + "HTTP sessions stored in Redis with prefix '%s' on " "%s:%s", + prefix or "", + host, + port, + ) http.Root.session_store = session_store http.session_gc = session_gc diff --git a/session_redis/json_encoding.py b/session_redis/json_encoding.py index f535a8f..bb1c57a 100644 --- a/session_redis/json_encoding.py +++ b/session_redis/json_encoding.py @@ -2,7 +2,6 @@ # License AGPL-3.0 or later (http://www.gnu.org/licenses/agpl.html) import json - from datetime import date, datetime import dateutil diff --git a/session_redis/session.py b/session_redis/session.py index 4c8457e..dce2774 100644 --- a/session_redis/session.py +++ b/session_redis/session.py @@ -17,10 +17,16 @@ _logger = logging.getLogger(__name__) class RedisSessionStore(SessionStore): - """ SessionStore that saves session to redis """ + """SessionStore that saves session to redis""" - def __init__(self, redis, session_class=None, - prefix='', expiration=None, anon_expiration=None): + def __init__( + self, + redis, + session_class=None, + prefix="", + expiration=None, + anon_expiration=None, + ): super().__init__(session_class=session_class) self.redis = redis if expiration is None: @@ -31,14 +37,12 @@ class RedisSessionStore(SessionStore): self.anon_expiration = DEFAULT_SESSION_TIMEOUT_ANONYMOUS else: self.anon_expiration = anon_expiration - self.prefix = 'session:' + self.prefix = "session:" if prefix: - self.prefix = '%s:%s:' % ( - self.prefix, prefix - ) + self.prefix = "%s:%s:" % (self.prefix, prefix) def build_key(self, sid): - return '%s%s' % (self.prefix, sid) + return "%s%s" % (self.prefix, sid) def save(self, session): key = self.build_key(session.sid) @@ -51,48 +55,56 @@ class RedisSessionStore(SessionStore): expiration = session.expiration or self.anon_expiration if _logger.isEnabledFor(logging.DEBUG): if session.uid: - user_msg = "user '%s' (id: %s)" % ( - session.login, session.uid) + user_msg = "user '%s' (id: %s)" % (session.login, session.uid) else: user_msg = "anonymous user" - _logger.debug("saving session with key '%s' and " - "expiration of %s seconds for %s", - key, expiration, user_msg) + _logger.debug( + "saving session with key '%s' and " "expiration of %s seconds for %s", + key, + expiration, + user_msg, + ) - data = json.dumps( - dict(session), cls=json_encoding.SessionEncoder - ).encode('utf-8') + data = json.dumps(dict(session), cls=json_encoding.SessionEncoder).encode( + "utf-8" + ) if self.redis.set(key, data): return self.redis.expire(key, expiration) def delete(self, session): key = self.build_key(session.sid) - _logger.debug('deleting session with key %s', key) + _logger.debug("deleting session with key %s", key) return self.redis.delete(key) def get(self, sid): if not self.is_valid_key(sid): - _logger.debug("session with invalid sid '%s' has been asked, " - "returning a new one", sid) + _logger.debug( + "session with invalid sid '%s' has been asked, " "returning a new one", + sid, + ) return self.new() key = self.build_key(sid) saved = self.redis.get(key) if not saved: - _logger.debug("session with non-existent key '%s' has been asked, " - "returning a new one", key) + _logger.debug( + "session with non-existent key '%s' has been asked, " + "returning a new one", + key, + ) return self.new() try: - data = json.loads( - saved.decode('utf-8'), cls=json_encoding.SessionDecoder - ) + data = json.loads(saved.decode("utf-8"), cls=json_encoding.SessionDecoder) except ValueError: - _logger.debug("session for key '%s' has been asked but its json " - "content could not be read, it has been reset", key) + _logger.debug( + "session for key '%s' has been asked but its json " + "content could not be read, it has been reset", + key, + ) data = {} return self.session_class(data, sid, False) def list(self): - keys = self.redis.keys('%s*' % self.prefix) + keys = self.redis.keys("%s*" % self.prefix) _logger.debug("a listing redis keys has been called") - return [key[len(self.prefix):] for key in keys] + return [key[len(self.prefix) :] for key in keys] diff --git a/setup/test_base_fileurl_field/odoo/addons/test_base_fileurl_field b/setup/test_base_fileurl_field/odoo/addons/test_base_fileurl_field new file mode 120000 index 0000000..54e790e --- /dev/null +++ b/setup/test_base_fileurl_field/odoo/addons/test_base_fileurl_field @@ -0,0 +1 @@ +../../../../test_base_fileurl_field \ No newline at end of file diff --git a/setup/test_base_fileurl_field/setup.py b/setup/test_base_fileurl_field/setup.py new file mode 100644 index 0000000..28c57bb --- /dev/null +++ b/setup/test_base_fileurl_field/setup.py @@ -0,0 +1,6 @@ +import setuptools + +setuptools.setup( + setup_requires=['setuptools-odoo'], + odoo_addon=True, +) diff --git a/test_base_fileurl_field/__manifest__.py b/test_base_fileurl_field/__manifest__.py index a2934b7..d5be248 100644 --- a/test_base_fileurl_field/__manifest__.py +++ b/test_base_fileurl_field/__manifest__.py @@ -1,19 +1,15 @@ # Copyright 2019 Camptocamp SA # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl) { - 'name': 'test base fileurl fields', - 'summary': """A module to verify fileurl field.""", - 'version': '12.0.1.0.0', - 'category': 'Tests', - 'author': 'Camptocamp,Odoo Community Association (OCA)', - 'license': 'AGPL-3', - 'depends': [ - 'base_fileurl_field' - ], - 'data': [ - "views/res_partner.xml", - "views/res_users.xml", - ], - 'installable': False, - 'auto_install': False, + "name": "test base fileurl fields", + "summary": """A module to verify fileurl field.""", + "version": "14.0.1.0.0", + "category": "Tests", + "author": "Camptocamp,Odoo Community Association (OCA)", + "website": "https://github.com/camptocamp/odoo-cloud-platform", + "license": "AGPL-3", + "depends": ["base_fileurl_field"], + "data": ["views/res_partner.xml", "views/res_users.xml"], + "installable": True, + "auto_install": False, } diff --git a/test_base_fileurl_field/data/sample.txt b/test_base_fileurl_field/data/sample.txt index 5251e0f..8a03e0e 100644 --- a/test_base_fileurl_field/data/sample.txt +++ b/test_base_fileurl_field/data/sample.txt @@ -1 +1 @@ -This is a simple text file. \ No newline at end of file +This is a simple text file. diff --git a/test_base_fileurl_field/models/res_partner.py b/test_base_fileurl_field/models/res_partner.py index 359843c..48ca0a4 100644 --- a/test_base_fileurl_field/models/res_partner.py +++ b/test_base_fileurl_field/models/res_partner.py @@ -1,44 +1,46 @@ # Copyright 2019 Camptocamp SA # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl) -from odoo import models, fields, api, _ +from odoo import _, api, fields, models from odoo.exceptions import ValidationError class ResPartner(models.Model): - _inherit = 'res.partner' + _inherit = "res.partner" name = fields.Char() url_file = fields.FileURL( - storage_location='s3', - filename='url_file_fname', - storage_path='partner' + storage_location="s3", filename="url_file_fname", storage_path="partner" ) url_file_fname = fields.Char() url_image = fields.FileURL( - storage_location='s3', - filename='url_image_fname', - storage_path='partner_image', + storage_location="s3", + filename="url_image_fname", + storage_path="partner_image", ) url_image_fname = fields.Char() - @api.constrains('url_file', 'url_file_fname') + @api.constrains("url_file", "url_file_fname") def _check_url_file_fname(self): - rec = self.search([('url_file_fname', '=', self.url_file_fname)]) + rec = self.search([("url_file_fname", "=", self.url_file_fname)]) if len(rec) > 1: - raise ValidationError(_( - "This file name is already used on an existing record. " - "Please use another file name or delete the url_file on :\n" - "Model: %s Id: %s" % (self._name, rec.id) - )) + raise ValidationError( + _( + "This file name is already used on an existing record. " + "Please use another file name or delete the url_file on :\n" + "Model: %s Id: %s" % (self._name, rec.id) + ) + ) - @api.constrains('url_image', 'url_image_fname') + @api.constrains("url_image", "url_image_fname") def _check_url_image_fname(self): - rec = self.search([('url_image_fname', '=', self.url_image_fname)]) + rec = self.search([("url_image_fname", "=", self.url_image_fname)]) if len(rec) > 1: - raise ValidationError(_( - "This file name is already used on an existing record. " - "Please use another file name or delete the url_image on :\n" - "Model: %s Id: %s" % (self._name, rec.id) - )) + raise ValidationError( + _( + "This file name is already used on an existing record. " + "Please use another file name or delete the url_image on :\n" + "Model: %s Id: %s" % (self._name, rec.id) + ) + ) diff --git a/test_base_fileurl_field/models/res_users.py b/test_base_fileurl_field/models/res_users.py index da0b80b..c8bc324 100644 --- a/test_base_fileurl_field/models/res_users.py +++ b/test_base_fileurl_field/models/res_users.py @@ -1,11 +1,11 @@ # Copyright 2019 Camptocamp SA # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl) -from odoo import models, fields +from odoo import fields, models class ResUsers(models.Model): - _inherit = 'res.users' + _inherit = "res.users" - partner_url_file = fields.FileURL(related='partner_id.url_file') - partner_url_file_fname = fields.Char(related='partner_id.url_file_fname') + partner_url_file = fields.FileURL(related="partner_id.url_file") + partner_url_file_fname = fields.Char(related="partner_id.url_file_fname") diff --git a/test_base_fileurl_field/tests/ir_attachment.py b/test_base_fileurl_field/tests/ir_attachment.py index d07017d..caa179f 100644 --- a/test_base_fileurl_field/tests/ir_attachment.py +++ b/test_base_fileurl_field/tests/ir_attachment.py @@ -3,7 +3,7 @@ import logging -from odoo import models, api +from odoo import api, models _logger = logging.getLogger(__name__) @@ -14,23 +14,23 @@ class IrAttachment(models.Model): _inherit = "ir.attachment" def _get_stores(self): - l = ['s3'] - l += super(IrAttachment, self)._get_stores() - return l + stores = ["s3"] + stores += super()._get_stores() + return stores @api.model def _store_file_read(self, fname, bin_size=False): - if fname.startswith('s3://'): + if fname.startswith("s3://"): return FAKE_S3_BUCKET.get(fname) else: return super(IrAttachment, self)._store_file_read(fname, bin_size) @api.model def _store_file_write(self, key, bin_data): - location = self.env.context.get('storage_location') or self._storage() - if location == 's3': + location = self.env.context.get("storage_location") or self._storage() + if location == "s3": FAKE_S3_BUCKET[key] = bin_data - filename = 's3://fake_bucket/%s' % key + filename = "s3://fake_bucket/%s" % key else: _super = super(IrAttachment, self) filename = _super._store_file_write(key, bin_data) @@ -38,7 +38,7 @@ class IrAttachment(models.Model): @api.model def _store_file_delete(self, fname): - if fname.startswith('s3://'): + if fname.startswith("s3://"): FAKE_S3_BUCKET.pop(fname) else: super(IrAttachment, self)._store_file_delete(fname) diff --git a/test_base_fileurl_field/tests/test_fileurl_fields.py b/test_base_fileurl_field/tests/test_fileurl_fields.py index c56bbe1..8be2bec 100644 --- a/test_base_fileurl_field/tests/test_fileurl_fields.py +++ b/test_base_fileurl_field/tests/test_fileurl_fields.py @@ -2,38 +2,41 @@ # License AGPL-3.0 or later (https://www.gnu.org/licenses/agpl) import base64 -from odoo.tests import TransactionCase -from odoo.modules.module import get_module_resource from odoo.exceptions import ValidationError +from odoo.modules.module import get_module_resource +from odoo.tests import TransactionCase class TestFileUrlFields(TransactionCase): - def test_fileurl_fields(self): - file_path = get_module_resource('test_base_fileurl_field', 'data', - 'sample.txt') - image_path = get_module_resource('test_base_fileurl_field', 'data', - 'pattern.png') - partner = self.env.ref('base.main_partner') - with open(file_path, 'rb') as f: - with open(image_path, 'rb') as i: - partner.write({ - 'url_file': base64.b64encode(f.read()), - 'url_file_fname': 'sample.txt', - 'url_image': base64.b64encode(i.read()), - 'url_image_fname': 'pattern.png', - }) + file_path = get_module_resource("test_base_fileurl_field", "data", "sample.txt") + image_path = get_module_resource( + "test_base_fileurl_field", "data", "pattern.png" + ) + partner = self.env.ref("base.main_partner") + with open(file_path, "rb") as f: + with open(image_path, "rb") as i: + partner.write( + { + "url_file": base64.b64encode(f.read()), + "url_file_fname": "sample.txt", + "url_image": base64.b64encode(i.read()), + "url_image_fname": "pattern.png", + } + ) - with open(file_path, 'rb') as f: + with open(file_path, "rb") as f: self.assertEqual(base64.decodebytes(partner.url_file), f.read()) - with open(image_path, 'rb') as i: + with open(image_path, "rb") as i: self.assertEqual(base64.decodebytes(partner.url_image), i.read()) - partner2 = self.env.ref('base.partner_admin') - with open(file_path, 'rb') as f: + partner2 = self.env.ref("base.partner_admin") + with open(file_path, "rb") as f: with self.assertRaises(ValidationError): - partner2.write({ - 'url_file': base64.b64encode(f.read()), - 'url_file_fname': 'sample.txt', - }) + partner2.write( + { + "url_file": base64.b64encode(f.read()), + "url_file_fname": "sample.txt", + } + ) diff --git a/test_base_fileurl_field/views/res_partner.xml b/test_base_fileurl_field/views/res_partner.xml index 08c33e4..f3367e9 100644 --- a/test_base_fileurl_field/views/res_partner.xml +++ b/test_base_fileurl_field/views/res_partner.xml @@ -1,4 +1,4 @@ - + res.partner.form.inherit @@ -9,11 +9,15 @@ - + - - + + diff --git a/test_base_fileurl_field/views/res_users.xml b/test_base_fileurl_field/views/res_users.xml index ffb5675..cbfdc72 100644 --- a/test_base_fileurl_field/views/res_users.xml +++ b/test_base_fileurl_field/views/res_users.xml @@ -1,4 +1,4 @@ - + res.users.form.inherit @@ -9,7 +9,7 @@ - +