From 61bdf7d0e22d10a1469ae834d876335daa7886f8 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 9 Nov 2023 08:50:45 +0100 Subject: [PATCH 1/7] Add ruff, remove flake8 and isort Also, add check-toml. Remove unnecessary config files and gather others in pyproject.toml. --- .pre-commit-config.yaml | 63 ++++- .pylintrc | 508 ---------------------------------------- pyproject.toml | 25 ++ pytest.ini | 6 - setup.cfg | 8 - 5 files changed, 76 insertions(+), 534 deletions(-) delete mode 100644 .pylintrc create mode 100644 pyproject.toml delete mode 100644 pytest.ini delete mode 100644 setup.cfg diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3ae7b398..265d25aa 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -7,6 +7,7 @@ repos: args: [--markdown-linebreak-ext=md] - id: check-yaml - id: check-json + - id: check-toml - id: end-of-file-fixer - repo: https://github.com/jumanjihouse/pre-commit-hook-yamlfmt @@ -23,7 +24,56 @@ repos: rev: 23.11.0 hooks: - id: black - name: Blacken + +# ruff is a Python linter, incl. import sorter and formatter +# It works partly on files in-place +# More information can be found in its documentation: +# https://docs.astral.sh/ruff/ +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.5 + hooks: + - id: ruff + name: ruff core code base + exclude: ^(.*\.py|(\.github|tests)/.*)$ + # Fix what can be fixed in-place and exit with non-zero status if files were + # changed and/or there are rules violations. + args: + - --fix + - --exit-non-zero-on-fix + - --show-fixes + - --no-unsafe-fixes + # Extend rule set to include: + # flake8-bandit + - --extend-select=S + # flake8-blind-except + - --extend-select=BLE + # pylint + - --extend-select=PL + # Self assignment of variable + - --extend-ignore=PLW0127 + # too-many-* rules + # Ignore these, as they are not relevant for our code base + # We will, e.g., extend the recommended number of branches, statements, function + # args, etc. + - --extend-ignore=PLR09 + +# ruff is a Python linter, incl. import sorter and formatter +# It works partly on files in-place +# More information can be found in its documentation: +# https://docs.astral.sh/ruff/ +- repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.5 + hooks: + - id: ruff + name: ruff non-core code base + exclude: ^aiida_optimade/.*$ + # Fix what can be fixed in-place and exit with non-zero status if files were + # changed and/or there are rules violations. + args: + - --fix + - --exit-non-zero-on-fix + - --show-fixes + - --no-unsafe-fixes - repo: local hooks: @@ -36,14 +86,3 @@ repos: .codecov.yml )$ language: system - -- repo: https://github.com/pycqa/flake8 - rev: 6.1.0 - hooks: - - id: flake8 - -- repo: https://github.com/timothycrosley/isort - rev: 5.12.0 - hooks: - - id: isort - args: [--profile, black, --filter-files, --skip-gitignore] diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index 68a49ef4..00000000 --- a/.pylintrc +++ /dev/null @@ -1,508 +0,0 @@ -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code. -extension-pkg-whitelist= - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the blacklist. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the -# number of processors available to use. -jobs=1 - -# Control the amount of potential inferred values when inferring a single -# object. This can help the performance when dealing with large functions or -# complex, nested conditions. -limit-inference-results=100 - -# List of plugins (as comma separated values of python module names) to load, -# usually to register additional checkers. -load-plugins= - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# When enabled, pylint would attempt to guess common misconfiguration and emit -# user-friendly hints instead of false-positive error messages. -suggestion-mode=yes - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED. -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once). You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use "--disable=all --enable=classes -# --disable=W". -disable=import-outside-toplevel, - missing-module-docstring, - locally-disabled, - bad-continuation, - fixme, - too-many-instance-attributes - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable=c-extension-no-member - - -[REPORTS] - -# Python expression which should return a score less than or equal to 10. You -# have access to the variables 'error', 'warning', 'refactor', and 'convention' -# which contain the number of messages in each category, as well as 'statement' -# which is the total number of statements analyzed. This score is used by the -# global evaluation report (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details. -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio). You can also give a reporter class, e.g. -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages. -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - -# Complete name of functions that never returns. When checking for -# inconsistent-return-statements if a never returning function is called then -# it will be considered as an explicit return statement and no message will be -# printed. -never-returning-functions=sys.exit - - -[LOGGING] - -# Format style used to check logging format string. `old` means using % -# formatting, `new` is for `{}` formatting,and `fstr` is for f-strings. -logging-format-style=old - -# Logging modules to check that the string format arguments are in logging -# function parameter format. -logging-modules=logging - - -[SPELLING] - -# Limits count of emitted suggestions for spelling mistakes. -max-spelling-suggestions=4 - -# Spelling dictionary name. Available dictionaries: none. To make it work, -# install the python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains the private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to the private dictionary (see the -# --spelling-private-dict-file option) instead of raising a message. -spelling-store-unknown-words=no - - -[BASIC] - -# Naming style matching correct argument names. -argument-naming-style=snake_case - -# Regular expression matching correct argument names. Overrides argument- -# naming-style. -#argument-rgx= - -# Naming style matching correct attribute names. -attr-naming-style=snake_case - -# Regular expression matching correct attribute names. Overrides attr-naming- -# style. -#attr-rgx= - -# Bad variable names which should always be refused, separated by a comma. -bad-names=foo, - bar, - baz, - toto, - tutu, - tata - -# Naming style matching correct class attribute names. -class-attribute-naming-style=any - -# Regular expression matching correct class attribute names. Overrides class- -# attribute-naming-style. -#class-attribute-rgx= - -# Naming style matching correct class names. -class-naming-style=PascalCase - -# Regular expression matching correct class names. Overrides class-naming- -# style. -#class-rgx= - -# Naming style matching correct constant names. -const-naming-style=UPPER_CASE - -# Regular expression matching correct constant names. Overrides const-naming- -# style. -#const-rgx= - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Naming style matching correct function names. -function-naming-style=snake_case - -# Regular expression matching correct function names. Overrides function- -# naming-style. -#function-rgx= - -# Good variable names which should always be accepted, separated by a comma. -good-names=i, - j, - k, - ex, - Run, - _ - -# Include a hint for the correct naming format with invalid-name. -include-naming-hint=no - -# Naming style matching correct inline iteration names. -inlinevar-naming-style=any - -# Regular expression matching correct inline iteration names. Overrides -# inlinevar-naming-style. -#inlinevar-rgx= - -# Naming style matching correct method names. -method-naming-style=snake_case - -# Regular expression matching correct method names. Overrides method-naming- -# style. -#method-rgx= - -# Naming style matching correct module names. -module-naming-style=snake_case - -# Regular expression matching correct module names. Overrides module-naming- -# style. -#module-rgx= - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -# These decorators are taken in consideration only for invalid-name. -property-classes=abc.abstractproperty - -# Naming style matching correct variable names. -variable-naming-style=snake_case - -# Regular expression matching correct variable names. Overrides variable- -# naming-style. -#variable-rgx= - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - -# Minimum lines number of a similarity. -min-similarity-lines=4 - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -expected-line-ending-format= - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=88 - -# Maximum number of lines in a module. -max-module-lines=1000 - -# List of optional constructs for which whitespace checking is disabled. `dict- -# separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. -# `trailing-comma` allows a space between comma and closing bracket: (a, ). -# `empty-line` allows space-only lines. -no-space-check=trailing-comma, - dict-separator - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME, - XXX, - TODO - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# Tells whether to warn about missing members when the owner of the attribute -# is inferred to be None. -ignore-none=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis). It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules= - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - -# List of decorators that change the signature of a decorated function. -signature-mutators= - - -[STRING] - -# This flag controls whether the implicit-str-concat-in-sequence should -# generate a warning on implicit string concatenation in sequences defined over -# several lines. -check-str-concat-over-line-jumps=no - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid defining new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_, - _cb - -# A regular expression matching the name of dummy variables (i.e. expected to -# not be used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore. -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io - - -[IMPORTS] - -# List of modules that can be imported at any level, not just the top level -# one. -allow-any-import-level= - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma. -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled). -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled). -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled). -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - -# Couples of modules and preferred modules, separated by a comma. -preferred-modules= - - -[DESIGN] - -# Maximum number of arguments for function / method. -max-args=5 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Maximum number of boolean expressions in an if statement (see R0916). -max-bool-expr=5 - -# Maximum number of branch for function / method body. -max-branches=12 - -# Maximum number of locals for function / method body. -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body. -max-returns=6 - -# Maximum number of statements in function / method body. -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__, - __new__, - setUp, - __post_init__ - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict, - _fields, - _replace, - _source, - _make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=cls - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "BaseException, Exception". -overgeneral-exceptions=BaseException, - Exception diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..0ed9d0f8 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,25 @@ +[tool.ruff.lint] +select = [ + # pycodestyle + "E", + # Pyflakes + "F", + # pyupgrade + "UP", + # flake8-bugbear + "B", + # flake8-simplify + "SIM", + # isort + "I", + # ruff + "RUF", +] + +[tool.pytest.ini_options] +filterwarnings = [ + "ignore:.*PY_SSIZE_T_CLEAN will be required for '#' formats.*:DeprecationWarning", + 'ignore:.*"@coroutine" decorator is deprecated since Python 3.8, use "async def" instead.*:DeprecationWarning', + "ignore:.*Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated.*:DeprecationWarning", + "ignore:Parsing optional attribute.*:UserWarning", +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 316b4b49..00000000 --- a/pytest.ini +++ /dev/null @@ -1,6 +0,0 @@ -[pytest] -filterwarnings = - ignore:.*PY_SSIZE_T_CLEAN will be required for '#' formats.*:DeprecationWarning - ignore:.*"@coroutine" decorator is deprecated since Python 3.8, use "async def" instead.*:DeprecationWarning - ignore:.*Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated.*:DeprecationWarning - ignore:Parsing optional attribute.*:UserWarning diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 2f584c74..00000000 --- a/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[flake8] -ignore = - # Line to long. Handled by black. - E501 - # Line break before binary operator. This is preferred formatting for black. - W503 - # Whitespace before ':' - E203 From dbc0e02da359a580abe9c18ae33456d8cf303d91 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 9 Nov 2023 08:58:28 +0100 Subject: [PATCH 2/7] Remove everything pylint --- .github/mongo/load_data.py | 2 +- aiida_optimade/cli/cmd_calc.py | 3 +-- aiida_optimade/cli/cmd_init.py | 3 +-- aiida_optimade/cli/cmd_run.py | 1 - aiida_optimade/common/__init__.py | 9 ++++----- aiida_optimade/entry_collections.py | 4 ++-- aiida_optimade/mappers/__init__.py | 7 +++---- aiida_optimade/mappers/entries.py | 1 - aiida_optimade/models/structures.py | 1 - aiida_optimade/routers/info.py | 1 - aiida_optimade/routers/links.py | 1 - aiida_optimade/routers/structures.py | 1 - aiida_optimade/transformers/__init__.py | 5 ++--- aiida_optimade/transformers/aiida.py | 1 - aiida_optimade/translators/__init__.py | 17 ++++++++--------- aiida_optimade/translators/entities.py | 2 +- aiida_optimade/translators/structures.py | 1 - requirements_dev.txt | 1 - setup.py | 2 +- tasks.py | 6 +++++- tests/cli/conftest.py | 1 - tests/cli/test_calc.py | 1 - tests/cli/test_init.py | 1 - tests/cli/test_run.py | 1 - tests/conftest.py | 1 - tests/server/conftest.py | 1 - tests/server/query_params/test_filter.py | 1 - tests/server/query_params/test_page_limit.py | 1 - tests/server/query_params/test_page_offset.py | 1 - tests/server/query_params/test_sort.py | 1 - tests/server/test_entry_collections.py | 1 - tests/server/test_middleware.py | 1 - tests/server/utils.py | 5 ++--- tests/transformers/test_aiida.py | 1 - 34 files changed, 31 insertions(+), 56 deletions(-) diff --git a/.github/mongo/load_data.py b/.github/mongo/load_data.py index 05818c73..8e86d270 100755 --- a/.github/mongo/load_data.py +++ b/.github/mongo/load_data.py @@ -14,7 +14,7 @@ try: print(f"Inserting {len(data)} structures into {collection.full_name}") collection.insert_many(data, ordered=False) -except Exception as exc: # pylint: disable=broad-except +except Exception as exc: print("An error occurred!") sys.exit(exc) else: diff --git a/aiida_optimade/cli/cmd_calc.py b/aiida_optimade/cli/cmd_calc.py index ff9f2385..d3f36589 100644 --- a/aiida_optimade/cli/cmd_calc.py +++ b/aiida_optimade/cli/cmd_calc.py @@ -1,4 +1,3 @@ -# pylint: disable=protected-access,too-many-locals,too-many-branches from typing import TYPE_CHECKING import click @@ -143,7 +142,7 @@ def calc(obj: "AttributeDict", fields: "Tuple[str]", force_yes: bool, silent: bo except click.Abort: echo.echo_warning("Aborted!") return - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: import traceback exception = traceback.format_exc() diff --git a/aiida_optimade/cli/cmd_init.py b/aiida_optimade/cli/cmd_init.py index e590e369..e3b578bc 100644 --- a/aiida_optimade/cli/cmd_init.py +++ b/aiida_optimade/cli/cmd_init.py @@ -1,4 +1,3 @@ -# pylint: disable=protected-access,too-many-statements from pathlib import Path from typing import TYPE_CHECKING @@ -201,7 +200,7 @@ def init(obj: "AttributeDict", force: bool, silent: bool, mongo: bool, filename: cli=not silent, entries=entries if mongo else None, ) - except Exception as exc: # pylint: disable=broad-except + except Exception as exc: import traceback exception = traceback.format_exc() diff --git a/aiida_optimade/cli/cmd_run.py b/aiida_optimade/cli/cmd_run.py index fc913229..fd9a52d0 100644 --- a/aiida_optimade/cli/cmd_run.py +++ b/aiida_optimade/cli/cmd_run.py @@ -1,4 +1,3 @@ -# pylint: disable=too-many-arguments from typing import TYPE_CHECKING import click diff --git a/aiida_optimade/common/__init__.py b/aiida_optimade/common/__init__.py index 2d152b60..4f446a1a 100644 --- a/aiida_optimade/common/__init__.py +++ b/aiida_optimade/common/__init__.py @@ -1,6 +1,5 @@ -# pylint: disable=undefined-variable -from .exceptions import * # noqa: F403 -from .logger import LOGGER # noqa: F401 -from .warnings import * # noqa: F403 +from .exceptions import * +from .logger import LOGGER +from .warnings import * -__all__ = ("LOGGER",) + exceptions.__all__ + warnings.__all__ # noqa: F405 +__all__ = ("LOGGER",) + exceptions.__all__ + warnings.__all__ diff --git a/aiida_optimade/entry_collections.py b/aiida_optimade/entry_collections.py index 8d0c9807..46ff7421 100644 --- a/aiida_optimade/entry_collections.py +++ b/aiida_optimade/entry_collections.py @@ -173,7 +173,7 @@ def count(self, **kwargs) -> int: return self._count.get("count", 0) - def find( # pylint: disable=too-many-branches + def find( self, params: Union[EntryListingQueryParams, SingleEntryQueryParams] ) -> tuple[ Union[list[EntryResource], EntryResource, None], int, bool, set[str], set[str] @@ -440,7 +440,7 @@ def _find_extras_fields(self, filters: Union[dict, list]) -> None: """ from copy import deepcopy - def __filter_fields_util( # pylint: disable=unused-private-member + def __filter_fields_util( _filters: Union[dict, list] ) -> Union[dict, list]: if isinstance(_filters, dict): diff --git a/aiida_optimade/mappers/__init__.py b/aiida_optimade/mappers/__init__.py index 77468a77..4871df8f 100644 --- a/aiida_optimade/mappers/__init__.py +++ b/aiida_optimade/mappers/__init__.py @@ -1,5 +1,4 @@ -# pylint: disable=undefined-variable -from .entries import * # noqa: F403 -from .structures import * # noqa: F403 +from .entries import * +from .structures import * -__all__ = entries.__all__ + structures.__all__ # noqa: F405 +__all__ = entries.__all__ + structures.__all__ diff --git a/aiida_optimade/mappers/entries.py b/aiida_optimade/mappers/entries.py index d98fe465..9a0424a9 100644 --- a/aiida_optimade/mappers/entries.py +++ b/aiida_optimade/mappers/entries.py @@ -1,4 +1,3 @@ -# pylint: disable=arguments-differ from typing import Any from optimade.server.mappers import BaseResourceMapper as OptimadeResourceMapper diff --git a/aiida_optimade/models/structures.py b/aiida_optimade/models/structures.py index c89c7984..8e4c9c69 100644 --- a/aiida_optimade/models/structures.py +++ b/aiida_optimade/models/structures.py @@ -1,4 +1,3 @@ -# pylint: disable=missing-class-docstring,too-few-public-methods from datetime import datetime from optimade.models import StructureResource as OptimadeStructureResource diff --git a/aiida_optimade/routers/info.py b/aiida_optimade/routers/info.py index 88631d81..d40f1f43 100644 --- a/aiida_optimade/routers/info.py +++ b/aiida_optimade/routers/info.py @@ -1,4 +1,3 @@ -# pylint: disable=missing-function-docstring import urllib from typing import Union diff --git a/aiida_optimade/routers/links.py b/aiida_optimade/routers/links.py index 6d681ce0..cb8db1c4 100644 --- a/aiida_optimade/routers/links.py +++ b/aiida_optimade/routers/links.py @@ -1,5 +1,4 @@ """Reusing the optimade-python-tools /links endpoint""" -# pylint: disable=missing-function-docstring from typing import Union from fastapi import APIRouter, Depends, Request diff --git a/aiida_optimade/routers/structures.py b/aiida_optimade/routers/structures.py index ac5074a3..7b89468a 100644 --- a/aiida_optimade/routers/structures.py +++ b/aiida_optimade/routers/structures.py @@ -1,4 +1,3 @@ -# pylint: disable=missing-function-docstring from typing import Union from fastapi import APIRouter, Depends, Request diff --git a/aiida_optimade/transformers/__init__.py b/aiida_optimade/transformers/__init__.py index caa7ab3a..6727b321 100644 --- a/aiida_optimade/transformers/__init__.py +++ b/aiida_optimade/transformers/__init__.py @@ -1,4 +1,3 @@ -# pylint: disable=undefined-variable -from .aiida import * # noqa: F403 +from .aiida import * -__all__ = aiida.__all__ # noqa: F405 +__all__ = aiida.__all__ diff --git a/aiida_optimade/transformers/aiida.py b/aiida_optimade/transformers/aiida.py index 32899a5a..6fe03ec8 100644 --- a/aiida_optimade/transformers/aiida.py +++ b/aiida_optimade/transformers/aiida.py @@ -1,4 +1,3 @@ -# pylint: disable=no-self-use,too-many-public-methods from lark import v_args from optimade.filtertransformers import BaseTransformer, Quantity from optimade.server.exceptions import BadRequest diff --git a/aiida_optimade/translators/__init__.py b/aiida_optimade/translators/__init__.py index b69fa044..9c298159 100644 --- a/aiida_optimade/translators/__init__.py +++ b/aiida_optimade/translators/__init__.py @@ -1,12 +1,11 @@ -# pylint: disable=undefined-variable -from .cifs import * # noqa: F403 -from .entities import * # noqa: F403 -from .structures import * # noqa: F403 -from .utils import * # noqa: F403 +from .cifs import * +from .entities import * +from .structures import * +from .utils import * __all__ = ( - entities.__all__ # noqa: F405 - + cifs.__all__ # noqa: F405 - + structures.__all__ # noqa: F405 - + utils.__all__ # noqa: F405 + entities.__all__ + + cifs.__all__ + + structures.__all__ + + utils.__all__ ) diff --git a/aiida_optimade/translators/entities.py b/aiida_optimade/translators/entities.py index 3fd233e1..6fd87fc6 100644 --- a/aiida_optimade/translators/entities.py +++ b/aiida_optimade/translators/entities.py @@ -9,7 +9,7 @@ __all__ = ("AiidaEntityTranslator",) -class AiidaEntityTranslator: # pylint: disable=too-few-public-methods +class AiidaEntityTranslator: """Create OPTIMADE entry attributes from an AiiDA Entity Node - Base class For speed and reusability, save attributes in the Node's extras. diff --git a/aiida_optimade/translators/structures.py b/aiida_optimade/translators/structures.py index 665ddfea..df07c6a2 100644 --- a/aiida_optimade/translators/structures.py +++ b/aiida_optimade/translators/structures.py @@ -1,4 +1,3 @@ -# pylint: disable=line-too-long,too-many-public-methods import itertools from math import fsum from typing import Any, Union diff --git a/requirements_dev.txt b/requirements_dev.txt index f5a90f77..e6e30d9d 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,3 +1,2 @@ invoke~=2.2 pre-commit~=3.5 -pylint~=3.0 diff --git a/setup.py b/setup.py index 8fd52fc0..90b80274 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ DEV = [f"{_.strip()}" for _ in handle.readlines()] + TESTING setup( - long_description=open(MODULE_DIR.joinpath("README.md")).read(), + long_description=(MODULE_DIR / "README.md").read_text(encoding="utf8"), long_description_content_type="text/markdown", packages=find_packages(exclude=["tests", "profiles"]), python_requires=">=3.9", diff --git a/tasks.py b/tasks.py index 100ff156..3cf416d7 100644 --- a/tasks.py +++ b/tasks.py @@ -1,9 +1,13 @@ import re +from typing import TYPE_CHECKING from invoke import task +if TYPE_CHECKING: + from typing import Optional -def update_file(filename: str, sub_line: tuple[str, str], strip: str = None): + +def update_file(filename: str, sub_line: tuple[str, str], strip: "Optional[str]" = None): """Utility function for tasks to read, update, and write files""" with open(filename) as handle: lines = [ diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py index 0379dc00..a4f34689 100644 --- a/tests/cli/conftest.py +++ b/tests/cli/conftest.py @@ -1,5 +1,4 @@ """Pytest fixtures for command line interface tests.""" -# pylint: disable=redefined-outer-name,import-error import os import signal from subprocess import PIPE, Popen, TimeoutExpired diff --git a/tests/cli/test_calc.py b/tests/cli/test_calc.py index ac89a691..f6b703cb 100644 --- a/tests/cli/test_calc.py +++ b/tests/cli/test_calc.py @@ -1,5 +1,4 @@ """Test CLI `aiida-optimade calc` command""" -# pylint: disable=unused-argument,too-many-locals,import-error import os import re diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py index 06e52a66..2dbee4d7 100644 --- a/tests/cli/test_init.py +++ b/tests/cli/test_init.py @@ -1,5 +1,4 @@ """Test CLI `aiida-optimade init` command""" -# pylint: disable=import-error,too-many-locals import os import re diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index f41450c9..55e35a30 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -1,4 +1,3 @@ -# pylint: disable=redefined-outer-name,unused-argument import json import os import signal diff --git a/tests/conftest.py b/tests/conftest.py index eb983da4..8cd7d1fa 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,3 @@ -# pylint: disable=unused-argument,redefined-outer-name,import-error import os from pathlib import Path diff --git a/tests/server/conftest.py b/tests/server/conftest.py index b436d161..8dffe1b3 100644 --- a/tests/server/conftest.py +++ b/tests/server/conftest.py @@ -1,4 +1,3 @@ -# pylint: disable=redefined-outer-name import re from typing import TYPE_CHECKING diff --git a/tests/server/query_params/test_filter.py b/tests/server/query_params/test_filter.py index d53ac6db..2dd6dc99 100644 --- a/tests/server/query_params/test_filter.py +++ b/tests/server/query_params/test_filter.py @@ -1,5 +1,4 @@ """Test the `filters` query parameter.""" -# pylint: disable=missing-function-docstring,protected-access,import-error,too-many-statements import os import pytest diff --git a/tests/server/query_params/test_page_limit.py b/tests/server/query_params/test_page_limit.py index 7e7e7033..b2ec7756 100644 --- a/tests/server/query_params/test_page_limit.py +++ b/tests/server/query_params/test_page_limit.py @@ -1,5 +1,4 @@ """Test the `page_limit` query parameter""" -# pylint: disable=import-error,protected-access def test_limit(get_good_response): diff --git a/tests/server/query_params/test_page_offset.py b/tests/server/query_params/test_page_offset.py index fe4dc0e2..b343a061 100644 --- a/tests/server/query_params/test_page_offset.py +++ b/tests/server/query_params/test_page_offset.py @@ -1,5 +1,4 @@ """Test the `page_offset` query parameter""" -# pylint: disable=import-error,protected-access def test_offset(get_good_response): diff --git a/tests/server/query_params/test_sort.py b/tests/server/query_params/test_sort.py index 2739b86a..d0ac62ea 100644 --- a/tests/server/query_params/test_sort.py +++ b/tests/server/query_params/test_sort.py @@ -1,5 +1,4 @@ """Test sort query parameter""" -# pylint: disable=import-error from datetime import datetime, timezone from aiida import orm diff --git a/tests/server/test_entry_collections.py b/tests/server/test_entry_collections.py index 093edacb..15cebfb3 100644 --- a/tests/server/test_entry_collections.py +++ b/tests/server/test_entry_collections.py @@ -1,5 +1,4 @@ """Tests for aiida_optimade.entry_collections.""" -# pylint: disable=protected-access from typing import Any, Callable import pytest diff --git a/tests/server/test_middleware.py b/tests/server/test_middleware.py index d4c5ced9..86cc9c2f 100644 --- a/tests/server/test_middleware.py +++ b/tests/server/test_middleware.py @@ -1,5 +1,4 @@ """Test middleware""" -# pylint: disable=import-error import pytest from optimade import __api_version__ diff --git a/tests/server/utils.py b/tests/server/utils.py index 5d34572f..6883ece0 100644 --- a/tests/server/utils.py +++ b/tests/server/utils.py @@ -1,4 +1,3 @@ -# pylint: disable=no-name-in-module,too-many-arguments,import-error import json import re import warnings @@ -65,7 +64,7 @@ def __init__( version = f"/v{__api_version__.split('.')[0]}" self.version = version - def request( # pylint: disable=too-many-locals + def request( self, method: str, url: "httpx._types.URLTypes", @@ -73,7 +72,7 @@ def request( # pylint: disable=too-many-locals content: "Optional[httpx._types.RequestContent]" = None, data: "Optional[testclient._RequestData]" = None, files: "Optional[httpx._types.RequestFiles]" = None, - json: "Any" = None, # pylint: disable=redefined-outer-name + json: "Any" = None, params: "Optional[httpx._types.QueryParamTypes]" = None, headers: "Optional[httpx._types.HeaderTypes]" = None, cookies: "Optional[httpx._types.CookieTypes]" = None, diff --git a/tests/transformers/test_aiida.py b/tests/transformers/test_aiida.py index 2e2b40aa..4397a94d 100644 --- a/tests/transformers/test_aiida.py +++ b/tests/transformers/test_aiida.py @@ -1,4 +1,3 @@ -# pylint: disable=import-error import pytest from lark.exceptions import VisitError from optimade.filterparser import LarkParser From 6d586b881b8a0acb9c44f2ab03c79bee78d98d19 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 9 Nov 2023 08:59:47 +0100 Subject: [PATCH 3/7] Fix regex for ruff hook --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 265d25aa..a1d52588 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: hooks: - id: ruff name: ruff core code base - exclude: ^(.*\.py|(\.github|tests)/.*)$ + exclude: ^([^/]*\.py|(\.github|tests)/.*)$ # Fix what can be fixed in-place and exit with non-zero status if files were # changed and/or there are rules violations. args: From c5d75a0b629debeb9bacf9259146fbdf0d31695f Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 9 Nov 2023 11:41:19 +0100 Subject: [PATCH 4/7] Update code according to ruff rules --- .pre-commit-config.yaml | 3 + aiida_optimade/cli/cmd_calc.py | 13 +-- aiida_optimade/cli/cmd_init.py | 16 +-- aiida_optimade/common/__init__.py | 21 +++- aiida_optimade/common/exceptions.py | 8 +- aiida_optimade/common/warnings.py | 3 +- aiida_optimade/config.py | 5 +- aiida_optimade/entry_collections.py | 40 +++---- aiida_optimade/mappers/__init__.py | 6 +- aiida_optimade/mappers/entries.py | 13 ++- aiida_optimade/mappers/structures.py | 14 ++- aiida_optimade/routers/info.py | 16 +-- aiida_optimade/transformers/__init__.py | 4 +- aiida_optimade/transformers/aiida.py | 29 +++-- aiida_optimade/translators/__init__.py | 16 +-- aiida_optimade/translators/cifs.py | 7 +- aiida_optimade/translators/entities.py | 7 +- aiida_optimade/translators/structures.py | 133 ++++++++++++++--------- aiida_optimade/translators/utils.py | 20 ++-- tasks.py | 4 +- tests/cli/conftest.py | 10 +- tests/cli/test_calc.py | 4 +- tests/cli/test_init.py | 4 +- tests/conftest.py | 3 +- tests/server/routers/test_info.py | 4 +- tests/server/utils.py | 7 +- 26 files changed, 239 insertions(+), 171 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a1d52588..753f5741 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -56,6 +56,9 @@ repos: # We will, e.g., extend the recommended number of branches, statements, function # args, etc. - --extend-ignore=PLR09 + # Ignore these, as they conflict with the intended FastAPI way of doing dependency + # injection. + - --extend-ignore=B008 # ruff is a Python linter, incl. import sorter and formatter # It works partly on files in-place diff --git a/aiida_optimade/cli/cmd_calc.py b/aiida_optimade/cli/cmd_calc.py index d3f36589..ecb0c529 100644 --- a/aiida_optimade/cli/cmd_calc.py +++ b/aiida_optimade/cli/cmd_calc.py @@ -74,12 +74,11 @@ def calc(obj: "AttributeDict", fields: "Tuple[str]", force_yes: bool, silent: bo } number_of_nodes = STRUCTURES.count(**query_kwargs) - if number_of_nodes: - if not silent: - echo.echo_info( - f"Field{'s' if len(fields) > 1 else ''} found for {number_of_nodes}" - f" Node{'s' if number_of_nodes > 1 else ''}." - ) + if number_of_nodes and not silent: + echo.echo_info( + f"Field{'s' if len(fields) > 1 else ''} found for {number_of_nodes}" + f" Node{'s' if number_of_nodes > 1 else ''}." + ) if not silent: echo.echo_info( f"Total number of Nodes in profile {profile!r}: {STRUCTURES.count()}" @@ -142,7 +141,7 @@ def calc(obj: "AttributeDict", fields: "Tuple[str]", force_yes: bool, silent: bo except click.Abort: echo.echo_warning("Aborted!") return - except Exception as exc: + except Exception as exc: # noqa: BLE001 import traceback exception = traceback.format_exc() diff --git a/aiida_optimade/cli/cmd_init.py b/aiida_optimade/cli/cmd_init.py index e3b578bc..e09ccb28 100644 --- a/aiida_optimade/cli/cmd_init.py +++ b/aiida_optimade/cli/cmd_init.py @@ -9,7 +9,7 @@ if TYPE_CHECKING: # pragma: no cover from collections.abc import Generator, Iterator - from typing import IO, List, Union + from typing import IO, List, Optional, Union from aiida.common.extendeddicts import AttributeDict @@ -200,7 +200,7 @@ def init(obj: "AttributeDict", force: bool, silent: bool, mongo: bool, filename: cli=not silent, entries=entries if mongo else None, ) - except Exception as exc: + except Exception as exc: # noqa: BLE001 import traceback exception = traceback.format_exc() @@ -226,7 +226,7 @@ def init(obj: "AttributeDict", force: bool, silent: bool, mongo: bool, filename: def read_chunks( - file_object: "IO", chunk_size: int = None + file_object: "IO", chunk_size: "Optional[int]" = None ) -> "Generator[Union[str, bytes], None, None]": """Generator to read a file piece by piece @@ -254,17 +254,17 @@ def get_documents( rest_chunk = "" for raw_chunk in chunk_iterator: - raw_chunk = rest_chunk + raw_chunk + full_raw_chunk = rest_chunk + raw_chunk rest_chunk = "" - curly_start_count = raw_chunk.count("{") - curly_end_count = raw_chunk.count("}") + curly_start_count = full_raw_chunk.count("{") + curly_end_count = full_raw_chunk.count("}") if curly_start_count == 0 or curly_end_count == 0: - rest_chunk = raw_chunk + rest_chunk = full_raw_chunk continue - chunk = raw_chunk + chunk = full_raw_chunk while curly_end_count - curly_start_count != 0: chunk = chunk.split("{") rest_chunk = "{" + f"{chunk[-1]}{rest_chunk}" diff --git a/aiida_optimade/common/__init__.py b/aiida_optimade/common/__init__.py index 4f446a1a..587580af 100644 --- a/aiida_optimade/common/__init__.py +++ b/aiida_optimade/common/__init__.py @@ -1,5 +1,20 @@ -from .exceptions import * +from .exceptions import ( + AiidaEntityNotFound, + AiidaError, + AiidaOptimadeException, + CausationError, + OptimadeIntegrityError, +) from .logger import LOGGER -from .warnings import * +from .warnings import AiidaOptimadeWarning, NotImplementedWarning -__all__ = ("LOGGER",) + exceptions.__all__ + warnings.__all__ +__all__ = ( + "LOGGER", + "AiidaOptimadeException", + "AiidaEntityNotFound", + "OptimadeIntegrityError", + "CausationError", + "AiidaError", + "AiidaOptimadeWarning", + "NotImplementedWarning", +) diff --git a/aiida_optimade/common/exceptions.py b/aiida_optimade/common/exceptions.py index 676fbb81..854609c7 100644 --- a/aiida_optimade/common/exceptions.py +++ b/aiida_optimade/common/exceptions.py @@ -1,10 +1,4 @@ -__all__ = ( - "AiidaOptimadeException", - "AiidaEntityNotFound", - "OptimadeIntegrityError", - "CausationError", - "AiidaError", -) +"""Exceptions for aiida-optimade.""" class AiidaOptimadeException(Exception): diff --git a/aiida_optimade/common/warnings.py b/aiida_optimade/common/warnings.py index 4d9986bc..44ba45ee 100644 --- a/aiida_optimade/common/warnings.py +++ b/aiida_optimade/common/warnings.py @@ -1,7 +1,6 @@ +"""Warnings for aiida-optimade.""" from optimade.server.warnings import OptimadeWarning -__all__ = ("AiidaOptimadeWarning", "NotImplementedWarning") - class AiidaOptimadeWarning(OptimadeWarning): """Root Warning for aiida-optimade. diff --git a/aiida_optimade/config.py b/aiida_optimade/config.py index 50700fec..b6aa9190 100644 --- a/aiida_optimade/config.py +++ b/aiida_optimade/config.py @@ -7,7 +7,10 @@ class CustomServerConfig(ServerConfig): query_group: Optional[str] = Field( None, - description="The AiiDA Group containing the data that will be served, allowing one to serve a curated set of data from a given database.", + description=( + "The AiiDA Group containing the data that will be served, allowing one to " + "serve a curated set of data from a given database." + ), ) diff --git a/aiida_optimade/entry_collections.py b/aiida_optimade/entry_collections.py index 46ff7421..dbdcea83 100644 --- a/aiida_optimade/entry_collections.py +++ b/aiida_optimade/entry_collections.py @@ -1,5 +1,5 @@ import warnings -from typing import Any, Optional, Union +from typing import TYPE_CHECKING, ClassVar from aiida.orm import Group from aiida.orm.nodes import Node @@ -17,11 +17,14 @@ from aiida_optimade.transformers import AiidaTransformer from aiida_optimade.utils import retrieve_queryable_properties +if TYPE_CHECKING: # pragma: no cover + from typing import Any, Optional, Union + class AiidaCollection(EntryCollection): """Collection of AiiDA entities""" - CAST_MAPPING = { + CAST_MAPPING: ClassVar = { "string": "t", "float": "f", "integer": "i", @@ -31,8 +34,8 @@ class AiidaCollection(EntryCollection): def __init__( self, - entities: Union[str, list[str]], - group: Optional[str], + entities: "Union[str, list[str]]", + group: "Optional[str]", resource_cls: EntryResource, resource_mapper: ResourceMapper, ): @@ -130,7 +133,7 @@ def count(self, **kwargs) -> int: "offset": kwargs.get("offset", None), } else: - for limiting_param in {"filters", "limit", "offset"}: + for limiting_param in ("filters", "limit", "offset"): if kwargs.get(limiting_param, None) != self._count.get( limiting_param, None ): @@ -174,9 +177,9 @@ def count(self, **kwargs) -> int: return self._count.get("count", 0) def find( - self, params: Union[EntryListingQueryParams, SingleEntryQueryParams] + self, params: "Union[EntryListingQueryParams, SingleEntryQueryParams]" ) -> tuple[ - Union[list[EntryResource], EntryResource, None], int, bool, set[str], set[str] + "Union[list[EntryResource], EntryResource, None], int, bool, set[str], set[str]" ]: self.set_data_available() @@ -243,7 +246,8 @@ def find( "Unrecognised field(s) for this provider requested in " f"`response_fields`: {bad_provider_fields}." ) - ) + ), + stacklevel=1, ) if bad_optimade_fields: @@ -266,8 +270,8 @@ def find( ) def _run_db_query( - self, criteria: dict[str, Any], single_entry: bool = False - ) -> tuple[list[dict[str, Any]], bool]: + self, criteria: dict[str, "Any"], single_entry: bool = False + ) -> tuple[list[dict[str, "Any"]], bool]: """Run the query on the backend and collect the results. Arguments: @@ -296,7 +300,7 @@ def _run_db_query( @staticmethod def _prepare_query( - node_types: list[str], group: Optional[str] = None, **kwargs + node_types: list[str], group: "Optional[str]" = None, **kwargs ) -> QueryBuilder: """Workhorse function to prepare an AiiDA QueryBuilder query""" for key in kwargs: @@ -343,8 +347,8 @@ def _perform_count(self, **kwargs) -> int: return res def handle_query_params( - self, params: Union[EntryListingQueryParams, SingleEntryQueryParams] - ) -> dict[str, Any]: + self, params: "Union[EntryListingQueryParams, SingleEntryQueryParams]" + ) -> dict[str, "Any"]: """Parse and interpret the backend-agnostic query parameter models into a dictionary that can be used by AiiDA's QueryBuilder. @@ -430,7 +434,7 @@ def parse_sort_params(self, sort_params: str) -> list[dict[str, dict[str, str]]] ) return sort_spec - def _find_extras_fields(self, filters: Union[dict, list]) -> None: + def _find_extras_fields(self, filters: "Union[dict, list]") -> None: """Collect all properties to be found in AiiDA Node extras. Parameters: @@ -440,9 +444,7 @@ def _find_extras_fields(self, filters: Union[dict, list]) -> None: """ from copy import deepcopy - def __filter_fields_util( - _filters: Union[dict, list] - ) -> Union[dict, list]: + def __filter_fields_util(_filters: "Union[dict, list]") -> "Union[dict, list]": if isinstance(_filters, dict): res = {} for key, value in _filters.items(): @@ -473,7 +475,7 @@ def __filter_fields_util( __filter_fields_util(deepcopy(filters)) def _check_and_calculate_entities( - self, cli: bool = False, entries: list[list[int]] = None + self, cli: bool = False, entries: "Optional[list[list[int]]]" = None ) -> list[int]: """Check all entities have OPTIMADE extras, else calculate them @@ -490,7 +492,7 @@ def _check_and_calculate_entities( """ - def _update_entities(entities: list[list[Any]], fields: list[str]): + def _update_entities(entities: list[list["Any"]], fields: list[str]): """Utility function to update entities within this method""" optimade_fields = [ self.resource_mapper.get_optimade_field(_) for _ in fields diff --git a/aiida_optimade/mappers/__init__.py b/aiida_optimade/mappers/__init__.py index 4871df8f..1a6f0351 100644 --- a/aiida_optimade/mappers/__init__.py +++ b/aiida_optimade/mappers/__init__.py @@ -1,4 +1,4 @@ -from .entries import * -from .structures import * +from .entries import ResourceMapper +from .structures import StructureMapper -__all__ = entries.__all__ + structures.__all__ +__all__ = ("ResourceMapper", "StructureMapper") diff --git a/aiida_optimade/mappers/entries.py b/aiida_optimade/mappers/entries.py index 9a0424a9..e9af2572 100644 --- a/aiida_optimade/mappers/entries.py +++ b/aiida_optimade/mappers/entries.py @@ -1,10 +1,11 @@ -from typing import Any +from typing import TYPE_CHECKING, ClassVar from optimade.server.mappers import BaseResourceMapper as OptimadeResourceMapper from aiida_optimade.translators.entities import AiidaEntityTranslator -__all__ = ("ResourceMapper",) +if TYPE_CHECKING: # pragma: no cover + from typing import Any, Optional class ResourceMapper(OptimadeResourceMapper): @@ -13,8 +14,8 @@ class ResourceMapper(OptimadeResourceMapper): PROJECT_PREFIX: str = "extras.optimade." TRANSLATORS: dict[str, AiidaEntityTranslator] - REQUIRED_ATTRIBUTES: set[str] = set() - TOP_LEVEL_NON_ATTRIBUTES_FIELDS: set[str] = { + REQUIRED_ATTRIBUTES: ClassVar[set[str]] = set() + TOP_LEVEL_NON_ATTRIBUTES_FIELDS: ClassVar[set[str]] = { "id", "type", "relationships", @@ -36,7 +37,7 @@ def all_aliases(cls) -> tuple[tuple[str, str]]: ) @classmethod - def map_back(cls, entity_properties: dict[str, Any]) -> dict: + def map_back(cls, entity_properties: dict[str, "Any"]) -> dict: """Map properties from AiiDA to OPTIMADE Parameters: @@ -82,7 +83,7 @@ def build_attributes( retrieved_attributes: dict, entry_pk: int, node_type: str, - missing_attributes: dict = None, + missing_attributes: "Optional[dict]" = None, ) -> dict: """Build attributes dictionary for OPTIMADE structure resource diff --git a/aiida_optimade/mappers/structures.py b/aiida_optimade/mappers/structures.py index c7f67b41..7119fad3 100644 --- a/aiida_optimade/mappers/structures.py +++ b/aiida_optimade/mappers/structures.py @@ -1,4 +1,5 @@ import warnings +from typing import TYPE_CHECKING, ClassVar from optimade.server.config import CONFIG, SupportedBackend @@ -12,17 +13,20 @@ hex_to_floats, ) -__all__ = ("StructureMapper",) +if TYPE_CHECKING: # pragma: no cover + from typing import Optional class StructureMapper(ResourceMapper): """Map 'structure' resources from OPTIMADE to AiiDA""" - TRANSLATORS: dict[str, AiidaEntityTranslator] = { + TRANSLATORS: ClassVar[dict[str, AiidaEntityTranslator]] = { "data.core.cif.CifData.": CifDataTranslator, "data.core.structure.StructureData.": StructureDataTranslator, } - REQUIRED_ATTRIBUTES = set(StructureResourceAttributes.schema().get("required")) + REQUIRED_ATTRIBUTES: ClassVar[set[str]] = set( + StructureResourceAttributes.schema().get("required") + ) # This should be REQUIRED_FIELDS, but should be set as such in `optimade` ENTRY_RESOURCE_CLASS = StructureResource @@ -32,7 +36,7 @@ def build_attributes( retrieved_attributes: dict, entry_pk: int, node_type: str, - missing_attributes: set = None, + missing_attributes: "Optional[set]" = None, ) -> dict: """Build attributes dictionary for OPTIMADE structure resource @@ -89,6 +93,7 @@ def build_attributes( f"{translator.__class__.__name__} has not yet been " "implemented.", NotImplementedWarning, + stacklevel=1, ) else: warnings.warn( @@ -97,6 +102,7 @@ def build_attributes( "implemented. This may be a mistake, but may also be fine, " "since a MongoDB is used.", NotImplementedWarning, + stacklevel=1, ) else: res[attribute] = create_attribute() diff --git a/aiida_optimade/routers/info.py b/aiida_optimade/routers/info.py index d40f1f43..39507b19 100644 --- a/aiida_optimade/routers/info.py +++ b/aiida_optimade/routers/info.py @@ -44,20 +44,23 @@ def get_info(request: Request): api_version=__api_version__, available_api_versions=[ { - "url": f"{base_url}{root_path_str}/v{__api_version__.split('-')[0].split('+')[0].split('.')[0]}", + "url": ( + f"{base_url}{root_path_str}/v" + f"{__api_version__.split('-')[0].split('+')[0].split('.')[0]}" + ), "version": __api_version__, } ], formats=["json"], - entry_types_by_format={"json": list(ENTRY_INFO_SCHEMAS.keys())}, + entry_types_by_format={"json": list(ENTRY_INFO_SCHEMAS)}, available_endpoints=[ "info", "links", "extensions/docs", "extensions/redoc", "extensions/openapi.json", - ] - + list(ENTRY_INFO_SCHEMAS.keys()), + *ENTRY_INFO_SCHEMAS, + ], is_index=False, ), ), @@ -74,12 +77,11 @@ def get_info(request: Request): def get_info_entry(request: Request, entry: str): from optimade.models import EntryInfoResource - valid_entry_info_endpoints = ENTRY_INFO_SCHEMAS.keys() - if entry not in valid_entry_info_endpoints: + if entry not in ENTRY_INFO_SCHEMAS: raise HTTPException( status_code=404, detail=f"Entry info not found for {entry}, valid entry info endpoints are:" - f" {valid_entry_info_endpoints}", + f" {ENTRY_INFO_SCHEMAS.keys()}", ) schema = ENTRY_INFO_SCHEMAS[entry]() diff --git a/aiida_optimade/transformers/__init__.py b/aiida_optimade/transformers/__init__.py index 6727b321..a184e295 100644 --- a/aiida_optimade/transformers/__init__.py +++ b/aiida_optimade/transformers/__init__.py @@ -1,3 +1,3 @@ -from .aiida import * +from .aiida import AiidaTransformer -__all__ = aiida.__all__ +__all__ = ("AiidaTransformer",) diff --git a/aiida_optimade/transformers/aiida.py b/aiida_optimade/transformers/aiida.py index 6fe03ec8..ad513622 100644 --- a/aiida_optimade/transformers/aiida.py +++ b/aiida_optimade/transformers/aiida.py @@ -1,16 +1,24 @@ +"""Transformer for converting OPTIMADE filter queries to AiiDA QueryBuilder queries.""" +from typing import TYPE_CHECKING, ClassVar + from lark import v_args from optimade.filtertransformers import BaseTransformer, Quantity from optimade.server.exceptions import BadRequest -__all__ = ("AiidaTransformer",) +if TYPE_CHECKING: # pragma: no cover + from typing import Optional class AiidaTransformer(BaseTransformer): """Transform OPTIMADE query to AiiDA QueryBuilder queryhelp query""" # Conversion map from the OPTIMADE operators to the QueryBuilder operators - operator_map = {"=": "==", "!=": "!==", "in": "contains"} - _reversed_operator_map = { + operator_map: ClassVar[dict[str, "Optional[str]"]] = { + "=": "==", + "!=": "!==", + "in": "contains", + } + _reversed_operator_map: ClassVar[dict[str, str]] = { "<": ">", "<=": ">=", ">": "<", @@ -18,7 +26,11 @@ class AiidaTransformer(BaseTransformer): "!=": "!==", "=": "==", } - list_operator_map = {"<": "shorter", ">": "longer", "=": "of_length"} + list_operator_map: ClassVar[dict[str, str]] = { + "<": "shorter", + ">": "longer", + "=": "of_length", + } def value_list(self, arg): """value_list: [ OPERATOR ] value ( "," [ OPERATOR ] value )*""" @@ -138,7 +150,8 @@ def set_op_rhs(self, arg): ANY value_list | ONLY value_list ) """ - if len(arg) == 2: + length_value_without_operator = 2 + if len(arg) == length_value_without_operator: # only value without OPERATOR return {"contains": [arg[1]]} @@ -169,10 +182,8 @@ def length_op_rhs(self, arg): """ length_op_rhs: LENGTH [ OPERATOR ] value """ - if len(arg) == 3: - operator = arg[1].value - else: - operator = "=" + length_including_operator = 3 + operator = arg[1].value if len(arg) == length_including_operator else "=" if operator in self.list_operator_map: return {self.list_operator_map[operator]: arg[-1]} diff --git a/aiida_optimade/translators/__init__.py b/aiida_optimade/translators/__init__.py index 9c298159..9e320a13 100644 --- a/aiida_optimade/translators/__init__.py +++ b/aiida_optimade/translators/__init__.py @@ -1,11 +1,11 @@ -from .cifs import * -from .entities import * -from .structures import * -from .utils import * +from .cifs import CifDataTranslator +from .entities import AiidaEntityTranslator +from .structures import StructureDataTranslator +from .utils import hex_to_floats __all__ = ( - entities.__all__ - + cifs.__all__ - + structures.__all__ - + utils.__all__ + "CifDataTranslator", + "AiidaEntityTranslator", + "StructureDataTranslator", + "hex_to_floats", ) diff --git a/aiida_optimade/translators/cifs.py b/aiida_optimade/translators/cifs.py index 69e85ed1..48fbc1a9 100644 --- a/aiida_optimade/translators/cifs.py +++ b/aiida_optimade/translators/cifs.py @@ -5,8 +5,6 @@ from aiida_optimade.translators.structures import StructureDataTranslator -__all__ = ("CifDataTranslator",) - def _get_aiida_structure_pymatgen_inline(cif, **kwargs) -> StructureData: """Copy of similar named function in AiiDA-Core. @@ -89,10 +87,9 @@ def __init__(self, pk: str): @property def _node(self) -> StructureData: - if not self._node_loaded: - self.__node = self._get_unique_node_property("*") - elif getattr(self.__node, "pk", 0) != self._pk: + if not self._node_loaded or getattr(self.__node, "pk", 0) != self._pk: self.__node = self._get_unique_node_property("*") + if isinstance(self.__node, StructureData): return self.__node diff --git a/aiida_optimade/translators/entities.py b/aiida_optimade/translators/entities.py index 6fd87fc6..4bee8f67 100644 --- a/aiida_optimade/translators/entities.py +++ b/aiida_optimade/translators/entities.py @@ -6,8 +6,6 @@ from aiida_optimade.common import LOGGER, AiidaEntityNotFound -__all__ = ("AiidaEntityTranslator",) - class AiidaEntityTranslator: """Create OPTIMADE entry attributes from an AiiDA Entity Node - Base class @@ -39,10 +37,9 @@ def _get_unique_node_property( @property def _node(self) -> Node: - if not self._node_loaded: - self.__node = self._get_unique_node_property("*") - elif getattr(self.__node, "pk", 0) != self._pk: + if not self._node_loaded or getattr(self.__node, "pk", 0) != self._pk: self.__node = self._get_unique_node_property("*") + return self.__node @_node.setter diff --git a/aiida_optimade/translators/structures.py b/aiida_optimade/translators/structures.py index df07c6a2..82629fe5 100644 --- a/aiida_optimade/translators/structures.py +++ b/aiida_optimade/translators/structures.py @@ -13,15 +13,14 @@ hex_to_floats, ) -__all__ = ("StructureDataTranslator",) - class StructureDataTranslator(AiidaEntityTranslator): """Create OPTIMADE "structures" attributes from an AiiDA StructureData Node Each OPTIMADE field is a method in this class. - NOTE: This class succeeds in *never* loading the actual AiiDA Node for optimization purposes. + NOTE: This class succeeds in *never* loading the actual AiiDA Node for optimization + purposes. """ AIIDA_ENTITY = StructureData @@ -103,7 +102,8 @@ def get_formula(self, mode="hill", separator=""): break else: raise AiidaError( - f"kind with name {site['kind_name']} cannot be found amongst the kinds {self._kinds}" + f"kind with name {site['kind_name']} cannot be found amongst the " + f"kinds {self._kinds}" ) symbol_list.append(get_symbols_string(kind["symbols"], kind["weights"])) @@ -121,7 +121,8 @@ def get_symbol_weights(self) -> dict: return occupation def has_partial_occupancy(self) -> bool: - """Check for partial occupancies (first vacancies, next through element ratios)""" + """Check for partial occupancies (first vacancies, next through element + ratios)""" if self.has_vacancies(): return True @@ -130,15 +131,12 @@ def has_partial_occupancy(self) -> bool: if not occ.is_integer(): return True - for kind in self._kinds: - if len(kind["weights"]) > 1: - return True - - return False + return any(len(kind["weights"]) > 1 for kind in self._kinds) # Start creating fields def elements(self) -> list[str]: - """Names of elements found in the structure as a list of strings, in alphabetical order.""" + """Names of elements found in the structure as a list of strings, in + alphabetical order.""" attribute = "elements" if attribute in self.new_attributes: @@ -150,7 +148,8 @@ def elements(self) -> list[str]: if "X" in res: res.remove("X") - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = res return res @@ -163,7 +162,8 @@ def nelements(self) -> int: res = len(self.elements()) - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = res return res @@ -179,12 +179,14 @@ def elements_ratios(self) -> list[float]: total_weight = fsum(ratios.values()) res = [ratios[symbol] / total_weight for symbol in self.elements()] - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = floats_to_hex(res) return res def chemical_formula_descriptive(self) -> str: - """The chemical formula for a structure as a string in a form chosen by the API implementation.""" + """The chemical formula for a structure as a string in a form chosen by the API + implementation.""" attribute = "chemical_formula_descriptive" if attribute in self.new_attributes: @@ -192,7 +194,8 @@ def chemical_formula_descriptive(self) -> str: res = self.get_formula() - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = res return res @@ -202,9 +205,10 @@ def chemical_formula_reduced(self) -> str: As a string with element symbols and integer chemical proportion numbers. The proportion number MUST be omitted if it is 1. - NOTE: For structures with partial occupation, the chemical proportion numbers are integers - that within reasonable approximation indicate the correct chemical proportions. - The precise details of how to perform the rounding is chosen by the API implementation. + NOTE: For structures with partial occupation, the chemical proportion numbers + are integers that within reasonable approximation indicate the correct chemical + proportions. The precise details of how to perform the rounding is chosen by + the API implementation. """ attribute = "chemical_formula_reduced" @@ -223,7 +227,8 @@ def chemical_formula_reduced(self) -> str: occupation[symbol] = "" if rounded_weight in (0, 1) else rounded_weight res = "".join([f"{symbol}{occupation[symbol]}" for symbol in self.elements()]) - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = res return res @@ -234,29 +239,29 @@ def chemical_formula_hill(self) -> str: The proportion number MUST be omitted if it is 1. NOTE: If the system has sites with partial occupation and the total occupations - of each element do not all sum up to integers, then the Hill formula SHOULD be handled as unset. + of each element do not all sum up to integers, then the Hill formula SHOULD be + handled as unset. - NOTE: This will always be equal to chemical_formula_descriptive if it should not be handled as unset. + NOTE: This will always be equal to chemical_formula_descriptive if it should not + be handled as unset. """ attribute = "chemical_formula_hill" if attribute in self.new_attributes: return self.new_attributes[attribute] - if self.has_partial_occupancy(): - res = None - else: - res = self.get_formula(mode="hill") + res = None if self.has_partial_occupancy() else self.get_formula(mode="hill") - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = res return res def chemical_formula_anonymous(self) -> str: """The anonymous formula is the chemical_formula_reduced - But where the elements are instead first ordered by their chemical proportion number, - and then, in order left to right, replaced by anonymous symbols: + But where the elements are instead first ordered by their chemical proportion + number, and then, in order left to right, replaced by anonymous symbols: A, B, C, ..., Z, Aa, Ba, ..., Za, Ab, Bb, ... and so on. """ attribute = "chemical_formula_anonymous" @@ -266,17 +271,25 @@ def chemical_formula_anonymous(self) -> str: weights = [weight for _, weight in self.get_symbol_weights().items()] - assert len(ANONYMOUS_ELEMENTS) >= len( - weights - ), f"Not enough generated anonymous elements to create `chemical_formula_anonymous` for Node . Found elements: {len(self.elements())}. Generated anonymous elements: {len(ANONYMOUS_ELEMENTS)}." + if len(ANONYMOUS_ELEMENTS) < len(weights): + raise ValueError( + "Not enough generated anonymous elements to create " + f"`chemical_formula_anonymous` for Node . Found " + "elements: {len(self.elements())}. Generated anonymous elements: " + f"{len(ANONYMOUS_ELEMENTS)}." + ) res = "" min_occupation = min(weights) if weights else None for index, occupation in enumerate(sorted(weights, reverse=True)): rounded_weight = round(occupation / min_occupation) - res += f"{ANONYMOUS_ELEMENTS[index]}{'' if rounded_weight in (0, 1) else rounded_weight}" + res += ( + f"{ANONYMOUS_ELEMENTS[index]}" + f"{'' if rounded_weight in (0, 1) else rounded_weight}" + ) - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = res return res @@ -284,9 +297,10 @@ def dimension_types(self) -> list[int]: """List of three integers. For each of the three directions indicated by the three lattice vectors - (see property lattice_vectors). This list indicates if the direction is periodic (value 1) - or non-periodic (value 0). Note: the elements in this list each refer to the direction - of the corresponding entry in property lattice_vectors and not the Cartesian x, y, z directions. + (see property lattice_vectors). This list indicates if the direction is periodic + (value 1) or non-periodic (value 0). Note: the elements in this list each refer + to the direction of the corresponding entry in property lattice_vectors and not + the Cartesian x, y, z directions. """ attribute = "dimension_types" @@ -295,7 +309,8 @@ def dimension_types(self) -> list[int]: res = self._pbc - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = res return res @@ -308,7 +323,8 @@ def nperiodic_dimensions(self) -> int: res = sum(self._pbc) - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = res return res @@ -321,7 +337,8 @@ def lattice_vectors(self) -> list[list[float]]: res = check_floating_round_errors(self._cell) - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = floats_to_hex(res) return res @@ -329,7 +346,8 @@ def cartesian_site_positions(self) -> list[list[Union[float, None]]]: """Cartesian positions of each site. A site is an atom, a site potentially occupied by an atom, - or a placeholder for a virtual mixture of atoms (e.g., in a virtual crystal approximation). + or a placeholder for a virtual mixture of atoms (e.g., in a virtual crystal + approximation). """ attribute = "cartesian_site_positions" @@ -339,7 +357,8 @@ def cartesian_site_positions(self) -> list[list[Union[float, None]]]: sites = [list(site["position"]) for site in self._sites] res = check_floating_round_errors(sites) - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = floats_to_hex(res) return res @@ -352,7 +371,8 @@ def nsites(self) -> int: res = len(self.cartesian_site_positions()) - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = res return res @@ -360,7 +380,8 @@ def species_at_sites(self) -> list[str]: """Name of the species at each site (Where values for sites are specified with the same order of the property - cartesian_site_positions). The properties of the species are found in the property species. + cartesian_site_positions). The properties of the species are found in the + property species. """ attribute = "species_at_sites" @@ -369,7 +390,8 @@ def species_at_sites(self) -> list[str]: res = [site["kind_name"] for site in self._sites] - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = res return res @@ -377,7 +399,8 @@ def species(self) -> list[dict]: """A list describing the species of the sites of this structure. Species can be pure chemical elements, or virtual-crystal atoms - representing a statistical occupation of a given site by multiple chemical elements. + representing a statistical occupation of a given site by multiple chemical + elements. """ import re @@ -415,14 +438,15 @@ def species(self) -> list[dict]: species["mass"].append(0.0) # Calculate vacancy concentration - if 0.0 <= kind_weight_sum <= 1.0: + if 0.0 <= kind_weight_sum <= 1.0: # noqa: PLR2004 species["concentration"].append(1.0 - kind_weight_sum) else: raise ValueError("kind_weight_sum must be in the interval [0;1]") res.append(species) - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = res return res @@ -438,7 +462,8 @@ def assemblies(self) -> Union[list[dict], None]: res = None - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = res return res @@ -467,15 +492,16 @@ def structure_features(self) -> list[str]: for item in species: if key not in item: raise OptimadeIntegrityError( - f'The required key {key} was not found for {item} in the "species" attribute' + f'The required key {key} was not found for {item} in the "species"' + " attribute" ) if len(item[key]) > 1: res.append("disorder") break # * Unknown positions * - # This flag MUST be present if at least one component of the cartesian_site_positions - # list of lists has value null. + # This flag MUST be present if at least one component of the + # cartesian_site_positions list of lists has value null. cartesian_site_positions = self.cartesian_site_positions() for site in cartesian_site_positions: if float("NaN") in site: @@ -487,6 +513,7 @@ def structure_features(self) -> list[str]: if self.assemblies(): res.append("assemblies") - # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node and return value + # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node + # and return value self.new_attributes[attribute] = res return res diff --git a/aiida_optimade/translators/utils.py b/aiida_optimade/translators/utils.py index cbb8c66d..5ad1fcd0 100644 --- a/aiida_optimade/translators/utils.py +++ b/aiida_optimade/translators/utils.py @@ -1,7 +1,5 @@ from typing import Union -__all__ = ("hex_to_floats",) - def check_floating_round_errors( some_list: list[Union[list[float], float]] @@ -20,9 +18,9 @@ def check_floating_round_errors( for item in some_list: if isinstance(item, list): res.append(check_floating_round_errors(item)) + elif abs(item) < might_as_well_be_zero: + res.append(0.0) else: - if abs(item) < might_as_well_be_zero: - item = 0.0 res.append(item) return res @@ -40,15 +38,16 @@ def floats_to_hex( if isinstance(item, list): res.append(floats_to_hex(item)) else: + item_updated = item if isinstance(item, float): - item = item.hex() - if not isinstance(item, str): + item_updated = item.hex() + if not isinstance(item_updated, str): raise TypeError( "Wrong type passed to floats_to_hex method, must be a " "list of either a list of floats or float values. " f"Item: {item!r}. Type: {type(item)}." ) - res.append(item) + res.append(item_updated) return res @@ -66,19 +65,20 @@ def hex_to_floats( if isinstance(item, list): res.append(hex_to_floats(item)) else: + item_updated = item if isinstance(item, str): try: - item = float.fromhex(item) + item_updated = float.fromhex(item) except ValueError as exc: raise ValueError( f"Could not turn item ({item}) into float from hex. " f"Original exception: {exc!r}" ) from exc - if not isinstance(item, float): + if not isinstance(item_updated, float): raise TypeError( "Wrong type passed to hex_to_floats method, must be a " "list of either a list of strings or string values. " f"Item: {item!r}. Type: {type(item)}." ) - res.append(item) + res.append(item_updated) return res diff --git a/tasks.py b/tasks.py index 3cf416d7..00676de4 100644 --- a/tasks.py +++ b/tasks.py @@ -7,7 +7,9 @@ from typing import Optional -def update_file(filename: str, sub_line: tuple[str, str], strip: "Optional[str]" = None): +def update_file( + filename: str, sub_line: tuple[str, str], strip: "Optional[str]" = None +): """Utility function for tasks to read, update, and write files""" with open(filename) as handle: lines = [ diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py index a4f34689..11f38e95 100644 --- a/tests/cli/conftest.py +++ b/tests/cli/conftest.py @@ -3,10 +3,14 @@ import signal from subprocess import PIPE, Popen, TimeoutExpired from time import sleep +from typing import TYPE_CHECKING import click import pytest +if TYPE_CHECKING: + from typing import Optional + @pytest.fixture def aiida_test_profile() -> str: @@ -26,7 +30,9 @@ def run_cli_command(aiida_test_profile: str): from click.testing import Result def _run_cli_command( - command: click.Command, options: list[str] = None, raises: bool = False + command: click.Command, + options: "Optional[list[str]]" = None, + raises: bool = False, ) -> Result: """Run the command and check the result. @@ -74,7 +80,7 @@ def run_and_terminate_server(aiida_test_profile: str): """ def _run_and_terminate_server( - command: str, options: list[str] = None + command: str, options: "Optional[list[str]]" = None ) -> tuple[str, str]: """Run the command and check the result. diff --git a/tests/cli/test_calc.py b/tests/cli/test_calc.py index f6b703cb..5d9f5bd0 100644 --- a/tests/cli/test_calc.py +++ b/tests/cli/test_calc.py @@ -62,7 +62,7 @@ def test_calc_all_new(run_cli_command, aiida_profile, top_dir, caplog): .count() ) - options = ["--force-yes"] + fields + options = ["--force-yes", *fields] result = run_cli_command(cmd_calc.calc, options) assert ( @@ -132,7 +132,7 @@ def test_calc(run_cli_command, aiida_profile, top_dir): .count() ) - options = ["--force-yes"] + fields + options = ["--force-yes", *fields] result = run_cli_command(cmd_calc.calc, options) assert f"Fields found for {n_structure_data} Nodes." in result.stdout, result.stdout diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py index 2dbee4d7..19d28141 100644 --- a/tests/cli/test_init.py +++ b/tests/cli/test_init.py @@ -360,8 +360,8 @@ def test_filename_aiida(run_cli_command, top_dir): "An exception happened while trying to initialize" in result.stdout ), result.stdout assert ( - "NotImplementedError('Passing a filename currently only works for a MongoDB backend'" - in result.stdout + "NotImplementedError('Passing a filename currently only works for a MongoDB " + "backend'" in result.stdout ), result.stdout diff --git a/tests/conftest.py b/tests/conftest.py index 8cd7d1fa..8533962f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,7 +55,8 @@ def aiida_profile(top_dir, setup_config) -> TestManager: manager.reset_db() profile = load_profile() - # If test locally `AIIDA_TEST_PROFILE` may not set and `test_profile` will be used + # If test locally `AIIDA_TEST_PROFILE` may not set and `test_profile` will + # be used assert profile.name in ["test_profile", "test_psql_dos"] os.environ["AIIDA_PROFILE"] = profile.name diff --git a/tests/server/routers/test_info.py b/tests/server/routers/test_info.py index c355d7c4..99cf57f6 100644 --- a/tests/server/routers/test_info.py +++ b/tests/server/routers/test_info.py @@ -79,7 +79,9 @@ def test_provider_fields(get_good_response): if not provider_fields: import warnings - warnings.warn("No provider-specific fields found for 'structures'!") + warnings.warn( + "No provider-specific fields found for 'structures'!", stacklevel=1 + ) return for field in provider_fields: diff --git a/tests/server/utils.py b/tests/server/utils.py index 6883ece0..1bdc82af 100644 --- a/tests/server/utils.py +++ b/tests/server/utils.py @@ -59,7 +59,8 @@ def __init__( if re.match(r"v[0-9](.[0-9]){0,2}", version) is None: warnings.warn( f"Invalid version passed to client: '{version}'. " - f"Will use the default: '/v{__api_version__.split('.')[0]}'" + f"Will use the default: '/v{__api_version__.split('.')[0]}'", + stacklevel=1, ) version = f"/v{__api_version__.split('.')[0]}" self.version = version @@ -76,10 +77,10 @@ def request( params: "Optional[httpx._types.QueryParamTypes]" = None, headers: "Optional[httpx._types.HeaderTypes]" = None, cookies: "Optional[httpx._types.CookieTypes]" = None, - auth: "Union[httpx._types.AuthTypes, httpx._client.UseClientDefault]" = USE_CLIENT_DEFAULT, + auth: "Union[httpx._types.AuthTypes, httpx._client.UseClientDefault]" = USE_CLIENT_DEFAULT, # noqa: E501 follow_redirects: "Optional[bool]" = None, allow_redirects: "Optional[bool]" = None, - timeout: "Union[httpx._types.TimeoutTypes, httpx._client.UseClientDefault]" = USE_CLIENT_DEFAULT, + timeout: "Union[httpx._types.TimeoutTypes, httpx._client.UseClientDefault]" = USE_CLIENT_DEFAULT, # noqa: E501 extensions: "Optional[Dict[str, Any]]" = None, ) -> Response: if ( From f6830483c37f8e41fd016a9c1b80fa962ae97356 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 9 Nov 2023 11:46:08 +0100 Subject: [PATCH 5/7] Add pyupgrade hook Import annotations from __future__ in tests.server.utils in order to remove some noqa line-too-long statements. --- .pre-commit-config.yaml | 6 +++++ tests/server/utils.py | 60 ++++++++++++++++++++++------------------- 2 files changed, 39 insertions(+), 27 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 753f5741..89d3d616 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -20,6 +20,12 @@ repos: - --offset=0 exclude: ^.github/dependabot.yml$ +- repo: https://github.com/asottile/pyupgrade + rev: v3.15.0 + hooks: + - id: pyupgrade + args: [--py39-plus] + - repo: https://github.com/ambv/black rev: 23.11.0 hooks: diff --git a/tests/server/utils.py b/tests/server/utils.py index 1bdc82af..0a75cb0c 100644 --- a/tests/server/utils.py +++ b/tests/server/utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import json import re import warnings @@ -12,9 +14,9 @@ from optimade.models import ResponseMeta from pydantic import BaseModel -if TYPE_CHECKING: +if TYPE_CHECKING: # pragma: no cover from collections.abc import Iterable - from typing import Any, Dict, Optional, Type, Union + from typing import Any import httpx from starlette import testclient, types @@ -30,13 +32,13 @@ class OptimadeTestClient(TestClient): def __init__( self, - app: "types.ASGIApp", + app: types.ASGIApp, base_url: str = "http://example.org", raise_server_exceptions: bool = True, root_path: str = "", backend: str = "asyncio", - backend_options: "Optional[Dict[str, Any]]" = None, - cookies: "Optional[httpx._types.CookieTypes]" = None, + backend_options: dict[str, Any] | None = None, + cookies: httpx._types.CookieTypes | None = None, version: str = "", ) -> None: optional_kwargs = {"cookies": cookies} @@ -68,20 +70,24 @@ def __init__( def request( self, method: str, - url: "httpx._types.URLTypes", + url: httpx._types.URLTypes, *, - content: "Optional[httpx._types.RequestContent]" = None, - data: "Optional[testclient._RequestData]" = None, - files: "Optional[httpx._types.RequestFiles]" = None, - json: "Any" = None, - params: "Optional[httpx._types.QueryParamTypes]" = None, - headers: "Optional[httpx._types.HeaderTypes]" = None, - cookies: "Optional[httpx._types.CookieTypes]" = None, - auth: "Union[httpx._types.AuthTypes, httpx._client.UseClientDefault]" = USE_CLIENT_DEFAULT, # noqa: E501 - follow_redirects: "Optional[bool]" = None, - allow_redirects: "Optional[bool]" = None, - timeout: "Union[httpx._types.TimeoutTypes, httpx._client.UseClientDefault]" = USE_CLIENT_DEFAULT, # noqa: E501 - extensions: "Optional[Dict[str, Any]]" = None, + content: httpx._types.RequestContent | None = None, + data: testclient._RequestData | None = None, + files: httpx._types.RequestFiles | None = None, + json: Any = None, + params: httpx._types.QueryParamTypes | None = None, + headers: httpx._types.HeaderTypes | None = None, + cookies: httpx._types.CookieTypes | None = None, + auth: ( + httpx._types.AuthTypes | httpx._client.UseClientDefault + ) = USE_CLIENT_DEFAULT, + follow_redirects: bool | None = None, + allow_redirects: bool | None = None, + timeout: ( + httpx._types.TimeoutTypes | httpx._client.UseClientDefault + ) = USE_CLIENT_DEFAULT, + extensions: dict[str, Any] | None = None, ) -> Response: if ( re.match(r"/?v[0-9](.[0-9]){0,2}/", str(url)) is None @@ -111,11 +117,11 @@ def request( class EndpointTests: """Base class for common tests of endpoints""" - request_str: "Optional[str]" = None - response_cls: "Optional[Type[BaseModel]]" = None + request_str: str | None = None + response_cls: type[BaseModel] | None = None - response: "Optional[Response]" = None - json_response: "Optional[Dict[str, Any]]" = None + response: Response | None = None + json_response: dict[str, Any] | None = None @pytest.fixture(autouse=True) def get_response(self, client): @@ -127,7 +133,7 @@ def get_response(self, client): self.json_response = None @staticmethod - def check_keys(keys: list, response_subset: "Iterable"): + def check_keys(keys: list, response_subset: Iterable): """Utility function to help validate dict keys""" for key in keys: assert ( @@ -167,7 +173,7 @@ def client_factory(): """Return TestClient for OPTIMADE server""" def inner( - version: "Optional[str]" = None, raise_server_exceptions: bool = True + version: str | None = None, raise_server_exceptions: bool = True ) -> OptimadeTestClient: from aiida_optimade.main import APP @@ -190,10 +196,10 @@ def inner( class NoJsonEndpointTests: """A simplified mixin class for tests on non-JSON endpoints.""" - request_str: "Optional[str]" = None - response_cls: "Optional[Type[BaseModel]]" = None + request_str: str | None = None + response_cls: type[BaseModel] | None = None - response: "Optional[Response]" = None + response: Response | None = None @pytest.fixture(autouse=True) def get_response(self, client: OptimadeTestClient): From 36609c7935a701525ef93b7ea8e1d354c2ec9a19 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 9 Nov 2023 11:50:20 +0100 Subject: [PATCH 6/7] Add bandit hook --- .pre-commit-config.yaml | 17 +++++++++++++++++ tasks.py | 3 ++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 89d3d616..3e318512 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,5 +1,10 @@ --- +# To install the git pre-commit hook run: +# pre-commit install repos: +# pre-commit-hooks supplies a multitude of small hooks +# To get an overview of them all as well as the ones used here, please see +# https://github.com/pre-commit/pre-commit-hooks#hooks-available - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: @@ -26,6 +31,8 @@ repos: - id: pyupgrade args: [--py39-plus] +# Black is a code style and formatter +# It works on files in-place - repo: https://github.com/ambv/black rev: 23.11.0 hooks: @@ -84,6 +91,16 @@ repos: - --show-fixes - --no-unsafe-fixes +# Bandit is a security linter +# More information can be found in its documentation: +# https://bandit.readthedocs.io/en/latest/ +- repo: https://github.com/PyCQA/bandit + rev: 1.7.5 + hooks: + - id: bandit + args: [-r] + exclude: ^tests/.*$ + - repo: local hooks: - id: codecov-validator diff --git a/tasks.py b/tasks.py index 00676de4..48e4d946 100644 --- a/tasks.py +++ b/tasks.py @@ -79,7 +79,8 @@ def optimade_req(_, ver=""): optimade_init = requests.get( "https://raw.githubusercontent.com/Materials-Consortia/optimade-python-tools" - f"/v{ver}/optimade/__init__.py" + f"/v{ver}/optimade/__init__.py", + timeout=10, ) if optimade_init.status_code != 200: raise RuntimeError(f"{ver} does not seem to be published on GitHub") From 24bb6d395ba5e7f1b09ed00f8da8c352a091d328 Mon Sep 17 00:00:00 2001 From: Casper Welzel Andersen Date: Thu, 9 Nov 2023 15:12:05 +0100 Subject: [PATCH 7/7] Add and comply with mypy --- .github/mongo/load_data.py | 2 +- .pre-commit-config.yaml | 17 ++++ aiida_optimade/cli/cmd_aiida_optimade.py | 2 +- aiida_optimade/cli/cmd_calc.py | 10 +-- aiida_optimade/cli/cmd_init.py | 64 ++++++++------- aiida_optimade/cli/cmd_run.py | 4 +- aiida_optimade/common/logger.py | 1 + aiida_optimade/entry_collections.py | 77 ++++++++++--------- aiida_optimade/mappers/entries.py | 5 +- aiida_optimade/mappers/structures.py | 6 +- aiida_optimade/routers/structures.py | 8 +- aiida_optimade/routers/utils.py | 37 +++++---- aiida_optimade/translators/cifs.py | 37 +++++---- aiida_optimade/translators/entities.py | 17 ++-- aiida_optimade/translators/structures.py | 31 ++++---- aiida_optimade/translators/utils.py | 34 ++++---- aiida_optimade/utils.py | 7 +- pyproject.toml | 10 +++ tests/cli/conftest.py | 51 +++++++++--- tests/cli/test_calc.py | 33 +++++++- tests/cli/test_init.py | 61 ++++++++++++--- tests/cli/test_run.py | 63 +++++++++------ tests/conftest.py | 24 ++++-- tests/server/__init__.py | 0 tests/server/conftest.py | 76 +++++++++++++----- tests/server/query_params/test_filter.py | 70 +++++++++-------- tests/server/query_params/test_page_limit.py | 16 +++- tests/server/query_params/test_page_offset.py | 12 ++- .../query_params/test_response_fields.py | 14 +++- tests/server/query_params/test_sort.py | 29 +++++-- tests/server/routers/test_info.py | 34 ++++++-- tests/server/routers/test_links.py | 10 ++- tests/server/routers/test_references.py | 31 ++++++-- tests/server/routers/test_response.py | 52 +++++++++---- tests/server/routers/test_structures.py | 64 +++++++++++---- tests/server/routers/test_versions.py | 13 +++- tests/server/test_entry_collections.py | 21 +++-- tests/server/test_middleware.py | 21 +++-- tests/server/test_optimade_validation.py | 14 +++- tests/server/test_server_misc.py | 10 ++- tests/transformers/test_aiida.py | 62 +++++++++------ 41 files changed, 790 insertions(+), 360 deletions(-) delete mode 100644 tests/server/__init__.py diff --git a/.github/mongo/load_data.py b/.github/mongo/load_data.py index 8e86d270..4b3d5444 100755 --- a/.github/mongo/load_data.py +++ b/.github/mongo/load_data.py @@ -16,6 +16,6 @@ collection.insert_many(data, ordered=False) except Exception as exc: print("An error occurred!") - sys.exit(exc) + sys.exit(str(exc)) else: print("Done!") diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3e318512..b7813b8e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -101,6 +101,23 @@ repos: args: [-r] exclude: ^tests/.*$ +# mypy is a static typing linter +# The main code repository can be found at: +# https://github.com/python/mypy +# The project's documentation can be found at: +# https://mypy.readthedocs.io/en/stable/index.html +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.6.1 + hooks: + - id: mypy + args: [--explicit-package-bases] + additional_dependencies: + - types-tqdm + - types-simplejson + - types-requests + - types-invoke + - pydantic<2 + - repo: local hooks: - id: codecov-validator diff --git a/aiida_optimade/cli/cmd_aiida_optimade.py b/aiida_optimade/cli/cmd_aiida_optimade.py index 66ce0f70..4c352343 100644 --- a/aiida_optimade/cli/cmd_aiida_optimade.py +++ b/aiida_optimade/cli/cmd_aiida_optimade.py @@ -49,7 +49,7 @@ def cli(ctx: "VerdiContext", profile: "Profile", dev: bool): # pragma: no cover # Set config if ( not os.getenv("OPTIMADE_CONFIG_FILE") - or not Path(os.getenv("OPTIMADE_CONFIG_FILE")).exists() + or not Path(os.getenv("OPTIMADE_CONFIG_FILE", "")).exists() ): os.environ["OPTIMADE_CONFIG_FILE"] = str( Path(__file__).parent.parent.joinpath("config.json").resolve() diff --git a/aiida_optimade/cli/cmd_calc.py b/aiida_optimade/cli/cmd_calc.py index ecb0c529..19aebcc2 100644 --- a/aiida_optimade/cli/cmd_calc.py +++ b/aiida_optimade/cli/cmd_calc.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import TYPE_CHECKING import click @@ -7,8 +9,6 @@ from aiida_optimade.common.logger import LOGGER, disable_logging if TYPE_CHECKING: # pragma: no cover - from typing import Tuple - from aiida.common.extendeddicts import AttributeDict @@ -39,7 +39,7 @@ help="Suppress informational output.", ) @click.pass_obj -def calc(obj: "AttributeDict", fields: "Tuple[str]", force_yes: bool, silent: bool): +def calc(obj: AttributeDict, fields: tuple[str], force_yes: bool, silent: bool): """Calculate OPTIMADE fields in the AiiDA database.""" from aiida import load_profile from aiida.cmdline.utils import echo @@ -49,7 +49,7 @@ def calc(obj: "AttributeDict", fields: "Tuple[str]", force_yes: bool, silent: bo echo.CMDLINE_LOGGER.setLevel("INFO") try: - profile: str = obj.profile.name + profile: str | None = obj.profile.name except AttributeError: profile = None profile = load_profile(profile).name @@ -102,7 +102,7 @@ def calc(obj: "AttributeDict", fields: "Tuple[str]", force_yes: bool, silent: bo "This may take several minutes!" ) - all_calculated_nodes = STRUCTURES._find_all(**query_kwargs) + all_calculated_nodes: list | tqdm = STRUCTURES._find_all(**query_kwargs) if not silent: all_calculated_nodes = tqdm( diff --git a/aiida_optimade/cli/cmd_init.py b/aiida_optimade/cli/cmd_init.py index e09ccb28..2ae70519 100644 --- a/aiida_optimade/cli/cmd_init.py +++ b/aiida_optimade/cli/cmd_init.py @@ -1,3 +1,5 @@ +from __future__ import annotations + from pathlib import Path from typing import TYPE_CHECKING @@ -8,8 +10,8 @@ from aiida_optimade.common.logger import LOGGER, disable_logging if TYPE_CHECKING: # pragma: no cover - from collections.abc import Generator, Iterator - from typing import IO, List, Optional, Union + from collections.abc import Generator + from typing import IO from aiida.common.extendeddicts import AttributeDict @@ -48,7 +50,7 @@ help="Filename to load as database (currently only usable for MongoDB).", ) @click.pass_obj -def init(obj: "AttributeDict", force: bool, silent: bool, mongo: bool, filename: str): +def init(obj: AttributeDict, force: bool, silent: bool, mongo: bool, filename: str): """Initialize an AiiDA database to be served with AiiDA-OPTIMADE.""" from aiida import load_profile from aiida.cmdline.utils import echo @@ -57,13 +59,13 @@ def init(obj: "AttributeDict", force: bool, silent: bool, mongo: bool, filename: # Here we use INFO loglevel for the operations echo.CMDLINE_LOGGER.setLevel("INFO") - filename: Path = Path(filename) if filename else filename + filename_path = Path(filename) - if mongo and filename: - profile = f"MongoDB JSON file {filename.name}" + if mongo and filename_path: + profile: str | None = f"MongoDB JSON file {filename_path.name}" else: try: - profile: str = obj.profile.name + profile = obj.profile.name except AttributeError: profile = None profile = load_profile(profile).name @@ -106,7 +108,7 @@ def init(obj: "AttributeDict", force: bool, silent: bool, mongo: bool, filename: ) echo.echo_warning("This may take several seconds!") - all_calculated_nodes = STRUCTURES._find_all(**query_kwargs) + all_calculated_nodes: list | tqdm = STRUCTURES._find_all(**query_kwargs) if not silent: all_calculated_nodes = tqdm( @@ -130,7 +132,7 @@ def init(obj: "AttributeDict", force: bool, silent: bool, mongo: bool, filename: echo.echo_info(f"Initializing {profile}.") echo.echo_warning("This may take several minutes!") - if filename: + if filename_path: if not mongo: LOGGER.debug( "Passed filename (%s) with AiiDA backend (mongo=%s)", @@ -143,7 +145,7 @@ def init(obj: "AttributeDict", force: bool, silent: bool, mongo: bool, filename: import bson.json_util - updated_pks = range(len(STRUCTURES_MONGO)) + updated_pks: range | list[int] = range(len(STRUCTURES_MONGO)) chunk_size = 2**24 # 16 MB if updated_pks and not silent: @@ -153,39 +155,43 @@ def init(obj: "AttributeDict", force: bool, silent: bool, mongo: bool, filename: "consider using --force to first drop the collection, if possible." ) - with open(filename) as handle: + with open(filename_path) as handle: + if TYPE_CHECKING: # pragma: no cover + all_chunks: Generator[str | bytes, None, None] | tqdm + if silent: all_chunks = read_chunks(handle, chunk_size=chunk_size) else: all_chunks = tqdm( read_chunks(handle, chunk_size=chunk_size), - total=(filename.stat().st_size // chunk_size) - + (1 if filename.stat().st_size % chunk_size else 0), - desc=f"Storing entries in {filename.name}", + total=(filename_path.stat().st_size // chunk_size) + + (1 if filename_path.stat().st_size % chunk_size else 0), + desc=f"Storing entries in {filename_path.name}", ) if updated_pks: - for data in get_documents(all_chunks): + for data in get_documents(all_chunks): # type: ignore[arg-type] for doc in bson.json_util.loads(data): STRUCTURES_MONGO.collection.replace_one( {"id": doc["id"]}, doc, upsert=True ) else: - for data in get_documents(all_chunks): + for data in get_documents(all_chunks): # type: ignore[arg-type] STRUCTURES_MONGO.collection.insert_many( bson.json_util.loads(data) ) updated_pks = range(len(STRUCTURES_MONGO) - len(updated_pks)) else: + entries_list: list[list[int]] = [] if mongo: CONFIG.database_backend = SupportedBackend.MONGODB - entries = {_[0] for _ in STRUCTURES._find_all(project="id")} - entries -= { + entries_set = {_[0] for _ in STRUCTURES._find_all(project="id")} + entries_set -= { int(_["id"]) for _ in STRUCTURES_MONGO.collection.find( filter={}, projection=["id"] ) } - entries = [[_] for _ in entries] + entries_list = [[_] for _ in entries_set] STRUCTURES._extras_fields = { STRUCTURES.resource_mapper.get_backend_field(_)[ @@ -198,7 +204,7 @@ def init(obj: "AttributeDict", force: bool, silent: bool, mongo: bool, filename: } updated_pks = STRUCTURES._check_and_calculate_entities( cli=not silent, - entries=entries if mongo else None, + entries=entries_list if mongo else None, ) except Exception as exc: # noqa: BLE001 import traceback @@ -226,8 +232,8 @@ def init(obj: "AttributeDict", force: bool, silent: bool, mongo: bool, filename: def read_chunks( - file_object: "IO", chunk_size: "Optional[int]" = None -) -> "Generator[Union[str, bytes], None, None]": + file_object: IO, chunk_size: int | None = None +) -> Generator[str | bytes, None, None]: """Generator to read a file piece by piece Parameters: @@ -248,13 +254,15 @@ def read_chunks( def get_documents( - chunk_iterator: "Union[Generator[str, None, None], Iterator[str]]", -) -> "Generator[List[dict], None, None]": + chunk_iterator: Generator[str | bytes, None, None], +) -> Generator[str, None, None]: """Generator to return MongoDB documents from file""" rest_chunk = "" for raw_chunk in chunk_iterator: - full_raw_chunk = rest_chunk + raw_chunk + full_raw_chunk: str = rest_chunk + ( + raw_chunk.decode("utf-8") if isinstance(raw_chunk, bytes) else raw_chunk + ) rest_chunk = "" curly_start_count = full_raw_chunk.count("{") @@ -266,9 +274,9 @@ def get_documents( chunk = full_raw_chunk while curly_end_count - curly_start_count != 0: - chunk = chunk.split("{") - rest_chunk = "{" + f"{chunk[-1]}{rest_chunk}" - chunk = "{".join(chunk[:-1]) + split_chunk = chunk.split("{") + rest_chunk = "{" + f"{split_chunk[-1]}{rest_chunk}" + chunk = "{".join(split_chunk[:-1]) curly_start_count = chunk.count("{") curly_end_count = chunk.count("}") diff --git a/aiida_optimade/cli/cmd_run.py b/aiida_optimade/cli/cmd_run.py index fd9a52d0..61543290 100644 --- a/aiida_optimade/cli/cmd_run.py +++ b/aiida_optimade/cli/cmd_run.py @@ -6,6 +6,8 @@ from aiida_optimade.cli.options import LOGGING_LEVELS if TYPE_CHECKING: # pragma: no cover + from typing import Optional + from aiida.common.extendeddicts import AttributeDict @@ -81,7 +83,7 @@ def run( from aiida import load_profile try: - profile: str = obj.profile.name + profile: "Optional[str]" = obj.profile.name except AttributeError: profile = None profile_name: str = load_profile(profile).name diff --git a/aiida_optimade/common/logger.py b/aiida_optimade/common/logger.py index 93952ff2..2c4e2a15 100644 --- a/aiida_optimade/common/logger.py +++ b/aiida_optimade/common/logger.py @@ -1,5 +1,6 @@ """Logging to both file and widget""" import logging +import logging.handlers import os import sys from contextlib import contextmanager diff --git a/aiida_optimade/entry_collections.py b/aiida_optimade/entry_collections.py index dbdcea83..e78ff5b7 100644 --- a/aiida_optimade/entry_collections.py +++ b/aiida_optimade/entry_collections.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import warnings from typing import TYPE_CHECKING, ClassVar @@ -18,7 +20,7 @@ from aiida_optimade.utils import retrieve_queryable_properties if TYPE_CHECKING: # pragma: no cover - from typing import Any, Optional, Union + from typing import Any class AiidaCollection(EntryCollection): @@ -34,10 +36,10 @@ class AiidaCollection(EntryCollection): def __init__( self, - entities: "Union[str, list[str]]", - group: "Optional[str]", - resource_cls: EntryResource, - resource_mapper: ResourceMapper, + entities: str | list[str], + group: str | None, + resource_cls: type[EntryResource], + resource_mapper: type[ResourceMapper], ): super().__init__( resource_cls=resource_cls, @@ -49,18 +51,18 @@ def __init__( self.group = group # "Cache" - self._data_available: int = None - self._data_returned: int = None - self._extras_fields: set[str] = None - self._latest_filter: dict[str, Any] = None - self._count: dict[str, Any] = None + self._data_available: int | None = None + self._data_returned: int | None = None + self._extras_fields: set[str] = set() + self._latest_filter: dict[str, Any] | None = None + self._count: dict[str, Any] | None = None self._checked_extras_filter_fields: set = set() - self._all_fields: set[str] = None + self._all_fields: set[str] | None = None @property def all_fields(self) -> set[str]: - if not self._all_fields: + if self._all_fields is None: self._all_fields = super().all_fields return self._all_fields @@ -107,12 +109,12 @@ def set_data_returned(self, **criteria): def _clear_cache(self) -> None: """Clear in-memory attributes cache""" - self._data_available: int = None - self._data_returned: int = None - self._extras_fields: set = None - self._latest_filter: dict = None - self._count: dict = None - self._checked_extras_filter_fields: set = set() + self._data_available = None + self._data_returned = None + self._extras_fields = set() + self._latest_filter = None + self._count = None + self._checked_extras_filter_fields = set() def __len__(self) -> int: return self.data_available @@ -177,9 +179,9 @@ def count(self, **kwargs) -> int: return self._count.get("count", 0) def find( - self, params: "Union[EntryListingQueryParams, SingleEntryQueryParams]" + self, params: EntryListingQueryParams | SingleEntryQueryParams ) -> tuple[ - "Union[list[EntryResource], EntryResource, None], int, bool, set[str], set[str]" + list[EntryResource] | EntryResource | None, int, bool, set[str], set[str] ]: self.set_data_available() @@ -220,7 +222,7 @@ def find( "found", ) - results = results[0] if results else None + results = results[0] if results else None # type: ignore[assignment] include_fields = ( response_fields - self.resource_mapper.TOP_LEVEL_NON_ATTRIBUTES_FIELDS @@ -270,8 +272,8 @@ def find( ) def _run_db_query( - self, criteria: dict[str, "Any"], single_entry: bool = False - ) -> tuple[list[dict[str, "Any"]], bool]: + self, criteria: dict[str, Any], single_entry: bool = False + ) -> tuple[list[dict[str, Any]], bool]: """Run the query on the backend and collect the results. Arguments: @@ -300,7 +302,7 @@ def _run_db_query( @staticmethod def _prepare_query( - node_types: list[str], group: "Optional[str]" = None, **kwargs + node_types: list[str], group: str | None = None, **kwargs ) -> QueryBuilder: """Workhorse function to prepare an AiiDA QueryBuilder query""" for key in kwargs: @@ -347,8 +349,8 @@ def _perform_count(self, **kwargs) -> int: return res def handle_query_params( - self, params: "Union[EntryListingQueryParams, SingleEntryQueryParams]" - ) -> dict[str, "Any"]: + self, params: EntryListingQueryParams | SingleEntryQueryParams + ) -> dict[str, Any]: """Parse and interpret the backend-agnostic query parameter models into a dictionary that can be used by AiiDA's QueryBuilder. @@ -434,7 +436,7 @@ def parse_sort_params(self, sort_params: str) -> list[dict[str, dict[str, str]]] ) return sort_spec - def _find_extras_fields(self, filters: "Union[dict, list]") -> None: + def _find_extras_fields(self, filters: dict | list) -> None: """Collect all properties to be found in AiiDA Node extras. Parameters: @@ -444,7 +446,9 @@ def _find_extras_fields(self, filters: "Union[dict, list]") -> None: """ from copy import deepcopy - def __filter_fields_util(_filters: "Union[dict, list]") -> "Union[dict, list]": + def __filter_fields_util( + _filters: dict[str, Any] | list + ) -> dict[str, Any] | list: if isinstance(_filters, dict): res = {} for key, value in _filters.items(): @@ -458,24 +462,23 @@ def __filter_fields_util(_filters: "Union[dict, list]") -> "Union[dict, list]": for key in _filters if key.startswith(self.resource_mapper.PROJECT_PREFIX) } + return res elif isinstance(_filters, list): - res = [ + return [ __filter_fields_util(item) if isinstance(item, (dict, list)) else item for item in _filters ] - else: - raise NotImplementedError( - "_find_extras_fields can only handle dict and list objects." - ) - return res + raise NotImplementedError( + "_find_extras_fields can only handle dict and list objects." + ) self._extras_fields = set() __filter_fields_util(deepcopy(filters)) def _check_and_calculate_entities( - self, cli: bool = False, entries: "Optional[list[list[int]]]" = None + self, cli: bool = False, entries: list[list[int]] | None = None ) -> list[int]: """Check all entities have OPTIMADE extras, else calculate them @@ -492,7 +495,7 @@ def _check_and_calculate_entities( """ - def _update_entities(entities: list[list["Any"]], fields: list[str]): + def _update_entities(entities: list[list[Any]], fields: list[str]): """Utility function to update entities within this method""" optimade_fields = [ self.resource_mapper.get_optimade_field(_) for _ in fields @@ -548,7 +551,7 @@ def _update_entities(entities: list[list["Any"]], fields: list[str]): with warnings.catch_warnings(): warnings.simplefilter("ignore") _update_entities( - tqdm(entities, desc="Calculating fields", leave=False), + tqdm(entities, desc="Calculating fields", leave=False), # type: ignore[arg-type] fields, ) else: diff --git a/aiida_optimade/mappers/entries.py b/aiida_optimade/mappers/entries.py index e9af2572..af6c1b8b 100644 --- a/aiida_optimade/mappers/entries.py +++ b/aiida_optimade/mappers/entries.py @@ -13,7 +13,7 @@ class ResourceMapper(OptimadeResourceMapper): PROJECT_PREFIX: str = "extras.optimade." - TRANSLATORS: dict[str, AiidaEntityTranslator] + TRANSLATORS: ClassVar[dict[str, type[AiidaEntityTranslator]]] REQUIRED_ATTRIBUTES: ClassVar[set[str]] = set() TOP_LEVEL_NON_ATTRIBUTES_FIELDS: ClassVar[set[str]] = { "id", @@ -83,7 +83,7 @@ def build_attributes( retrieved_attributes: dict, entry_pk: int, node_type: str, - missing_attributes: "Optional[dict]" = None, + missing_attributes: "Optional[set]" = None, ) -> dict: """Build attributes dictionary for OPTIMADE structure resource @@ -96,3 +96,4 @@ def build_attributes( :param node_type: The AiiDA Node's type :type node_type: str """ + raise NotImplementedError("Should be implemented in a sub-class.") diff --git a/aiida_optimade/mappers/structures.py b/aiida_optimade/mappers/structures.py index 7119fad3..bfb4c402 100644 --- a/aiida_optimade/mappers/structures.py +++ b/aiida_optimade/mappers/structures.py @@ -7,7 +7,6 @@ from aiida_optimade.mappers.entries import ResourceMapper from aiida_optimade.models import StructureResource, StructureResourceAttributes from aiida_optimade.translators import ( - AiidaEntityTranslator, CifDataTranslator, StructureDataTranslator, hex_to_floats, @@ -16,11 +15,13 @@ if TYPE_CHECKING: # pragma: no cover from typing import Optional + from aiida_optimade.translators import AiidaEntityTranslator + class StructureMapper(ResourceMapper): """Map 'structure' resources from OPTIMADE to AiiDA""" - TRANSLATORS: ClassVar[dict[str, AiidaEntityTranslator]] = { + TRANSLATORS: ClassVar[dict[str, type["AiidaEntityTranslator"]]] = { "data.core.cif.CifData.": CifDataTranslator, "data.core.structure.StructureData.": StructureDataTranslator, } @@ -81,7 +82,6 @@ def build_attributes( except AttributeError as exc: if CONFIG.database_backend != SupportedBackend.MONGODB: if attribute in cls.REQUIRED_ATTRIBUTES: - translator = None raise NotImplementedError( f"Parsing required attribute {attribute!r} from " f"{translator.__class__.__name__} has not yet been " diff --git a/aiida_optimade/routers/structures.py b/aiida_optimade/routers/structures.py index 7b89468a..5114f54d 100644 --- a/aiida_optimade/routers/structures.py +++ b/aiida_optimade/routers/structures.py @@ -61,9 +61,11 @@ def get_single_structure( request: Request, entry_id: int, params: SingleEntryQueryParams = Depends() ): return get_single_entry( - collection=STRUCTURES_MONGO - if CONFIG.database_backend == SupportedBackend.MONGODB - else STRUCTURES, + collection=( + STRUCTURES_MONGO + if CONFIG.database_backend == SupportedBackend.MONGODB + else STRUCTURES + ), entry_id=entry_id, response=StructureResponseOne, request=request, diff --git a/aiida_optimade/routers/utils.py b/aiida_optimade/routers/utils.py index 5ce1139b..d48cf270 100644 --- a/aiida_optimade/routers/utils.py +++ b/aiida_optimade/routers/utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import functools import urllib.parse from typing import TYPE_CHECKING @@ -12,28 +14,29 @@ from aiida_optimade.entry_collections import AiidaCollection if TYPE_CHECKING: # pragma: no cover - from typing import Type, Union + from typing import Any def handle_pagination( request: Request, more_data_available: bool, nresults: int -) -> dict: +) -> dict[str, Any]: """Handle pagination for request with number of results equal nresults""" from optimade.server.routers.utils import get_base_url - pagination = {} + pagination: dict[str, Any] = {} # "prev" parse_result = urllib.parse.urlparse(str(request.url)) base_url = get_base_url(parse_result) query = urllib.parse.parse_qs(parse_result.query) - query["page_offset"] = int(query.get("page_offset", ["0"])[0]) - int( + page_offset = int(query.get("page_offset", ["0"])[0]) - int( query.get("page_limit", [CONFIG.page_limit])[0] ) + query["page_offset"] = [str(page_offset)] urlencoded_prev = None - if query["page_offset"] > 0: + if page_offset > 0: # type: ignore[operator] urlencoded_prev = urllib.parse.urlencode(query, doseq=True) - elif query["page_offset"] == 0 or abs(query["page_offset"]) < int( + elif page_offset == 0 or abs(page_offset) < int( query.get("page_limit", [CONFIG.page_limit])[0] ): prev_query = query.copy() @@ -45,11 +48,13 @@ def handle_pagination( # "next" if more_data_available: - query["page_offset"] = ( - int(query.get("page_offset", 0)) - + nresults - + int(query.get("page_limit", [CONFIG.page_limit])[0]) - ) + query["page_offset"] = [ + str( + page_offset + + nresults + + int(query.get("page_limit", [CONFIG.page_limit])[0]) + ) + ] urlencoded_next = urllib.parse.urlencode(query, doseq=True) pagination["next"] = f"{base_url}{parse_result.path}" if urlencoded_next: @@ -61,8 +66,8 @@ def handle_pagination( def get_entries( - collection: "Union[AiidaCollection, MongoCollection]", - response: "Type[EntryResponseMany]", + collection: AiidaCollection | MongoCollection, + response: type[EntryResponseMany], request: Request, params: EntryListingQueryParams, ) -> EntryResponseMany: @@ -112,9 +117,9 @@ def get_entries( def get_single_entry( - collection: "Union[AiidaCollection, MongoCollection]", - entry_id: str, - response: "Type[EntryResponseOne]", + collection: AiidaCollection | MongoCollection, + entry_id: int, + response: type[EntryResponseOne], request: Request, params: SingleEntryQueryParams, ) -> EntryResponseOne: diff --git a/aiida_optimade/translators/cifs.py b/aiida_optimade/translators/cifs.py index 48fbc1a9..c0e75a2e 100644 --- a/aiida_optimade/translators/cifs.py +++ b/aiida_optimade/translators/cifs.py @@ -1,10 +1,16 @@ -from typing import Union +from __future__ import annotations + +from typing import TYPE_CHECKING from aiida.orm.nodes.data.cif import CifData from aiida.orm.nodes.data.structure import StructureData +from aiida_optimade.common.exceptions import AiidaEntityNotFound from aiida_optimade.translators.structures import StructureDataTranslator +if TYPE_CHECKING: # pragma: no cover + from typing import Any + def _get_aiida_structure_pymatgen_inline(cif, **kwargs) -> StructureData: """Copy of similar named function in AiiDA-Core. @@ -77,13 +83,13 @@ class CifDataTranslator(StructureDataTranslator): AIIDA_ENTITY = CifData - def __init__(self, pk: str): + def __init__(self, pk: int): super().__init__(pk) - self.__kinds = None - self.__sites = None - self.__pbc = None - self.__cell = None + self.__kinds: list[dict[str, Any]] | None = None + self.__sites: list[dict[str, Any]] | None = None + self.__pbc: list[int] | None = None + self.__cell: list[list[float]] | None = None @property def _node(self) -> StructureData: @@ -93,37 +99,42 @@ def _node(self) -> StructureData: if isinstance(self.__node, StructureData): return self.__node + if self.__node is None: + raise AiidaEntityNotFound( + f"Could not find {self.AIIDA_ENTITY} with PK {self._pk}." + ) + extras = self.__node.extras.copy() self.__node = _get_aiida_structure_pymatgen_inline(cif=self.__node) self.__node.set_extra_many(extras) return self.__node @_node.setter - def _node(self, value: Union[None, CifData, StructureData]): + def _node(self, value: None | CifData | StructureData): if self._node_loaded: del self.__node self.__node = value @property - def _kinds(self) -> list: + def _kinds(self) -> list[dict[str, Any]]: if not self.__kinds or self.__kinds is None: self.__kinds = [_.get_raw() for _ in self._node.kinds] return self.__kinds @property - def _sites(self) -> list: + def _sites(self) -> list[dict[str, Any]]: if not self.__sites or self.__sites is None: self.__sites = [_.get_raw() for _ in self._node.sites] return self.__sites @property - def _pbc(self) -> list: + def _pbc(self) -> list[int]: if not self.__pbc: self.__pbc = [int(_) for _ in self._node.pbc] return self.__pbc @property - def _cell(self) -> list: + def _cell(self) -> list[list[float]]: if not self.__cell: - self.__cell = self._node.cell.copy() - return self.__cell + self.__cell = self._node.cell + return self.__cell # type: ignore[return-value] diff --git a/aiida_optimade/translators/entities.py b/aiida_optimade/translators/entities.py index 4bee8f67..fa1866a7 100644 --- a/aiida_optimade/translators/entities.py +++ b/aiida_optimade/translators/entities.py @@ -1,4 +1,6 @@ -from typing import Any, Union +from __future__ import annotations + +from typing import TYPE_CHECKING from aiida import orm from aiida.orm.nodes import Node @@ -6,6 +8,9 @@ from aiida_optimade.common import LOGGER, AiidaEntityNotFound +if TYPE_CHECKING: # pragma: no cover + from typing import Any + class AiidaEntityTranslator: """Create OPTIMADE entry attributes from an AiiDA Entity Node - Base class @@ -19,12 +24,10 @@ class AiidaEntityTranslator: def __init__(self, pk: int): self._pk = pk - self.new_attributes = {} + self.new_attributes: dict[str, Any] = {} self.__node = None - def _get_unique_node_property( - self, project: Union[list[str], str] - ) -> Union[Node, Any]: + def _get_unique_node_property(self, project: list[str] | str) -> Node | Any: query = QueryBuilder(limit=1) query.append(self.AIIDA_ENTITY, filters={"id": self._pk}, project=project) if query.count() != 1: @@ -43,7 +46,7 @@ def _node(self) -> Node: return self.__node @_node.setter - def _node(self, value: Union[None, Node]): + def _node(self, value: None | Node): if self._node_loaded: del self.__node self.__node = value @@ -52,7 +55,7 @@ def _node(self, value: Union[None, Node]): def _node_loaded(self): return bool(self.__node) - def _get_optimade_extras(self) -> Union[None, dict]: + def _get_optimade_extras(self) -> None | dict: if self._node_loaded: return self._node.extras.get(self.EXTRAS_KEY, None) return self._get_unique_node_property(f"extras.{self.EXTRAS_KEY}") diff --git a/aiida_optimade/translators/structures.py b/aiida_optimade/translators/structures.py index 82629fe5..28a71785 100644 --- a/aiida_optimade/translators/structures.py +++ b/aiida_optimade/translators/structures.py @@ -1,6 +1,8 @@ +from __future__ import annotations + import itertools from math import fsum -from typing import Any, Union +from typing import TYPE_CHECKING from aiida.orm.nodes.data.structure import StructureData from optimade.models.utils import ANONYMOUS_ELEMENTS @@ -13,6 +15,9 @@ hex_to_floats, ) +if TYPE_CHECKING: # pragma: no cover + from typing import Any + class StructureDataTranslator(AiidaEntityTranslator): """Create OPTIMADE "structures" attributes from an AiiDA StructureData Node @@ -26,10 +31,10 @@ class StructureDataTranslator(AiidaEntityTranslator): AIIDA_ENTITY = StructureData # StructureData specific properties - def __init__(self, pk: str): + def __init__(self, pk: int): super().__init__(pk) - self.__properties = None + self.__properties: dict[str, Any] | None = None @property def _kinds(self) -> list: @@ -172,7 +177,7 @@ def elements_ratios(self) -> list[float]: attribute = "elements_ratios" if attribute in self.new_attributes: - return hex_to_floats(self.new_attributes[attribute]) + return hex_to_floats(self.new_attributes[attribute]) # type: ignore[return-value] ratios = self.get_symbol_weights() @@ -232,7 +237,7 @@ def chemical_formula_reduced(self) -> str: self.new_attributes[attribute] = res return res - def chemical_formula_hill(self) -> str: + def chemical_formula_hill(self) -> str | None: """The chemical formula for a structure in Hill form With element symbols followed by integer chemical proportion numbers. @@ -333,16 +338,16 @@ def lattice_vectors(self) -> list[list[float]]: attribute = "lattice_vectors" if attribute in self.new_attributes: - return hex_to_floats(self.new_attributes[attribute]) + return hex_to_floats(self.new_attributes[attribute]) # type: ignore[return-value] res = check_floating_round_errors(self._cell) # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node # and return value self.new_attributes[attribute] = floats_to_hex(res) - return res + return res # type: ignore[return-value] - def cartesian_site_positions(self) -> list[list[Union[float, None]]]: + def cartesian_site_positions(self) -> list[list[float | None]]: """Cartesian positions of each site. A site is an atom, a site potentially occupied by an atom, @@ -352,15 +357,15 @@ def cartesian_site_positions(self) -> list[list[Union[float, None]]]: attribute = "cartesian_site_positions" if attribute in self.new_attributes: - return hex_to_floats(self.new_attributes[attribute]) + return hex_to_floats(self.new_attributes[attribute]) # type: ignore[return-value] sites = [list(site["position"]) for site in self._sites] - res = check_floating_round_errors(sites) + res = check_floating_round_errors(sites) # type: ignore[arg-type] # Finally, save OPTIMADE attribute for later storage in extras for AiiDA Node # and return value self.new_attributes[attribute] = floats_to_hex(res) - return res + return res # type: ignore[return-value] def nsites(self) -> int: """An integer specifying the length of the cartesian_site_positions property.""" @@ -450,7 +455,7 @@ def species(self) -> list[dict]: self.new_attributes[attribute] = res return res - def assemblies(self) -> Union[list[dict], None]: + def assemblies(self) -> list[dict] | None: """A description of groups of sites that are statistically correlated. NOTE: Currently not supported. @@ -477,7 +482,7 @@ def structure_features(self) -> list[str]: if attribute in self.new_attributes: return self.new_attributes[attribute] - res = [] + res: list[str] = [] # Figure out if there are partial occupancies if not self.has_partial_occupancy(): diff --git a/aiida_optimade/translators/utils.py b/aiida_optimade/translators/utils.py index 5ad1fcd0..e964a173 100644 --- a/aiida_optimade/translators/utils.py +++ b/aiida_optimade/translators/utils.py @@ -1,9 +1,15 @@ -from typing import Union +from typing import TYPE_CHECKING, Union + +if TYPE_CHECKING: # pragma: no cover + from typing import TypeVar + + NonListType = TypeVar("NonListType", float, str) + RecursiveList = list[Union["RecursiveList[NonListType]", NonListType]] def check_floating_round_errors( - some_list: list[Union[list[float], float]] -) -> list[Union[list[float], float]]: + some_list: "RecursiveList[float]", +) -> "RecursiveList[float]": """Check whether there are some float rounding errors (check only for close to zero numbers) @@ -13,7 +19,7 @@ def check_floating_round_errors( might_as_well_be_zero = ( 1e-8 # This is for Å, so 1e-8 Å can by all means be considered 0 Å ) - res = [] + res: "RecursiveList[float]" = [] for item in some_list: if isinstance(item, list): @@ -22,25 +28,25 @@ def check_floating_round_errors( res.append(0.0) else: res.append(item) + return res -def floats_to_hex( - some_list: list[Union[list[float], float]] -) -> list[Union[list[str], str]]: +def floats_to_hex(some_list: "RecursiveList[float]") -> "RecursiveList[str]": """Convert floats embedded in lists to hex strings (for storing "precise" floats) :param some_list: Must be a list of either lists or float values :type some_list: list """ - res = [] + res: "RecursiveList[str]" = [] + for item in some_list: if isinstance(item, list): res.append(floats_to_hex(item)) else: item_updated = item if isinstance(item, float): - item_updated = item.hex() + item_updated = item.hex() # type: ignore[assignment] if not isinstance(item_updated, str): raise TypeError( "Wrong type passed to floats_to_hex method, must be a " @@ -48,18 +54,17 @@ def floats_to_hex( f"Item: {item!r}. Type: {type(item)}." ) res.append(item_updated) + return res -def hex_to_floats( - some_list: list[Union[list[str], str]] -) -> list[Union[list[float], float]]: +def hex_to_floats(some_list: "RecursiveList[str]") -> "RecursiveList[float]": """Convert hex strings embedded in lists (back) to floats :param some_list: Must be a list of either lists or string values :type some_list: list """ - res = [] + res: "RecursiveList[float]" = [] for item in some_list: if isinstance(item, list): @@ -68,7 +73,7 @@ def hex_to_floats( item_updated = item if isinstance(item, str): try: - item_updated = float.fromhex(item) + item_updated = float.fromhex(item) # type: ignore[assignment] except ValueError as exc: raise ValueError( f"Could not turn item ({item}) into float from hex. " @@ -81,4 +86,5 @@ def hex_to_floats( f"Item: {item!r}. Type: {type(item)}." ) res.append(item_updated) + return res diff --git a/aiida_optimade/utils.py b/aiida_optimade/utils.py index 3bca6f66..bbca8a76 100644 --- a/aiida_optimade/utils.py +++ b/aiida_optimade/utils.py @@ -1,3 +1,5 @@ +from typing import TYPE_CHECKING + from optimade.models import DataType OPEN_API_ENDPOINTS = { @@ -6,9 +8,12 @@ "openapi": "/extensions/openapi.json", } +if TYPE_CHECKING: # pragma: no cover + from collections.abc import Iterable + def retrieve_queryable_properties( - schema: dict, queryable_properties: list + schema: dict, queryable_properties: "Iterable[str]" ) -> tuple[dict, dict]: """Get all queryable properties from an OPTIMADE schema""" properties = {} diff --git a/pyproject.toml b/pyproject.toml index 0ed9d0f8..8f13e26b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,3 +23,13 @@ filterwarnings = [ "ignore:.*Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated.*:DeprecationWarning", "ignore:Parsing optional attribute.*:UserWarning", ] + +[tool.mypy] +python_version = "3.9" +ignore_missing_imports = true +scripts_are_modules = true +warn_unused_configs = true +show_error_codes = true +allow_redefinition = true +check_untyped_defs = true +plugins = ["pydantic.mypy"] diff --git a/tests/cli/conftest.py b/tests/cli/conftest.py index 11f38e95..51689e87 100644 --- a/tests/cli/conftest.py +++ b/tests/cli/conftest.py @@ -1,15 +1,36 @@ """Pytest fixtures for command line interface tests.""" -import os -import signal -from subprocess import PIPE, Popen, TimeoutExpired -from time import sleep +from __future__ import annotations + from typing import TYPE_CHECKING -import click import pytest if TYPE_CHECKING: - from typing import Optional + from typing import Protocol + + from click import Command + from click.testing import Result + + class RunCliCommand(Protocol): + """Protocol for `run_cli_command` fixture""" + + def __call__( + self, + command: Command, + options: list[str] | None = None, + raises: bool = False, + ) -> Result: + ... + + class RunAndTerminateServer(Protocol): + """Protocol for `run_and_terminate_server` fixture""" + + def __call__( + self, + command: str, + options: list[str] | None = None, + ) -> tuple[str, str]: + ... @pytest.fixture @@ -21,17 +42,19 @@ def aiida_test_profile() -> str: @pytest.fixture -def run_cli_command(aiida_test_profile: str): +def run_cli_command(aiida_test_profile: str) -> RunCliCommand: """Run a `click` command with the given options. The call will raise if the command triggered an exception or the exit code returned is non-zero. """ - from click.testing import Result + import os + + import click.testing def _run_cli_command( - command: click.Command, - options: "Optional[list[str]]" = None, + command: Command, + options: list[str] | None = None, raises: bool = False, ) -> Result: """Run the command and check the result. @@ -72,15 +95,19 @@ def _run_cli_command( @pytest.fixture -def run_and_terminate_server(aiida_test_profile: str): +def run_and_terminate_server(aiida_test_profile: str) -> RunAndTerminateServer: """Run a `click` command with the given options. The call will raise if the command triggered an exception or the exit code returned is non-zero. """ + import os + import signal + from subprocess import PIPE, Popen, TimeoutExpired + from time import sleep def _run_and_terminate_server( - command: str, options: "Optional[list[str]]" = None + command: str, options: list[str] | None = None ) -> tuple[str, str]: """Run the command and check the result. diff --git a/tests/cli/test_calc.py b/tests/cli/test_calc.py index 5d9f5bd0..b20002f0 100644 --- a/tests/cli/test_calc.py +++ b/tests/cli/test_calc.py @@ -1,20 +1,36 @@ """Test CLI `aiida-optimade calc` command""" +from __future__ import annotations + import os -import re +from typing import TYPE_CHECKING import pytest +if TYPE_CHECKING: + from pathlib import Path + + from aiida.manage.tests import TestManager + + from .conftest import RunCliCommand + @pytest.mark.skipif( os.getenv("PYTEST_OPTIMADE_CONFIG_FILE") is not None, reason="Test is not for MongoDB", ) -def test_calc_all_new(run_cli_command, aiida_profile, top_dir, caplog): +def test_calc_all_new( + run_cli_command: RunCliCommand, + aiida_profile: TestManager, + top_dir: Path, + caplog: pytest.LogCaptureFixture, +) -> None: """Test `aiida-optimade -p profile_name calc` works for non-existent fields. By "non-existent" the meaning is calculating fields that don't already exist for any Nodes. """ + import re + from aiida import orm from aiida.tools.archive.imports import import_archive @@ -104,7 +120,9 @@ def test_calc_all_new(run_cli_command, aiida_profile, top_dir, caplog): os.getenv("PYTEST_OPTIMADE_CONFIG_FILE") is not None, reason="Test is not for MongoDB", ) -def test_calc(run_cli_command, aiida_profile, top_dir): +def test_calc( + run_cli_command: RunCliCommand, aiida_profile: TestManager, top_dir: Path +) -> None: """Test `aiida-optimade -p profile_name calc` works.""" from aiida import orm from aiida.tools.archive.imports import import_archive @@ -166,8 +184,15 @@ def test_calc(run_cli_command, aiida_profile, top_dir): os.getenv("PYTEST_OPTIMADE_CONFIG_FILE") is not None, reason="Test is not for MongoDB", ) -def test_calc_partially_init(run_cli_command, aiida_profile, top_dir, caplog): +def test_calc_partially_init( + run_cli_command: RunCliCommand, + aiida_profile: TestManager, + top_dir: Path, + caplog: pytest.LogCaptureFixture, +) -> None: """Test `aiida-optimade -p profile_name calc` works for a partially initalized DB""" + import re + from aiida import orm from aiida.tools.archive.imports import import_archive diff --git a/tests/cli/test_init.py b/tests/cli/test_init.py index 19d28141..92d03e6b 100644 --- a/tests/cli/test_init.py +++ b/tests/cli/test_init.py @@ -1,19 +1,35 @@ """Test CLI `aiida-optimade init` command""" +from __future__ import annotations + import os -import re +from typing import TYPE_CHECKING import pytest +if TYPE_CHECKING: + from pathlib import Path + + from aiida.manage.tests import TestManager + + from .conftest import RunCliCommand + @pytest.mark.skipif( os.getenv("PYTEST_OPTIMADE_CONFIG_FILE") is not None, reason="Test is not for MongoDB", ) -def test_init_structuredata(run_cli_command, aiida_profile, top_dir, caplog): +def test_init_structuredata( + run_cli_command: RunCliCommand, + aiida_profile: TestManager, + top_dir: Path, + caplog: pytest.LogCaptureFixture, +) -> None: """Test `aiida-optimade -p profile_name init` works for StructureData Nodes. Also, check the `-f/--force` option. """ + import re + from aiida import orm from aiida.tools.archive.imports import import_archive @@ -90,8 +106,15 @@ def test_init_structuredata(run_cli_command, aiida_profile, top_dir, caplog): os.getenv("PYTEST_OPTIMADE_CONFIG_FILE") is not None, reason="Test is not for MongoDB", ) -def test_init_cifdata(run_cli_command, aiida_profile, top_dir, caplog): +def test_init_cifdata( + run_cli_command: RunCliCommand, + aiida_profile: TestManager, + top_dir: Path, + caplog: pytest.LogCaptureFixture, +) -> None: """Test `aiida-optimade -p profile_name init` works for CifData Nodes.""" + import re + from aiida import orm from aiida.tools.archive.imports import import_archive @@ -143,11 +166,18 @@ def test_init_cifdata(run_cli_command, aiida_profile, top_dir, caplog): @pytest.mark.skipif( os.getenv("PYTEST_OPTIMADE_CONFIG_FILE") is None, reason="Test is only for MongoDB" ) -def test_init_structuredata_mongo(run_cli_command, aiida_profile, top_dir, caplog): +def test_init_structuredata_mongo( + run_cli_command: RunCliCommand, + aiida_profile: TestManager, + top_dir: Path, + caplog: pytest.LogCaptureFixture, +) -> None: """Test `aiida-optimade -p profile_name init --mongo` works for StructureData Nodes. Also, check the `-f/--force` option. """ + import re + import bson.json_util from aiida import orm from aiida.tools.archive.imports import import_archive @@ -232,8 +262,15 @@ def test_init_structuredata_mongo(run_cli_command, aiida_profile, top_dir, caplo @pytest.mark.skipif( os.getenv("PYTEST_OPTIMADE_CONFIG_FILE") is None, reason="Test is only for MongoDB" ) -def test_init_cifdata_mongo(run_cli_command, aiida_profile, top_dir, caplog): +def test_init_cifdata_mongo( + run_cli_command: RunCliCommand, + aiida_profile: TestManager, + top_dir: Path, + caplog: pytest.LogCaptureFixture, +) -> None: """Test `aiida-optimade -p profile_name init` works for CifData Nodes.""" + import re + import bson.json_util from aiida import orm from aiida.tools.archive.imports import import_archive @@ -290,7 +327,7 @@ def test_init_cifdata_mongo(run_cli_command, aiida_profile, top_dir, caplog): STRUCTURES_MONGO.collection.insert_many(data) -def test_get_documents(top_dir): +def test_get_documents(top_dir: Path) -> None: """Test get_documents()""" import bson.json_util @@ -315,11 +352,13 @@ def test_get_documents(top_dir): @pytest.mark.parametrize( "bad_file", ["", '[{ {"key": "value"}]', '[/{"key": "value"}]'] ) -def test_get_documents_bad_file(bad_file): +def test_get_documents_bad_file(bad_file: str) -> None: """Test get_documents() with syntactically bad files""" import tempfile - def load_documents(handle, all_loaded_documents): + def load_documents( + handle: tempfile._TemporaryFileWrapper[str], all_loaded_documents: list[dict] + ) -> None: """Helper function for test""" import bson.json_util @@ -335,7 +374,7 @@ def load_documents(handle, all_loaded_documents): handle.seek(0, 0) assert handle.tell() == 0 - all_loaded_documents = [] + all_loaded_documents: list[dict] = [] if "/" in bad_file: with pytest.raises( SyntaxError, match=r"^Chunk found, but it is not self-consistent.*" @@ -346,7 +385,7 @@ def load_documents(handle, all_loaded_documents): assert not all_loaded_documents -def test_filename_aiida(run_cli_command, top_dir): +def test_filename_aiida(run_cli_command: RunCliCommand, top_dir: Path) -> None: """Ensure init excepts when using --filename without --mongo""" from aiida_optimade.cli import cmd_init @@ -368,7 +407,7 @@ def test_filename_aiida(run_cli_command, top_dir): @pytest.mark.skipif( os.getenv("PYTEST_OPTIMADE_CONFIG_FILE") is None, reason="Test is only for MongoDB" ) -def test_filename_mongo(run_cli_command, top_dir): +def test_filename_mongo(run_cli_command: RunCliCommand, top_dir: Path) -> None: """Ensure --filename works with --mongo""" import bson.json_util diff --git a/tests/cli/test_run.py b/tests/cli/test_run.py index 55e35a30..2006f84c 100644 --- a/tests/cli/test_run.py +++ b/tests/cli/test_run.py @@ -1,21 +1,29 @@ -import json -import os -import signal -from subprocess import PIPE, Popen, TimeoutExpired -from time import sleep +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Generator + + from .conftest import RunAndTerminateServer + import pytest -import requests @pytest.fixture -def run_server(aiida_test_profile: str): +def run_server(aiida_test_profile: str) -> Generator[None, None, None]: """Run the server using `aiida-optimade run` :param options: the list of command line options to pass to `aiida-optimade run` invocation :param raises: whether `aiida-optimade run` is expected to raise an exception """ + import os + import signal + from subprocess import PIPE, Popen, TimeoutExpired + from time import sleep + profile = os.getenv("AIIDA_PROFILE", aiida_test_profile) if profile == "test_profile": # This is for local tests only @@ -31,18 +39,20 @@ def run_server(aiida_test_profile: str): sleep(10) # The server needs time to start up yield finally: - result.send_signal(signal.SIGINT) - try: - result.wait(10) - except TimeoutExpired: - result.kill() - sleep(2) - assert result is not None + if result is not None: + result.send_signal(signal.SIGINT) + try: + result.wait(10) + except TimeoutExpired: + result.kill() + sleep(2) + assert result is not None @pytest.mark.usefixtures("run_server") -def test_run(): +def test_run() -> None: """Test running `aiida-optimade run`""" + import requests from optimade import __api_version__ from optimade.models import InfoResponse @@ -56,7 +66,7 @@ def test_run(): InfoResponse(**response_json) -def test_log_level_debug(run_and_terminate_server): +def test_log_level_debug(run_and_terminate_server: RunAndTerminateServer) -> None: """Test passing log level "debug" to `aiida-optimade run` In the latest versions of uvicorn, setting the log-level to "debug" @@ -70,7 +80,7 @@ def test_log_level_debug(run_and_terminate_server): assert "DEBUG:" not in output, f"output: {output!r}, errors: {errors!r}" -def test_log_level_warning(run_and_terminate_server): +def test_log_level_warning(run_and_terminate_server: RunAndTerminateServer) -> None: """Test passing log level "warning" to `aiida-optimade run`""" options = ["--log-level", "warning"] output, errors = run_and_terminate_server(command="run", options=options) @@ -80,7 +90,7 @@ def test_log_level_warning(run_and_terminate_server): ), f"output: {output!r}, errors: {errors!r}" -def test_non_valid_log_level(run_and_terminate_server): +def test_non_valid_log_level(run_and_terminate_server: RunAndTerminateServer) -> None: """Test passing a non-valid log level to `aiida-optimade run`""" options = ["--log-level", "novalidloglevel"] output, errors = run_and_terminate_server(command="run", options=options) @@ -93,7 +103,7 @@ def test_non_valid_log_level(run_and_terminate_server): @pytest.mark.skip( "Cannot handle reloading the server with the run_and_terminate_server fixture." ) -def test_debug(run_and_terminate_server): +def test_debug(run_and_terminate_server: RunAndTerminateServer) -> None: """Test --debug flag""" options = ["--debug"] output, errors = run_and_terminate_server(command="run", options=options) @@ -104,7 +114,7 @@ def test_debug(run_and_terminate_server): @pytest.mark.skip( "Cannot handle reloading the server with the run_and_terminate_server fixture." ) -def test_logging_precedence(run_and_terminate_server): +def test_logging_precedence(run_and_terminate_server: RunAndTerminateServer) -> None: """Test --log-level takes precedence over --debug""" options = ["--debug", "--log-level", "warning"] output, errors = run_and_terminate_server(command="run", options=options) @@ -114,7 +124,9 @@ def test_logging_precedence(run_and_terminate_server): ), f"output: {output!r}, errors: {errors!r}" -def test_env_var_is_set(run_and_terminate_server, aiida_test_profile: str): +def test_env_var_is_set( + run_and_terminate_server: RunAndTerminateServer, aiida_test_profile: str +) -> None: """Test the AIIDA_PROFILE env var is set The issue with this test, is that the set "AIIDA_PROFILE" environment variable @@ -125,6 +137,7 @@ def test_env_var_is_set(run_and_terminate_server, aiida_test_profile: str): Since `run_and_terminate_server` automatically sets the "AIIDA_PROFILE" environment variable to the current "AIIDA_PROFILE", we will check that here. """ + import os fixture_profile = os.getenv("AIIDA_PROFILE") assert fixture_profile is not None @@ -136,8 +149,12 @@ def test_env_var_is_set(run_and_terminate_server, aiida_test_profile: str): @pytest.mark.usefixtures("run_server") -def test_last_modified(): +def test_last_modified() -> None: """Ensure last_modified does not change upon requests""" + import json + from time import sleep + + import requests from optimade import __api_version__ request = ( @@ -168,7 +185,7 @@ def test_last_modified(): @pytest.mark.skip( "Cannot handle reloading the server with the run_and_terminate_server fixture." ) -def test_dev_option(run_and_terminate_server): +def test_dev_option(run_and_terminate_server: RunAndTerminateServer) -> None: """Test --dev flag This should be equivalent to running with the `--debug` option for diff --git a/tests/conftest.py b/tests/conftest.py index 8533962f..5b43afe5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,19 +1,29 @@ -import os -from pathlib import Path +from __future__ import annotations + +from typing import TYPE_CHECKING import pytest -from aiida.manage.tests import TestManager + +if TYPE_CHECKING: + from collections.abc import Generator + from pathlib import Path + + from aiida.manage.tests import TestManager @pytest.fixture(scope="session") def top_dir() -> Path: """Return Path instance for the repository's top (root) directory""" + from pathlib import Path + return Path(__file__).parent.parent.resolve() @pytest.fixture(scope="session", autouse=True) -def setup_config(top_dir) -> None: +def setup_config(top_dir: Path) -> Generator[None, None, None]: """Method that runs before pytest collects tests so no modules are imported""" + import os + filename = top_dir / "tests/static/test_config.json" original_env_var = os.getenv("OPTIMADE_CONFIG_FILE") @@ -31,11 +41,15 @@ def setup_config(top_dir) -> None: @pytest.fixture(scope="session", autouse=True) -def aiida_profile(top_dir, setup_config) -> TestManager: +def aiida_profile( + top_dir: Path, setup_config: None +) -> Generator[TestManager, None, None]: """Load test data for AiiDA test profile It is necessary to remove `AIIDA_PROFILE`, since it clashes with the test profile """ + import os + from aiida import load_profile from aiida.manage.tests import ( get_test_backend_name, diff --git a/tests/server/__init__.py b/tests/server/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/server/conftest.py b/tests/server/conftest.py index 8dffe1b3..d8fceb7b 100644 --- a/tests/server/conftest.py +++ b/tests/server/conftest.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import re from typing import TYPE_CHECKING @@ -5,15 +7,47 @@ if TYPE_CHECKING: from collections.abc import Iterable - from typing import Any, Callable, Dict, List, Optional, Union + from typing import Any, Protocol from httpx import Response + from optimade.server.config import CONFIG from .utils import OptimadeTestClient + class GetGoodResponse(Protocol): + def __call__( + self, request: str, raw: bool = False + ) -> Response | dict[str, Any]: + ... + + class CheckKeys(Protocol): + def __call__(self, keys: list[str], response_subset: Iterable) -> None: + ... + + class CheckResponse(Protocol): + def __call__( + self, + request: str, + expected_uuid: list[str], + page_limit: int = CONFIG.page_limit, + expect_id: bool = False, + expected_as_is: bool = False, + ) -> None: + ... + + class CheckErrorResponse(Protocol): + def __call__( + self, + request: str, + expected_status: int | None = None, + expected_title: str | None = None, + expected_detail: str | None = None, + ) -> None: + ... + @pytest.fixture(scope="module") -def client() -> "OptimadeTestClient": +def client() -> OptimadeTestClient: """Return TestClient for OPTIMADE server""" from .utils import client_factory @@ -21,7 +55,7 @@ def client() -> "OptimadeTestClient": @pytest.fixture(scope="module") -def remote_client() -> "OptimadeTestClient": +def remote_client() -> OptimadeTestClient: """Return TestClient for OPTIMADE server, mimicking a remote client""" from .utils import client_factory @@ -30,13 +64,13 @@ def remote_client() -> "OptimadeTestClient": @pytest.fixture def get_good_response( - client: "OptimadeTestClient", caplog: pytest.LogCaptureFixture -) -> "Callable[[str, bool], Union[Response, Dict[str, Any]]]": + client: OptimadeTestClient, caplog: pytest.LogCaptureFixture +) -> GetGoodResponse: """Get OPTIMADE response with some sanity checks""" - def inner(request: str, raw: bool = False) -> "Union[Response, Dict[str, Any]]": + def inner(request: str, raw: bool = False) -> Response | dict[str, Any]: if TYPE_CHECKING: - response: "Union[Response, Dict[str, Any]]" + response: Response | dict[str, Any] try: response = client.get(request) @@ -65,12 +99,12 @@ def inner(request: str, raw: bool = False) -> "Union[Response, Dict[str, Any]]": @pytest.fixture -def check_keys() -> "Callable[[list, Iterable], None]": +def check_keys() -> CheckKeys: """Utility function to help validate dict keys""" def inner( - keys: list, - response_subset: "Iterable", + keys: list[str], + response_subset: Iterable, ) -> None: for key in keys: assert ( @@ -82,14 +116,14 @@ def inner( @pytest.fixture def check_response( - get_good_response: "Callable[[str, bool], Union[Response, Dict[str, Any]]]", -) -> "Callable[[str, List[str], int, bool, bool], None]": + get_good_response: GetGoodResponse, +) -> CheckResponse: """Fixture to check response using client fixture""" from optimade.server.config import CONFIG def inner( request: str, - expected_uuid: "List[str]", + expected_uuid: list[str], page_limit: int = CONFIG.page_limit, expect_id: bool = False, expected_as_is: bool = False, @@ -123,17 +157,17 @@ def inner( @pytest.fixture def check_error_response( - remote_client: "OptimadeTestClient", caplog: pytest.LogCaptureFixture -): + remote_client: OptimadeTestClient, caplog: pytest.LogCaptureFixture +) -> CheckErrorResponse: """General method for testing expected erroneous response""" def inner( request: str, - expected_status: "Optional[int]" = None, - expected_title: "Optional[str]" = None, - expected_detail: "Optional[str]" = None, - ): - response: "Optional[Response]" = None + expected_status: int | None = None, + expected_title: str | None = None, + expected_detail: str | None = None, + ) -> None: + response: Response | None = None try: response = remote_client.get(request) @@ -163,7 +197,7 @@ def inner( f"\nResponse:\n{response.json()}", ) - json_response: "Dict[str, Any]" = response.json() + json_response: dict[str, Any] = response.json() assert len(json_response["errors"]) == 1, json_response.get( "errors", "'errors' not found" ) diff --git a/tests/server/query_params/test_filter.py b/tests/server/query_params/test_filter.py index 2dd6dc99..af30b159 100644 --- a/tests/server/query_params/test_filter.py +++ b/tests/server/query_params/test_filter.py @@ -1,13 +1,21 @@ """Test the `filters` query parameter.""" +from __future__ import annotations + import os +from typing import TYPE_CHECKING import pytest +if TYPE_CHECKING: + from typing import Any + + from ..conftest import CheckErrorResponse, CheckResponse + @pytest.mark.skip( "Un-skip when a fix for optimade-python-tools issue #102 is in place." ) -def test_custom_field(check_response): +def test_custom_field(check_response: CheckResponse) -> None: from optimade.server.config import CONFIG request = ( @@ -19,33 +27,33 @@ def test_custom_field(check_response): check_response(request, expected_uuids) -def test_id(check_response, get_valid_id): +def test_id(check_response: CheckResponse, get_valid_id: str) -> None: request = f'/structures?filter=id="{get_valid_id}"' expected_ids = [str(get_valid_id)] check_response(request, expected_ids, expect_id=True) -def test_geq(check_response): +def test_geq(check_response: CheckResponse) -> None: request = "/structures?filter=nelements>=18" expected_uuids = ["b6175807-826a-459f-8a5a-7bff75ff1d36"] check_response(request, expected_uuids) -def test_gt(check_response): +def test_gt(check_response: CheckResponse) -> None: request = "/structures?filter=nelements>17" expected_uuids = ["b6175807-826a-459f-8a5a-7bff75ff1d36"] check_response(request, expected_uuids) -def test_gt_none(check_response): +def test_gt_none(check_response: CheckResponse) -> None: request = "/structures?filter=nelements>18" - expected_uuids = [] + expected_uuids: list[str] = [] check_response(request, expected_uuids) -def test_rhs_statements(check_response, get_valid_id): +def test_rhs_statements(check_response: CheckResponse, get_valid_id: str) -> None: request = "/structures?filter=18 None: request = '/structures?filter=elements HAS "Ga"' expected_uuids = [ "199bf419-0393-4970-8822-f1014e457d3c", @@ -78,7 +86,7 @@ def test_list_has(check_response): check_response(request, expected_uuids) -def test_page_limit(check_response): +def test_page_limit(check_response: CheckResponse) -> None: request = '/structures?filter=elements HAS ALL "Ge","S"&page_limit=2' expected_uuids = [ "02548222-8f47-4fb4-afdb-197e2984f818", @@ -92,13 +100,13 @@ def test_page_limit(check_response): check_response(request, expected_uuids, page_limit=2) -def test_list_has_all(check_response): +def test_list_has_all(check_response: CheckResponse) -> None: request = '/structures?filter=elements HAS ALL "Ge","Na","Al","Cl","O"' expected_uuids = ["254947de-54c8-4cdb-afc5-1cee237f9f98"] check_response(request, expected_uuids) -def test_list_has_any(check_response): +def test_list_has_any(check_response: CheckResponse) -> None: elements = '"La","Ba"' request = f"/structures?filter=elements HAS ALL {elements}" expected_uuids = ["c8368624-e49a-46ad-aef7-daaee4ff89e3"] @@ -142,13 +150,13 @@ def test_list_has_any(check_response): check_response(request, expected_uuids) -def test_list_length_basic(check_response): +def test_list_length_basic(check_response: CheckResponse) -> None: request = "/structures?filter=elements LENGTH 18" expected_uuids = ["b6175807-826a-459f-8a5a-7bff75ff1d36"] check_response(request, expected_uuids) -def test_list_length_operators(check_response): +def test_list_length_operators(check_response: CheckResponse) -> None: request = "/structures?filter=elements LENGTH = 17" expected_uuids = ["dd369206-2ccb-4528-8c73-141d77fe5fa1"] check_response(request, expected_uuids) @@ -168,7 +176,7 @@ def test_list_length_operators(check_response): check_response(request, expected_uuids) -def test_list_length_bad_operators(check_error_response): +def test_list_length_bad_operators(check_error_response: CheckErrorResponse) -> None: """Check NonImplementedError is raised when using a valid, but not-supported operator""" from optimade.server.config import CONFIG, SupportedBackend @@ -197,7 +205,7 @@ def test_list_length_bad_operators(check_error_response): os.getenv("PYTEST_OPTIMADE_CONFIG_FILE") is not None, reason="Test is not for MongoDB", ) -def test_list_has_only(check_error_response): +def test_list_has_only(check_error_response: CheckErrorResponse) -> None: # HAS ONLY is not yet implemented request = '/structures?filter=elements HAS ONLY "Ac"' check_error_response( @@ -208,7 +216,7 @@ def test_list_has_only(check_error_response): ) -def test_list_correlated(check_error_response): +def test_list_correlated(check_error_response: CheckErrorResponse) -> None: # Zipped lists are not yet implemented request = '/structures?filter=elements:elements_ratios HAS "Ag":"0.2"' expected_detail = ( @@ -224,7 +232,7 @@ def test_list_correlated(check_error_response): ) -def test_saved_extras_is_known(check_response): +def test_saved_extras_is_known(check_response: CheckResponse) -> None: request = "/structures?filter=nsites IS KNOWN AND nsites>=5280" expected_uuids = [ "d99ddab5-026b-45f6-88b7-d81bf0e41988", @@ -240,7 +248,7 @@ def test_saved_extras_is_known(check_response): check_response(request, expected_uuids) -def test_node_columns_is_known(check_response): +def test_node_columns_is_known(check_response: CheckResponse) -> None: from optimade.server.config import CONFIG, SupportedBackend request = ( @@ -270,13 +278,13 @@ def test_node_columns_is_known(check_response): check_response(request, expected_uuids) -def test_node_column_fields(check_response, get_valid_id): +def test_node_column_fields(check_response: CheckResponse, get_valid_id: str) -> None: request = f'/structures?filter=id="{get_valid_id}"' expected_ids = [str(get_valid_id)] check_response(request, expected_ids, expect_id=True) -def test_saved_extras_fields(check_response): +def test_saved_extras_fields(check_response: CheckResponse) -> None: request = '/structures?filter=chemical_formula_anonymous CONTAINS "A7B4"' expected_uuids = [ "2ebd7c96-cadd-464b-aeda-58e3b86f1347", @@ -288,7 +296,7 @@ def test_saved_extras_fields(check_response): check_response(request, expected_uuids) -def test_string_contains(check_response): +def test_string_contains(check_response: CheckResponse) -> None: request = '/structures?filter=chemical_formula_descriptive CONTAINS "Ag4Cl"' expected_uuids = [ "8223bf92-829b-4ba7-9bf6-887f8f21dee8", @@ -297,7 +305,7 @@ def test_string_contains(check_response): check_response(request, expected_uuids) -def test_string_start(check_response): +def test_string_start(check_response: CheckResponse) -> None: request = '/structures?filter=chemical_formula_descriptive STARTS WITH "H"' expected_uuids = [ "02c28e40-0072-418a-9069-7e6ea123ce70", @@ -320,7 +328,7 @@ def test_string_start(check_response): check_response(request, expected_uuids) -def test_string_end(check_response): +def test_string_end(check_response: CheckResponse) -> None: request = '/structures?filter=chemical_formula_descriptive ENDS WITH "0}9"' expected_uuids = [ "7ddb0679-3255-4cea-91be-749e44e9e900", @@ -330,13 +338,13 @@ def test_string_end(check_response): check_response(request, expected_uuids) -def test_list_has_and(check_response): +def test_list_has_and(check_response: CheckResponse) -> None: request = '/structures?filter=elements HAS "Na" AND nelements=18' expected_uuids = ["b6175807-826a-459f-8a5a-7bff75ff1d36"] check_response(request, expected_uuids) -def test_not_or_and_precedence(check_response): +def test_not_or_and_precedence(check_response: CheckResponse) -> None: request = '/structures?filter=NOT elements HAS "Na" AND nelements=5' expected_uuids = [ "705cf9c6-25b3-4720-a079-a342f34712a2", @@ -374,7 +382,7 @@ def test_not_or_and_precedence(check_response): check_response(request, expected_uuids) -def test_brackets(check_response): +def test_brackets(check_response: CheckResponse) -> None: request = '/structures?filter=elements HAS "Ga" AND nelements=7 OR nsites=464' expected_uuids = [ "b9e0df95-6029-48cf-a4b4-ddbe0a613572", @@ -395,14 +403,14 @@ def test_brackets(check_response): check_response(request, expected_uuids) -def test_count_filter(caplog): +def test_count_filter(caplog: pytest.LogCaptureFixture) -> None: """Test EntryCollection.count() when changing filters""" from aiida_optimade.routers.structures import STRUCTURES STRUCTURES._count = None # The _count attribute should be None - filters = {} + filters: dict[str, Any] = {} count_one = STRUCTURES.count(filters=filters) assert "self._count is None" in caplog.text assert "was not the same as was found in self._count" not in caplog.text @@ -454,7 +462,9 @@ def test_count_filter(caplog): os.getenv("PYTEST_OPTIMADE_CONFIG_FILE") is not None, reason="Test is not for MongoDB", ) -def test_querybuilder_calls(caplog, get_valid_id): +def test_querybuilder_calls( + caplog: pytest.LogCaptureFixture, get_valid_id: str +) -> None: """Check the expected number of QueryBuilder calls are respected""" from fastapi.params import Query from optimade.server.query_params import EntryListingQueryParams diff --git a/tests/server/query_params/test_page_limit.py b/tests/server/query_params/test_page_limit.py index b2ec7756..1d5baf7d 100644 --- a/tests/server/query_params/test_page_limit.py +++ b/tests/server/query_params/test_page_limit.py @@ -1,7 +1,15 @@ """Test the `page_limit` query parameter""" +from __future__ import annotations +from typing import TYPE_CHECKING -def test_limit(get_good_response): +if TYPE_CHECKING: + import pytest + + from ..conftest import CheckErrorResponse, GetGoodResponse + + +def test_limit(get_good_response: GetGoodResponse) -> None: """Check page_limit is respected""" page_limit = [5, 10] for limit in page_limit: @@ -11,7 +19,7 @@ def test_limit(get_good_response): assert len(response["data"]) == limit -def test_count_limit(caplog): +def test_count_limit(caplog: pytest.LogCaptureFixture) -> None: """Test EntryCollection.count() when changing limit""" from aiida_optimade.routers.structures import STRUCTURES @@ -60,7 +68,9 @@ def test_count_limit(caplog): } -def test_page_limit_max(get_good_response, check_error_response): +def test_page_limit_max( + get_good_response: GetGoodResponse, check_error_response: CheckErrorResponse +) -> None: """Ensure the configuration page_limit_max is respected""" from optimade.server.config import CONFIG diff --git a/tests/server/query_params/test_page_offset.py b/tests/server/query_params/test_page_offset.py index b343a061..0eba3ecc 100644 --- a/tests/server/query_params/test_page_offset.py +++ b/tests/server/query_params/test_page_offset.py @@ -1,7 +1,15 @@ """Test the `page_offset` query parameter""" +from __future__ import annotations +from typing import TYPE_CHECKING -def test_offset(get_good_response): +if TYPE_CHECKING: + import pytest + + from ..conftest import GetGoodResponse + + +def test_offset(get_good_response: GetGoodResponse) -> None: """Apply low offset, comparing two requests with and without offset""" page_limit = 5 request = f"/structures?page_offset=0&page_limit={page_limit}&sort=immutable_id" @@ -22,7 +30,7 @@ def test_offset(get_good_response): assert expected_uuids == [_["attributes"]["immutable_id"] for _ in response["data"]] -def test_count_offset(caplog): +def test_count_offset(caplog: pytest.LogCaptureFixture) -> None: """Test EntryCollection.count() when changing offset""" from aiida_optimade.routers.structures import STRUCTURES diff --git a/tests/server/query_params/test_response_fields.py b/tests/server/query_params/test_response_fields.py index d7be7d70..79481098 100644 --- a/tests/server/query_params/test_response_fields.py +++ b/tests/server/query_params/test_response_fields.py @@ -1,4 +1,12 @@ -def test_provider_fields(get_good_response): +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..conftest import CheckErrorResponse, GetGoodResponse + + +def test_provider_fields(get_good_response: GetGoodResponse) -> None: """Ensure provider fields can be requested""" from optimade.server.config import CONFIG @@ -17,7 +25,7 @@ def test_provider_fields(get_good_response): } -def test_non_provider_fields(get_good_response): +def test_non_provider_fields(get_good_response: GetGoodResponse) -> None: """Ensure provider fields are excluded when not requested""" non_provider_specific_field = "elements" request = f"/structures?response_fields={non_provider_specific_field}" @@ -31,7 +39,7 @@ def test_non_provider_fields(get_good_response): } -def test_wrong_alias_provider_fields(check_error_response): +def test_wrong_alias_provider_fields(check_error_response: CheckErrorResponse) -> None: """Ensure wrongly aliased provider fields raise a 400 Bad Request""" from optimade.server.config import CONFIG diff --git a/tests/server/query_params/test_sort.py b/tests/server/query_params/test_sort.py index d0ac62ea..ba24a3fd 100644 --- a/tests/server/query_params/test_sort.py +++ b/tests/server/query_params/test_sort.py @@ -1,16 +1,24 @@ """Test sort query parameter""" -from datetime import datetime, timezone +from __future__ import annotations -from aiida import orm +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from datetime import datetime + + from ..conftest import CheckResponse, GetGoodResponse def fmt_datetime(object_: datetime) -> str: """Parse datetime into pydantic's JSON encoded datetime string""" + from datetime import timezone + return object_.astimezone(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") -def test_int_asc(get_good_response): +def test_int_asc(get_good_response: GetGoodResponse) -> None: """Ascending sort (integer)""" + from aiida import orm from optimade.server.config import CONFIG, SupportedBackend limit = 5 @@ -50,8 +58,9 @@ def test_int_asc(get_good_response): assert nelements_list == expected_nelements -def test_int_desc(get_good_response): +def test_int_desc(get_good_response: GetGoodResponse) -> None: """Descending sort (integer)""" + from aiida import orm from optimade.server.config import CONFIG, SupportedBackend limit = 5 @@ -91,8 +100,9 @@ def test_int_desc(get_good_response): assert nelements_list == expected_nelements -def test_str_asc(check_response): +def test_str_asc(check_response: CheckResponse) -> None: """Ascending sort (string)""" + from aiida import orm from optimade.server.config import CONFIG, SupportedBackend request = "/structures?sort=immutable_id&page_limit=5" @@ -124,8 +134,9 @@ def test_str_asc(check_response): ) -def test_str_desc(check_response): +def test_str_desc(check_response: CheckResponse) -> None: """Descending sort (string)""" + from aiida import orm from optimade.server.config import CONFIG, SupportedBackend request = "/structures?sort=-immutable_id&page_limit=5" @@ -157,8 +168,9 @@ def test_str_desc(check_response): ) -def test_datetime_asc(get_good_response): +def test_datetime_asc(get_good_response: GetGoodResponse) -> None: """Ascending sort (datetime)""" + from aiida import orm from optimade.server.config import CONFIG, SupportedBackend request = "/structures?sort=last_modified&page_limit=5" @@ -190,8 +202,9 @@ def test_datetime_asc(get_good_response): assert last_modified_list == expected_mtime -def test_datetime_desc(get_good_response): +def test_datetime_desc(get_good_response: GetGoodResponse) -> None: """Descending sort (datetime)""" + from aiida import orm from optimade.server.config import CONFIG, SupportedBackend request = "/structures?sort=-last_modified&page_limit=5" diff --git a/tests/server/routers/test_info.py b/tests/server/routers/test_info.py index 99cf57f6..5b072c09 100644 --- a/tests/server/routers/test_info.py +++ b/tests/server/routers/test_info.py @@ -1,9 +1,19 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import pytest -from optimade.models import BaseInfoAttributes, EntryInfoResource + +if TYPE_CHECKING: + from ..conftest import CheckKeys, GetGoodResponse -def test_info_endpoint_attributes(get_good_response, check_keys): +def test_info_endpoint_attributes( + get_good_response: GetGoodResponse, check_keys: CheckKeys +) -> None: """Check known properties/attributes for successful response""" + from optimade.models import BaseInfoAttributes + response = get_good_response("/info") assert "data" in response @@ -14,8 +24,12 @@ def test_info_endpoint_attributes(get_good_response, check_keys): check_keys(attributes, response["data"]["attributes"]) -def test_info_structures_endpoint_data(get_good_response, check_keys): +def test_info_structures_endpoint_data( + get_good_response: GetGoodResponse, check_keys: CheckKeys +) -> None: """Check known properties/attributes for successful response""" + from optimade.models import EntryInfoResource + response = get_good_response("/info/structures") assert "data" in response @@ -23,7 +37,7 @@ def test_info_structures_endpoint_data(get_good_response, check_keys): check_keys(data, response["data"]) -def test_info_structures_sortable(get_good_response): +def test_info_structures_sortable(get_good_response: GetGoodResponse) -> None: """Check the sortable key is present for all properties""" response = get_good_response("/info/structures") @@ -31,7 +45,7 @@ def test_info_structures_sortable(get_good_response): assert "sortable" in info_keys -def test_sortable_values(get_good_response): +def test_sortable_values(get_good_response: GetGoodResponse) -> None: """Make sure certain properties are and are not sortable""" response = get_good_response("/info/structures") sortable = ["id", "nelements", "nsites"] @@ -58,7 +72,7 @@ def test_sortable_values(get_good_response): assert sortable_info_value is False -def test_info_structures_unit(get_good_response): +def test_info_structures_unit(get_good_response: GetGoodResponse) -> None: """Check the unit key is present for certain properties""" response = get_good_response("/info/structures") unit_fields = ["lattice_vectors", "cartesian_site_positions"] @@ -69,7 +83,7 @@ def test_info_structures_unit(get_good_response): assert "unit" not in info_keys, f"Field: {field}" -def test_provider_fields(get_good_response): +def test_provider_fields(get_good_response: GetGoodResponse) -> None: """Check the presence of AiiDA-specific fields""" from optimade.server.config import CONFIG @@ -95,8 +109,12 @@ def test_provider_fields(get_good_response): @pytest.mark.skip("References has not yet been implemented") -def test_info_references_endpoint_data(get_good_response, check_keys): +def test_info_references_endpoint_data( + get_good_response: GetGoodResponse, check_keys: CheckKeys +) -> None: """Check known properties/attributes for successful response""" + from optimade.models import EntryInfoResource + response = get_good_response("/info/reference") assert "data" in response diff --git a/tests/server/routers/test_links.py b/tests/server/routers/test_links.py index f69d9a90..d12f74c6 100644 --- a/tests/server/routers/test_links.py +++ b/tests/server/routers/test_links.py @@ -1,4 +1,12 @@ -def test_links(get_good_response): +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from ..conftest import GetGoodResponse + + +def test_links(get_good_response: GetGoodResponse) -> None: """Check /links for successful response""" response = get_good_response("/links") diff --git a/tests/server/routers/test_references.py b/tests/server/routers/test_references.py index 4586e7dd..6accd6db 100644 --- a/tests/server/routers/test_references.py +++ b/tests/server/routers/test_references.py @@ -1,16 +1,34 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import pytest -from optimade.models import ReferenceResponseMany, ReferenceResponseOne from ..utils import EndpointTests +if TYPE_CHECKING: + from optimade.models import ReferenceResponseMany, ReferenceResponseOne + pytestmark = pytest.mark.skip("References has not yet been implemented") +def _get_optimade_reference_response_model( + name: str, +) -> type[ReferenceResponseMany] | type[ReferenceResponseOne]: + from optimade.models import ReferenceResponseMany, ReferenceResponseOne + + if name == "ReferenceResponseMany": + return ReferenceResponseMany + if name == "ReferenceResponseOne": + return ReferenceResponseOne + raise ValueError(f"Unknown response model name: {name}") + + class TestReferencesEndpoint(EndpointTests): """Tests for /references""" request_str = "/references" - response_cls = ReferenceResponseMany + response_cls = _get_optimade_reference_response_model("ReferenceResponseMany") class TestSingleReferenceEndpoint(EndpointTests): @@ -18,7 +36,7 @@ class TestSingleReferenceEndpoint(EndpointTests): test_id = "dijkstra1968" request_str = f"/references/{test_id}" - response_cls = ReferenceResponseOne + response_cls = _get_optimade_reference_response_model("ReferenceResponseOne") class TestSingleReferenceEndpointDifficult(EndpointTests): @@ -27,7 +45,7 @@ class TestSingleReferenceEndpointDifficult(EndpointTests): test_id = "dummy/20.19" request_str = f"/references/{test_id}" - response_cls = ReferenceResponseOne + response_cls = _get_optimade_reference_response_model("ReferenceResponseOne") class TestMissingSingleReferenceEndpoint(EndpointTests): @@ -35,10 +53,11 @@ class TestMissingSingleReferenceEndpoint(EndpointTests): test_id = "random_string_that_is_not_in_test_data" request_str = f"/references/{test_id}" - response_cls = ReferenceResponseOne + response_cls = _get_optimade_reference_response_model("ReferenceResponseOne") - def test_references_endpoint_data(self): + def test_references_endpoint_data(self) -> None: """Check known properties/attributes for successful response""" + assert isinstance(self.json_response, dict) assert "data" in self.json_response assert "meta" in self.json_response assert self.json_response["data"] is None diff --git a/tests/server/routers/test_response.py b/tests/server/routers/test_response.py index 38041026..26cdfeab 100644 --- a/tests/server/routers/test_response.py +++ b/tests/server/routers/test_response.py @@ -1,19 +1,28 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING + import pytest -from optimade.models import ( - EntryInfoResponse, - InfoResponse, - LinksResponse, - ReferenceResponseMany, - ReferenceResponseOne, - ResponseMeta, - StructureResponseMany, - StructureResponseOne, -) +if TYPE_CHECKING: + from _pytest.mark.structures import ParameterSet + from optimade.models import Response -@pytest.mark.parametrize( - "request_str, ResponseType", - [ + from ..conftest import CheckKeys, GetGoodResponse + + +def _serialize_response_parameters() -> list[tuple[str, type[Response]] | ParameterSet]: + from optimade.models import ( + EntryInfoResponse, + InfoResponse, + LinksResponse, + ReferenceResponseMany, + ReferenceResponseOne, + StructureResponseMany, + StructureResponseOne, + ) + + return [ ("/info", InfoResponse), ("/info/structures", EntryInfoResponse), ("/links", LinksResponse), @@ -34,9 +43,16 @@ ReferenceResponseOne, marks=pytest.mark.xfail(reason="References has not yet been implemented"), ), - ], + ] + + +@pytest.mark.parametrize( + "request_str, ResponseType", + _serialize_response_parameters(), ) -def test_serialize_response(get_good_response, request_str, ResponseType): +def test_serialize_response( + get_good_response: GetGoodResponse, request_str: str, ResponseType: type[Response] +) -> None: response = get_good_response(request_str) ResponseType(**response) @@ -56,8 +72,12 @@ def test_serialize_response(get_good_response, request_str, ResponseType): ), ], ) -def test_meta_response(request_str, get_good_response, check_keys): +def test_meta_response( + request_str: str, get_good_response: GetGoodResponse, check_keys: CheckKeys +) -> None: """Check `meta` property in response""" + from optimade.models import ResponseMeta + response = get_good_response(request_str) assert "meta" in response diff --git a/tests/server/routers/test_structures.py b/tests/server/routers/test_structures.py index dfbc785f..de5b26df 100644 --- a/tests/server/routers/test_structures.py +++ b/tests/server/routers/test_structures.py @@ -1,21 +1,43 @@ +from __future__ import annotations + import os +from typing import TYPE_CHECKING import pytest -from optimade.models import ( - ReferenceResource, - StructureResponseMany, - StructureResponseOne, -) from ..utils import EndpointTests +if TYPE_CHECKING: + from optimade.models import ( + StructureResponseMany, + StructureResponseOne, + ) + + from ..conftest import GetGoodResponse + from ..utils import OptimadeTestClient + + +def _get_optimade_structure_response_model( + model_name: str, +) -> type[StructureResponseOne] | type[StructureResponseMany]: + from optimade.models import ( + StructureResponseMany, + StructureResponseOne, + ) + + if model_name == "StructureResponseMany": + return StructureResponseMany + if model_name == "StructureResponseOne": + return StructureResponseOne + raise ValueError(f"Unknown model name: {model_name}") + @pytest.mark.skipif( os.getenv("PYTEST_OPTIMADE_CONFIG_FILE") != "./tests/static/test_data_curation_config.json", reason="Test is not for data curation", ) -def test_structures_endpoint_data(get_good_response): +def test_structures_endpoint_data(get_good_response: GetGoodResponse) -> None: """Check known properties/attributes for successful response""" from optimade.server.config import CONFIG @@ -28,7 +50,9 @@ def test_structures_endpoint_data(get_good_response): assert response["meta"]["more_data_available"] -def test_get_next_responses(get_good_response, client): +def test_get_next_responses( + get_good_response: GetGoodResponse, client: OptimadeTestClient +) -> None: """Check pagination""" response = get_good_response("/structures") @@ -60,7 +84,7 @@ def test_get_next_responses(get_good_response, client): @pytest.mark.skip("Profile database mess up by tests in cli tests") -def test_structures_id_endpoint_data(get_good_response): +def test_structures_id_endpoint_data(get_good_response: GetGoodResponse) -> None: """Check known properties/attributes for successful response""" from optimade.server.config import CONFIG @@ -76,7 +100,7 @@ def test_structures_id_endpoint_data(get_good_response): ) -def test_structures_missing_endpoint_data(get_good_response): +def test_structures_missing_endpoint_data(get_good_response: GetGoodResponse) -> None: """Check known properties/attributes for successful response""" test_id = "0" response = get_good_response(f"/structures/{test_id}") @@ -94,10 +118,13 @@ class TestSingleStructureWithRelationships(EndpointTests): test_id = "1" request_str = f"/structures/{test_id}" - response_cls = StructureResponseOne + response_cls = _get_optimade_structure_response_model("StructureResponseOne") - def test_structures_endpoint_data(self): + def test_structures_endpoint_data(self) -> None: """Check known properties/attributes for successful response""" + from optimade.models import ReferenceResource + + assert isinstance(self.json_response, dict) assert "data" in self.json_response assert self.json_response["data"]["id"] == self.test_id assert self.json_response["data"]["type"] == "structures" @@ -119,12 +146,13 @@ class TestMultiStructureWithSharedRelationships(EndpointTests): """Tests for /structures for entries with shared relationships""" request_str = "/structures?filter=id=mpf_1 OR id=mpf_2" - response_cls = StructureResponseMany + response_cls = _get_optimade_structure_response_model("StructureResponseMany") - def test_structures_endpoint_data(self): + def test_structures_endpoint_data(self) -> None: """Check known properties/attributes for successful response""" # mpf_1 and mpf_2 both contain the same reference relationship, # so the response should not duplicate it + assert isinstance(self.json_response, dict) assert "data" in self.json_response assert len(self.json_response["data"]) == 2 assert "included" in self.json_response @@ -136,11 +164,12 @@ class TestMultiStructureWithRelationships(EndpointTests): """Tests for /structures for mixed entries with and without relationships""" request_str = "/structures?filter=id=mpf_1 OR id=mpf_23" - response_cls = StructureResponseMany + response_cls = _get_optimade_structure_response_model("StructureResponseMany") - def test_structures_endpoint_data(self): + def test_structures_endpoint_data(self) -> None: """Check known properties/attributes for successful response""" # mpf_23 contains no relationships, which shouldn't break anything + assert isinstance(self.json_response, dict) assert "data" in self.json_response assert len(self.json_response["data"]) == 2 assert "included" in self.json_response @@ -156,10 +185,11 @@ class TestMultiStructureWithOverlappingRelationships(EndpointTests): """ request_str = "/structures?filter=id=mpf_1 OR id=mpf_3" - response_cls = StructureResponseMany + response_cls = _get_optimade_structure_response_model("StructureResponseMany") - def test_structures_endpoint_data(self): + def test_structures_endpoint_data(self) -> None: """Check known properties/attributes for successful response""" + assert isinstance(self.json_response, dict) assert "data" in self.json_response assert len(self.json_response["data"]) == 2 assert "included" in self.json_response diff --git a/tests/server/routers/test_versions.py b/tests/server/routers/test_versions.py index 08ff6fa4..932e8e65 100644 --- a/tests/server/routers/test_versions.py +++ b/tests/server/routers/test_versions.py @@ -1,10 +1,19 @@ -from optimade import __api_version__ +from __future__ import annotations +from typing import TYPE_CHECKING -def test_versions_endpoint(get_good_response): +if TYPE_CHECKING: + from ..conftest import GetGoodResponse + + +def test_versions_endpoint(get_good_response: GetGoodResponse) -> None: """Check known content for a successful response""" + from optimade import __api_version__ + response = get_good_response("/versions", raw=True) + assert not isinstance(response, dict) + assert response.text == f"version\n{__api_version__.replace('v', '').split('.')[0]}" assert "text/csv" in response.headers.get("content-type") assert "header=present" in response.headers.get("content-type") diff --git a/tests/server/test_entry_collections.py b/tests/server/test_entry_collections.py index 15cebfb3..6fb97c14 100644 --- a/tests/server/test_entry_collections.py +++ b/tests/server/test_entry_collections.py @@ -1,10 +1,15 @@ """Tests for aiida_optimade.entry_collections.""" -from typing import Any, Callable +from __future__ import annotations + +from typing import TYPE_CHECKING import pytest +if TYPE_CHECKING: + from .conftest import CheckErrorResponse, GetGoodResponse + -def test_insert(): +def test_insert() -> None: """Test AiidaCollection.insert() raises NotImplentedError.""" from aiida_optimade.routers.structures import STRUCTURES @@ -15,7 +20,7 @@ def test_insert(): @pytest.mark.parametrize("attribute", ["data_available", "data_returned"]) -def test_causation_errors(attribute: str): +def test_causation_errors(attribute: str) -> None: """Test CausationError is returned if requesting `data_available` or `data_returned` before setting them.""" from aiida_optimade.common.exceptions import CausationError @@ -28,9 +33,9 @@ def test_causation_errors(attribute: str): def test_bad_fields( - get_good_response: Callable[[str], dict[str, Any]], - check_error_response: Callable[[str, int, str, str], None], -): + get_good_response: GetGoodResponse, + check_error_response: CheckErrorResponse, +) -> None: """Test a UnknownProviderProperty warning is emitted for unrecognized provider fields.""" from optimade.server.config import CONFIG @@ -67,7 +72,7 @@ def test_bad_fields( ) -def test_prepare_query_kwargs(): +def test_prepare_query_kwargs() -> None: """Check only valid QueryBuilder arguments are allowed for _prepare_query().""" from aiida_optimade.routers.structures import STRUCTURES @@ -75,7 +80,7 @@ def test_prepare_query_kwargs(): STRUCTURES._prepare_query(node_types=[], **{"wrong_arg": "some_value"}) -def test_array_sort_type(): +def test_array_sort_type() -> None: """Check TypeError is raised if sorting on list value types.""" from aiida_optimade.routers.structures import STRUCTURES diff --git a/tests/server/test_middleware.py b/tests/server/test_middleware.py index 86cc9c2f..1936b0c1 100644 --- a/tests/server/test_middleware.py +++ b/tests/server/test_middleware.py @@ -1,18 +1,23 @@ """Test middleware""" import pytest -from optimade import __api_version__ -# major, major.minor, major.minor.patch -@pytest.mark.parametrize( - "version", - [ +def _get_api_versions() -> list[str]: + from optimade import __api_version__ + + return [ f"v{__api_version__.split('-')[0].split('+')[0].split('.')[0]}", f"v{'.'.join(__api_version__.split('-')[0].split('+')[0].split('.')[:2])}", f"v{__api_version__.split('-')[0].split('+')[0]}", - ], + ] + + +# major, major.minor, major.minor.patch +@pytest.mark.parametrize( + "version", + _get_api_versions(), ) -def test_redirect_docs(version: str): +def test_redirect_docs(version: str) -> None: """Check Open API endpoints redirection Open API docs endpoints from vMAJOR.MINOR.PATCH and vMAJOR.MINOR base URLs should @@ -20,6 +25,8 @@ def test_redirect_docs(version: str): """ from urllib.parse import urljoin, urlparse + from optimade import __api_version__ + from aiida_optimade.utils import OPEN_API_ENDPOINTS from .utils import client_factory diff --git a/tests/server/test_optimade_validation.py b/tests/server/test_optimade_validation.py index 038a712a..a4d7b719 100644 --- a/tests/server/test_optimade_validation.py +++ b/tests/server/test_optimade_validation.py @@ -1,4 +1,12 @@ -def test_with_validator(remote_client): +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .utils import OptimadeTestClient + + +def test_with_validator(remote_client: OptimadeTestClient) -> None: """Validate server""" from optimade.validator import ImplementationValidator @@ -8,12 +16,12 @@ def test_with_validator(remote_client): assert validator.valid -def test_versioned_base_urls(client): +def test_versioned_base_urls(client: OptimadeTestClient) -> None: """Test all expected versioned base URLs responds with 200""" try: import simplejson as json except ImportError: - import json + import json # type: ignore[no-redef] from optimade.server.routers.utils import BASE_URL_PREFIXES diff --git a/tests/server/test_server_misc.py b/tests/server/test_server_misc.py index b02ddbe4..059f2935 100644 --- a/tests/server/test_server_misc.py +++ b/tests/server/test_server_misc.py @@ -1,4 +1,12 @@ -def test_last_modified(get_good_response): +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from .conftest import GetGoodResponse + + +def test_last_modified(get_good_response: GetGoodResponse) -> None: """Ensure last_modified does not change upon requests""" from time import sleep diff --git a/tests/transformers/test_aiida.py b/tests/transformers/test_aiida.py index 4397a94d..eb276cb8 100644 --- a/tests/transformers/test_aiida.py +++ b/tests/transformers/test_aiida.py @@ -1,30 +1,38 @@ -import pytest -from lark.exceptions import VisitError -from optimade.filterparser import LarkParser -from optimade.server.exceptions import BadRequest +from __future__ import annotations -from aiida_optimade.transformers import AiidaTransformer +from typing import TYPE_CHECKING -VERSION = (1, 1, 0) -VARIANT = "default" +import pytest -PARSER = LarkParser(version=VERSION, variant=VARIANT) -TRANSFORMER = AiidaTransformer() +if TYPE_CHECKING: + from typing import Any -def transform(filter_value: str): +def transform(filter_value: str) -> Any: """Transform `filter` value using TRANSFORMER and PARSER""" + from optimade.filterparser import LarkParser + + from aiida_optimade.transformers import AiidaTransformer + + VERSION = (1, 1, 0) + VARIANT = "default" + + PARSER = LarkParser(version=VERSION, variant=VARIANT) + TRANSFORMER = AiidaTransformer() + return TRANSFORMER.transform(PARSER.parse(filter_value)) -def test_empty(): +def test_empty() -> None: """Check passing "empty" strings""" assert transform(" ") is None assert transform("") is None -def test_property_names(): +def test_property_names() -> None: """Check `property` names""" + from optimade.server.exceptions import BadRequest + assert transform("band_gap = 1") == {"band_gap": {"==": 1}} assert transform("cell_length_a = 1") == {"cell_length_a": {"==": 1}} assert transform("cell_volume = 1") == {"cell_volume": {"==": 1}} @@ -48,7 +56,7 @@ def test_property_names(): } -def test_string_values(): +def test_string_values() -> None: """Check various string values validity""" assert transform('author="Sąžininga Žąsis"') == { "author": {"==": "Sąžininga Žąsis"} @@ -58,8 +66,10 @@ def test_string_values(): } -def test_number_values(): +def test_number_values() -> None: """Check various number values validity""" + from optimade.server.exceptions import BadRequest + assert transform("a = 12345") == {"a": {"==": 12345}} assert transform("b = +12") == {"b": {"==": 12}} assert transform("c = -34") == {"c": {"==": -34}} @@ -90,7 +100,7 @@ def test_number_values(): transform("number=0.0.1") -def test_simple_comparisons(): +def test_simple_comparisons() -> None: """Check simple comparisons""" assert transform("a<3") == {"a": {"<": 3}} assert transform("a<=3") == {"a": {"<=": 3}} @@ -100,7 +110,7 @@ def test_simple_comparisons(): assert transform("a!=3") == {"a": {"!==": 3}} -def test_id(): +def test_id() -> None: """Test `id` valued `property` name""" assert transform('id="example/1"') == {"id": {"==": "example/1"}} assert transform('"example/1" = id') == {"id": {"==": "example/1"}} @@ -109,8 +119,10 @@ def test_id(): } -def test_operators(): +def test_operators() -> None: """Test OPTIMADE filter operators""" + from lark.exceptions import VisitError + # Basic boolean operations # TODO: {"!and": [{"a": {"<": 3}}]} can be simplified to {"a": {">=": 3}} assert transform("NOT a<3") == {"!and": [{"a": {"<": 3}}]} @@ -262,7 +274,7 @@ def test_operators(): @pytest.mark.skip("Relationships have not yet been implemented") -def test_filtering_on_relationships(): +def test_filtering_on_relationships() -> None: """Test the nested properties with special names like "structures", "references" etc. are applied to the relationships field""" @@ -322,9 +334,11 @@ def test_filtering_on_relationships(): # ) -def test_not_implemented(): +def test_not_implemented() -> None: """Test list properties that are currently not implemented give a sensible response""" + from lark.exceptions import VisitError + # NOTE: Lark catches underlying filtertransformer exceptions and # raises VisitErrors, most of these actually correspond to NotImplementedError with pytest.raises(VisitError, match="not been implemented"): @@ -361,7 +375,7 @@ def test_not_implemented(): ) -def test_unaliased_length_operator(): +def test_unaliased_length_operator() -> None: """Check unaliased LENGTH lists""" assert transform("cartesian_site_positions LENGTH 3") == ( @@ -381,7 +395,7 @@ def test_unaliased_length_operator(): ) -def test_list_properties(): +def test_list_properties() -> None: """Test the HAS ALL, ANY and optional ONLY queries""" # NOTE: HAS ONLY has not yet been implemented. # assert transform('elements HAS ONLY "H","He","Ga","Ta"') == ( @@ -421,7 +435,7 @@ def test_list_properties(): # ) -def test_properties(): +def test_properties() -> None: """Filtering on Properties with unknown value""" # The { !and: [{ >: 1.99 }] } is different from the <= operator. # { <=: 1.99 } returns only the documents where price field exists and its @@ -443,7 +457,7 @@ def test_properties(): } -def test_precedence(): +def test_precedence() -> None: """Check OPERATOR precedence""" assert transform('NOT a > b OR c = 100 AND f = "C2 H6"') == ( { @@ -461,7 +475,7 @@ def test_precedence(): ) -def test_special_cases(): +def test_special_cases() -> None: """Check special cases""" assert transform("te < st") == {"te": {"<": "st"}} assert transform('spacegroup="P2"') == {"spacegroup": {"==": "P2"}}