From ef5e0b87b292ada708de5380c063ac8539c4db41 Mon Sep 17 00:00:00 2001 From: khalid Date: Thu, 6 Aug 2026 13:51:50 +0100 Subject: [PATCH 1/5] feat: add custom validation extension func to validator --- src/simdb/validation/validator.py | 43 ++++++++++++++++++++++++++++--- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/src/simdb/validation/validator.py b/src/simdb/validation/validator.py index f9a3bb2b..37640201 100644 --- a/src/simdb/validation/validator.py +++ b/src/simdb/validation/validator.py @@ -200,12 +200,47 @@ def validation_schemas( return schemas - def __init__(self, schema: Dict): + def _custom_validation_ext(self, config: Config): + from importlib import import_module + + module_path = config.get_option("validation.custom_validator", default=None) + if module_path is None: + return CustomValidator + + if not isinstance(module_path, str): + raise TypeError( + f"Expected 'custom_validator config value' to be a string, got {type(module_path).__name__}" + ) + + if "." not in module_path: + raise ValueError( + f"Invalid validator path '{module_path}'. Expected format: 'package.module.ClassName'" + ) + + module_name, class_name = module_path.rsplit(".", 1) + + try: + module = import_module(module_name) + except ModuleNotFoundError as e: + raise ImportError( + f"Unable to import module '{module_name}': {e}. " + "Please ensure the necessary validation package is installed" + ) from e + try: + validation_cls = getattr(module, class_name) + except AttributeError as e: + raise AttributeError( + f"Module '{module_name}' does not have class or attribute '{class_name}'" + ) from e + return validation_cls + + def __init__(self, schema: Dict, config: Config): try: - self._validator = CustomValidator(schema) + validation_cls = self._custom_validation_ext(config) + self._validator = validation_cls(schema) self._validator.allow_unknown = True - except cerberus.SchemaError as err: - raise LoadError("Failed to parse validation schema") from err + except cerberus.SchemaError: + raise LoadError("Failed to parse validation schema") def validate(self, sim: Simulation) -> None: # convert sim to dictionary From 280fc29c1609d54c6b8a5f6c540519f4708dc8ba Mon Sep 17 00:00:00 2001 From: khalid Date: Thu, 6 Aug 2026 13:53:03 +0100 Subject: [PATCH 2/5] fix usage of Validator class to include config option --- src/simdb/cli/commands/simulation.py | 4 ++-- src/simdb/remote/apis/v1/simulations.py | 2 +- src/simdb/remote/apis/v1_1/simulations.py | 2 +- src/simdb/remote/apis/v1_2/simulations.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/simdb/cli/commands/simulation.py b/src/simdb/cli/commands/simulation.py index 59bd0bdd..d7327c2a 100644 --- a/src/simdb/cli/commands/simulation.py +++ b/src/simdb/cli/commands/simulation.py @@ -248,7 +248,7 @@ def simulation_push( schemas = api.get_validation_schemas() try: for schema in schemas: - Validator(schema).validate(simulation) + Validator(schema, config).validate(simulation) except ValidationError as err: raise click.ClickException(f"Simulation does not validate: {err}") from err @@ -471,7 +471,7 @@ def simulation_validate( click.echo("validating metadata ... ", nl=False) for schema in schemas: - Validator(schema).validate(simulation) + Validator(schema, config).validate(simulation) ids_list = [] for file in chain(simulation.inputs, simulation.outputs): diff --git a/src/simdb/remote/apis/v1/simulations.py b/src/simdb/remote/apis/v1/simulations.py index 3a56c9cb..c22acfa4 100644 --- a/src/simdb/remote/apis/v1/simulations.py +++ b/src/simdb/remote/apis/v1/simulations.py @@ -52,7 +52,7 @@ def _update_simulation_status( def _validate(simulation, user) -> Dict: schema = Validator.validation_schema() try: - Validator(schema).validate(simulation) + Validator(schema, current_app.simdb_config).validate(simulation) _update_simulation_status(simulation, models_sim.Simulation.Status.PASSED, user) return { "passed": True, diff --git a/src/simdb/remote/apis/v1_1/simulations.py b/src/simdb/remote/apis/v1_1/simulations.py index 03c956b9..105b92e9 100644 --- a/src/simdb/remote/apis/v1_1/simulations.py +++ b/src/simdb/remote/apis/v1_1/simulations.py @@ -53,7 +53,7 @@ def _validate(simulation, user) -> Dict: schemas = Validator.validation_schemas(current_app.simdb_config, simulation) try: for schema in schemas: - Validator(schema).validate(simulation) + Validator(schema, current_app.simdb_config).validate(simulation) _update_simulation_status( simulation, models_sim.Simulation.Status.PASSED, user ) diff --git a/src/simdb/remote/apis/v1_2/simulations.py b/src/simdb/remote/apis/v1_2/simulations.py index a8594490..b55719af 100644 --- a/src/simdb/remote/apis/v1_2/simulations.py +++ b/src/simdb/remote/apis/v1_2/simulations.py @@ -82,7 +82,7 @@ def _validate(simulation, user) -> ValidationResult: schemas = Validator.validation_schemas(current_app.simdb_config, simulation) try: for schema in schemas: - Validator(schema).validate(simulation) + Validator(schema, current_app.simdb_config).validate(simulation) _update_simulation_status( simulation, models_sim.Simulation.Status.PASSED, user ) From 62433b828557496de9dcb435aceb65d204161163 Mon Sep 17 00:00:00 2001 From: khalid Date: Thu, 6 Aug 2026 17:04:50 +0100 Subject: [PATCH 3/5] fix: lenghty lines failing ruff --- src/simdb/validation/validator.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/simdb/validation/validator.py b/src/simdb/validation/validator.py index 37640201..184b6db3 100644 --- a/src/simdb/validation/validator.py +++ b/src/simdb/validation/validator.py @@ -209,12 +209,14 @@ def _custom_validation_ext(self, config: Config): if not isinstance(module_path, str): raise TypeError( - f"Expected 'custom_validator config value' to be a string, got {type(module_path).__name__}" + "Expected 'custom_validator config value' to be a string, " + f"got {type(module_path).__name__}" ) if "." not in module_path: raise ValueError( - f"Invalid validator path '{module_path}'. Expected format: 'package.module.ClassName'" + f"Invalid validator path '{module_path}'." + "Expected format: 'package.module.ClassName'" ) module_name, class_name = module_path.rsplit(".", 1) @@ -230,7 +232,8 @@ def _custom_validation_ext(self, config: Config): validation_cls = getattr(module, class_name) except AttributeError as e: raise AttributeError( - f"Module '{module_name}' does not have class or attribute '{class_name}'" + f"Module '{module_name}' does not have class" + f"or attribute '{class_name}'" ) from e return validation_cls From 48a18e6f7701783961dade03a86aa325a2ec69d0 Mon Sep 17 00:00:00 2001 From: khalid Date: Fri, 7 Aug 2026 10:07:42 +0100 Subject: [PATCH 4/5] fix: reformat validator file --- src/simdb/validation/validator.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/simdb/validation/validator.py b/src/simdb/validation/validator.py index 184b6db3..f00555d8 100644 --- a/src/simdb/validation/validator.py +++ b/src/simdb/validation/validator.py @@ -232,8 +232,7 @@ def _custom_validation_ext(self, config: Config): validation_cls = getattr(module, class_name) except AttributeError as e: raise AttributeError( - f"Module '{module_name}' does not have class" - f"or attribute '{class_name}'" + f"Module '{module_name}' does not have classor attribute '{class_name}'" ) from e return validation_cls From 878f9e7b851325e58d209b56fe65ab8f9506f5a6 Mon Sep 17 00:00:00 2001 From: khalid Date: Fri, 7 Aug 2026 11:31:55 +0100 Subject: [PATCH 5/5] add ruff fixes --- src/simdb/cli/remote_api.py | 2 +- src/simdb/database/models/simulation.py | 2 +- src/simdb/validation/file/ids_validator.py | 2 +- src/simdb/validation/validator.py | 16 ++++++++-------- tests/remote/api/conftest.py | 1 - tests/remote/test_authentication.py | 1 - 6 files changed, 11 insertions(+), 13 deletions(-) diff --git a/src/simdb/cli/remote_api.py b/src/simdb/cli/remote_api.py index 99f87363..9b569b62 100644 --- a/src/simdb/cli/remote_api.py +++ b/src/simdb/cli/remote_api.py @@ -42,7 +42,7 @@ from .manifest import DataType if TYPE_CHECKING: - from simdb.database.models import File, Simulation, Watcher + from simdb.database.models import File, Watcher if TYPE_CHECKING or "sphinx" in sys.modules: # Only importing these for type checking and documentation generation in order to diff --git a/src/simdb/database/models/simulation.py b/src/simdb/database/models/simulation.py index ede09fdf..57db96f1 100644 --- a/src/simdb/database/models/simulation.py +++ b/src/simdb/database/models/simulation.py @@ -163,7 +163,7 @@ def meta(self) -> List[MetaDataWrapper]: Returns a list of MetaDataWrapper objects from the JSON metadata. """ meta_dict = self._get_metadata_dict() - return [MetaDataWrapper(k, v) for k, v in meta_dict.items()] + return list(itertools.starmap(MetaDataWrapper, meta_dict.items())) def _get_metadata_dict(self) -> Dict[str, Any]: if self._metadata is None: diff --git a/src/simdb/validation/file/ids_validator.py b/src/simdb/validation/file/ids_validator.py index 07452a41..9a7714bf 100644 --- a/src/simdb/validation/file/ids_validator.py +++ b/src/simdb/validation/file/ids_validator.py @@ -43,7 +43,7 @@ def configure(self, arguments: dict): .split(",") ] - ### Define logic for rule_filter + # Define logic for rule_filter list_of_filter_names = ( arguments.get("rule_filter_name", "").strip('"').split(",") ) diff --git a/src/simdb/validation/validator.py b/src/simdb/validation/validator.py index f00555d8..3decf052 100644 --- a/src/simdb/validation/validator.py +++ b/src/simdb/validation/validator.py @@ -1,5 +1,6 @@ import re import warnings +from importlib import import_module from pathlib import Path from typing import Any, Dict, List, Optional, Union, cast @@ -201,7 +202,6 @@ def validation_schemas( return schemas def _custom_validation_ext(self, config: Config): - from importlib import import_module module_path = config.get_option("validation.custom_validator", default=None) if module_path is None: @@ -223,17 +223,17 @@ def _custom_validation_ext(self, config: Config): try: module = import_module(module_name) - except ModuleNotFoundError as e: + except ModuleNotFoundError as err: raise ImportError( - f"Unable to import module '{module_name}': {e}. " + f"Unable to import module '{module_name}': {err}. " "Please ensure the necessary validation package is installed" - ) from e + ) from err try: validation_cls = getattr(module, class_name) - except AttributeError as e: + except AttributeError as err: raise AttributeError( f"Module '{module_name}' does not have classor attribute '{class_name}'" - ) from e + ) from err return validation_cls def __init__(self, schema: Dict, config: Config): @@ -241,8 +241,8 @@ def __init__(self, schema: Dict, config: Config): validation_cls = self._custom_validation_ext(config) self._validator = validation_cls(schema) self._validator.allow_unknown = True - except cerberus.SchemaError: - raise LoadError("Failed to parse validation schema") + except cerberus.SchemaError as err: + raise LoadError("Failed to parse validation schema") from err def validate(self, sim: Simulation) -> None: # convert sim to dictionary diff --git a/tests/remote/api/conftest.py b/tests/remote/api/conftest.py index 4310350c..da2c7b17 100644 --- a/tests/remote/api/conftest.py +++ b/tests/remote/api/conftest.py @@ -1,5 +1,4 @@ import base64 -import importlib import importlib.util import os import shutil diff --git a/tests/remote/test_authentication.py b/tests/remote/test_authentication.py index d62f56a3..b5522092 100644 --- a/tests/remote/test_authentication.py +++ b/tests/remote/test_authentication.py @@ -1,4 +1,3 @@ -import importlib import importlib.util from typing import TYPE_CHECKING, ClassVar, cast from unittest import mock