diff --git a/.github/workflows/doc.yml b/.github/workflows/doc.yml index dd0284f..4528c24 100644 --- a/.github/workflows/doc.yml +++ b/.github/workflows/doc.yml @@ -1,33 +1,30 @@ -name: Python package -on: [push] +name: Documentation + +on: [push, pull_request] + jobs: build: + name: Build Documentation runs-on: ubuntu-latest timeout-minutes: 60 strategy: matrix: - python-version: ['3.10'] - # python-version: ['3.10'] + python-version: ["3.10"] steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install Pandoc - run: | - sudo apt-get install pandoc - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install cython - pip install ".[dev]" - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 + with: + enable-cache: true + python-version: ${{ matrix.python-version }} + + - name: Install Pandoc + run: | + sudo apt-get update + sudo apt-get install -y pandoc - - name: Test doc - run: | - make doc \ No newline at end of file + - name: Build documentation + run: | + uv run --extra doc make doc \ No newline at end of file diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index bf6e4d0..c03ea1f 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -1,33 +1,42 @@ -name: Python package -on: [push] +name: Lint and Test + +on: [push, pull_request] + jobs: - build: + # lint: + # name: Lint with Ruff + # runs-on: ubuntu-latest + # steps: + # - name: Checkout code + # uses: actions/checkout@v7 + + # - name: Set up uv + # uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 + # with: + # enable-cache: true + # python-version: "3.12" + + # - name: Lint + # run: | + # uv run --group dev make lint + + test: + name: Test on Python ${{ matrix.python-version }} runs-on: ubuntu-latest timeout-minutes: 60 strategy: matrix: - python-version: ['3.8', '3.9', '3.10', '3.11'] - # python-version: ['3.10'] + python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install cython - pip install ".[dev]" - # pip install cython poetry - # poetry config virtualenvs.create false - # poetry install --no-interaction --with dev - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - make test \ No newline at end of file + - name: Checkout code + uses: actions/checkout@v7 + + - name: Set up uv + uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 + with: + enable-cache: true + python-version: ${{ matrix.python-version }} + + - name: Run tests + run: | + uv run --extra test make test \ No newline at end of file diff --git a/.gitignore b/.gitignore index a100487..e5bb792 100644 --- a/.gitignore +++ b/.gitignore @@ -229,3 +229,5 @@ generated_datasets/* !generated_datasets/71GPa_experimental_data.hspy results .DS_Store +espm/_version.py + diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..6177a68 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,10 @@ +repos: +- repo: https://github.com/astral-sh/ruff-pre-commit + # Ruff version. + rev: v0.15.20 + hooks: + # Run the linter. + - id: ruff-check + args: [ --fix, --exit-zero ] + # Run the formatter. + - id: ruff-format diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 4d5c065..a598574 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -14,4 +14,4 @@ python: - method: pip path: . extra_requirements: - - dev \ No newline at end of file + - doc \ No newline at end of file diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 78eac1c..e07b1f6 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -17,8 +17,7 @@ development with the following:: $ pip install -e .[dev] You can improve or add functionality in the ``espm`` folder, along with -corresponding unit tests in ``espm/tests/test_*.py`` (with reasonable -coverage). +corresponding unit tests in ``tests/test_*.py`` (with reasonable coverage). If you have a nice example to demonstrate the use of the introduced functionality, please consider adding a notebook in ``doc/introduction/notebooks``. In general, this can be done as a relative symbolic link to a notebook in the @@ -35,8 +34,8 @@ documentation with the following :: To iterate faster, you can partially run the test suite, at various degrees of granularity, as follows:: - $ pytest espm.test.test_utils.py - $ pytest espm.test.test_estimators.py + $ pytest tests/test_utils.py + $ pytest tests/test_estimators.py Making a release ---------------- @@ -45,8 +44,7 @@ Making a release #. Log to `readthedoc `, launch a build on latest and check that it can compile the latest version of the documentation. -#. Update the version number and release date in ``setup.py``, - ``espm/__init__.py`` and ``CHANGELOG.rst``. +#. Update the release notes and release date in ``CHANGELOG.rst``. #. Commit the changes with ``git commit -am "Bump to version 0.2.0"``. #. Create a git tag with ``git tag -a v0.2.0 -m "espm v0.2.0"``. #. Push the tag to GitHub with ``git push --tag``. The tag should now @@ -72,22 +70,20 @@ Repository organization LICENSE.txt Project license *.rst Important documentation Makefile Targets for make - setup.py Meta information about package (published on PyPI) + pyproject.toml Meta information about package, dependencies and tools (Ruff, pytest) .gitignore Files ignored by the git revision control system - .travis.yml Defines testing on Travis continuous integration - espm/ Contains the modules (the actual toolbox implementation) + espm/ Contains the modules (the actual toolbox implementation) __init.py__ Load modules at package import *.py One file per module - espm/tables/ Contains data tables for the EDXS and EDSS models + espm/tables/ Contains data tables for the EDXS and EDSS models __init.py__ Load modules at package import *.json data_files for the package *.txt data_files for the package - espm/tests/ Contains the test suites (will be distributed to end user) - __init.py__ Load modules at package import + tests/ Contains the test suites (not distributed to end user) test_*.py One test suite per module - test_docstrings.py Test the examples in the docstrings (reference doc) + test_docstring.py Test the examples in the docstrings (reference doc) doc/ Package documentation conf.py Sphinx configuration diff --git a/Makefile b/Makefile index 23b975c..8c27788 100644 --- a/Makefile +++ b/Makefile @@ -10,11 +10,11 @@ help: @echo "release package and upload to PyPI" clean: - git clean -Xdf + rm -rf build/ dist/ *.egg-info/ .pytest_cache/ .ruff_cache/ doc/_build/ jupyter nbconvert --inplace --ClearOutputPreprocessor.enabled=True $(NB) lint: - flake8 --doctests --exclude=doc + ruff check # Matplotlib doesn't print to screen. Also faster. export MPLBACKEND = agg @@ -22,7 +22,7 @@ export MPLBACKEND = agg export DISPLAY = :99 test: - pytest espm + pytest clean-doc: rm -rf doc/_build @@ -32,8 +32,7 @@ doc: sphinx-build -b linkcheck -d doc/_build/doctrees doc doc/_build/linkcheck dist: clean - python setup.py sdist - python setup.py bdist_wheel --universal + python -m build ls -lh dist/* twine check dist/* diff --git a/README.rst b/README.rst index b093f21..bc45306 100644 --- a/README.rst +++ b/README.rst @@ -28,8 +28,7 @@ If you want to develop, please use the option:: $ git clone https://github.com/adriente/espm.git $ cd espm - $ pip install cython - $ pip install -e .[dev] + $ pip install -e .[dev] If you get issues regarding pandoc when using `make doc`, you can install it using:: @@ -73,23 +72,23 @@ CITING If you use this library, please cite on of the following papers:: @article{teurtrie2023espm, - title={espm: A Python library for the simulation of STEM-EDXS datasets}, - author={Teurtrie, Adrien and Perraudin, Nathana{\"e}l and Holvoet, Thomas and Chen, Hui and Alexander, Duncan TL and Obozinski, Guillaume and H{\'e}bert, C{\'e}cile}, - journal={Ultramicroscopy}, - pages={113719}, - year={2023}, - publisher={Elsevier} + title={espm: A Python library for the simulation of STEM-EDXS datasets}, + author={Teurtrie, Adrien and Perraudin, Nathana{\"e}l and Holvoet, Thomas and Chen, Hui and Alexander, Duncan TL and Obozinski, Guillaume and H{\'e}bert, C{\'e}cile}, + journal={Ultramicroscopy}, + pages={113719}, + year={2023}, + publisher={Elsevier} } @article{Teurtrie_2024, - doi = {10.1088/2632-2153/ad9192}, - year = {2024}, - month = {nov}, - publisher = {IOP Publishing}, - volume = {5}, - number = {4}, - pages = {045050}, - author = {Teurtrie, Adrien and Perraudin, Nathanaël and Holvoet, Thomas and Chen, Hui and Alexander, Duncan T L and Obozinski, Guillaume and Hébert, Cécile}, - title = {From STEM-EDXS data to phase separation and quantification using physics-guided NMF}, - journal = {Machine Learning: Science and Technology} + doi = {10.1088/2632-2153/ad9192}, + year = {2024}, + month = {nov}, + publisher = {IOP Publishing}, + volume = {5}, + number = {4}, + pages = {045050}, + author = {Teurtrie, Adrien and Perraudin, Nathanaël and Holvoet, Thomas and Chen, Hui and Alexander, Duncan T L and Obozinski, Guillaume and Hébert, Cécile}, + title = {From STEM-EDXS data to phase separation and quantification using physics-guided NMF}, + journal = {Machine Learning: Science and Technology} } diff --git a/doc/conf.py b/doc/conf.py index a40818a..13c18a6 100644 --- a/doc/conf.py +++ b/doc/conf.py @@ -1,45 +1,41 @@ -# -*- coding: utf-8 -*- - import espm extensions = [ - 'sphinx.ext.viewcode', - 'sphinx.ext.autosummary', - 'sphinx.ext.mathjax', - 'sphinx.ext.inheritance_diagram', - 'nbsphinx', - 'sphinx_gallery.load_style', - 'IPython.sphinxext.ipython_console_highlighting', + "sphinx.ext.viewcode", + "sphinx.ext.autosummary", + "sphinx.ext.mathjax", + "sphinx.ext.inheritance_diagram", + "nbsphinx", + "sphinx_gallery.load_style", + "IPython.sphinxext.ipython_console_highlighting", ] -extensions.append('sphinx.ext.autodoc') +extensions.append("sphinx.ext.autodoc") autodoc_default_options = { - 'members': True, - 'undoc-members': True, - 'member-order': 'groupwise', # alphabetical, groupwise, bysource + "members": True, + "undoc-members": True, + "member-order": "groupwise", # alphabetical, groupwise, bysource } -extensions.append('sphinx.ext.intersphinx') +extensions.append("sphinx.ext.intersphinx") intersphinx_mapping = { - 'python': ('https://docs.python.org/3', None), - 'numpy': ('https://numpy.org/doc/stable', None), - 'scipy': ('https://docs.scipy.org/doc/scipy/reference', None), - 'matplotlib': ('https://matplotlib.org/stable', None), - 'hyperspy': ('https://hyperspy.readthedocs.io/en/stable', None), + "python": ("https://docs.python.org/3", None), + "numpy": ("https://numpy.org/doc/stable", None), + "scipy": ("https://docs.scipy.org/doc/scipy/reference", None), + "matplotlib": ("https://matplotlib.org/stable", None), + "hyperspy": ("https://hyperspy.readthedocs.io/en/stable", None), } -extensions.append('numpydoc') +extensions.append("numpydoc") numpydoc_show_class_members = False numpydoc_use_plots = True # Add the plot directive whenever mpl is imported. -extensions.append('matplotlib.sphinxext.plot_directive') +extensions.append("matplotlib.sphinxext.plot_directive") plot_include_source = True plot_html_show_source_link = False plot_html_show_formats = False -plot_working_directory = '.' -plot_rcparams = { - 'figure.figsize': (10, 4) -} +plot_working_directory = "." +plot_rcparams = {"figure.figsize": (10, 4)} plot_pre_code = """ import numpy as np import matplotlib.pyplot as plt @@ -48,31 +44,36 @@ """ -extensions.append('sphinx_copybutton') +extensions.append("sphinx_copybutton") copybutton_prompt_text = ">>> " -extensions.append('sphinxcontrib.bibtex') -bibtex_bibfiles = ['references.bib'] +extensions.append("sphinxcontrib.bibtex") +bibtex_bibfiles = ["references.bib"] -exclude_patterns = ['_build'] -source_suffix = '.rst' -master_doc = 'index' +exclude_patterns = ["_build"] +source_suffix = ".rst" +master_doc = "index" -project = 'espm' +project = "espm" version = espm.__version__ release = espm.__version__ -copyright = 'Adrien Teurtrie and Nathanael Perraudin' +copyright = "Adrien Teurtrie and Nathanael Perraudin" -pygments_style = 'sphinx' -html_theme = 'sphinx_rtd_theme' +pygments_style = "sphinx" +html_theme = "sphinx_rtd_theme" html_theme_options = { - 'navigation_depth': 2, + "navigation_depth": 2, } latex_elements = { - 'papersize': 'a4paper', - 'pointsize': '10pt', + "papersize": "a4paper", + "pointsize": "10pt", } latex_documents = [ - ('index', 'espm.tex', 'espm documentation', - 'Adrien Teurtrie and Nathanael Perraudin', 'manual'), + ( + "index", + "espm.tex", + "espm documentation", + "Adrien Teurtrie and Nathanael Perraudin", + "manual", + ), ] diff --git a/espm/__init__.py b/espm/__init__.py index f74f503..7dd1069 100644 --- a/espm/__init__.py +++ b/espm/__init__.py @@ -1 +1,4 @@ -__version__ = "1.1.3" \ No newline at end of file +try: + from ._version import version as __version__ +except ImportError: + __version__ = "unknown" diff --git a/espm/conf.py b/espm/conf.py index f054190..d80ce11 100644 --- a/espm/conf.py +++ b/espm/conf.py @@ -1,7 +1,9 @@ from pathlib import Path + import exspy._misc.eds.ffast_mac as macs + # from exspy.misc.eds import ffast_mac as macs -HSPY_MAC = macs.ffast_mac # Tabulated absorption coefficient in Hyperspy +HSPY_MAC = macs.ffast_mac # Tabulated absorption coefficient in Hyperspy # Path of the base BASE_PATH = Path(__file__).parent @@ -23,33 +25,32 @@ DATASETS_PATH.mkdir(exist_ok=True, parents=True) - DEFAULT_MISC_PARAMS = { - "data_folder" : "default_synth_data", - "shape_2d" : (80,80), - "N" : 100, - "densities" : [2.33,19.3], - "model" : "EDXS", - "seed" : 0 + "data_folder": "default_synth_data", + "shape_2d": (80, 80), + "N": 100, + "densities": [2.33, 19.3], + "model": "EDXS", + "seed": 0, } DEFAULT_EDXS_PARAMS = { - "e_offset" : 0.200, - "e_size" : 1980, - "e_scale" : 0.01, - "width_slope" : 0.01, - "width_intercept" : 0.065, - "db_name" : "200keV_xrays.json", - "E0" : 200, - "params_dict" : { - "Det" : DEFAULT_SDD_EFF, - "Abs" : { - "thickness" : 100.0e-7, - "toa" : 22, - "density" : None, - "atomic_fraction" : False - } - } + "e_offset": 0.200, + "e_size": 1980, + "e_scale": 0.01, + "width_slope": 0.01, + "width_intercept": 0.065, + "db_name": "200keV_xrays.json", + "E0": 200, + "params_dict": { + "Det": DEFAULT_SDD_EFF, + "Abs": { + "thickness": 100.0e-7, + "toa": 22, + "density": None, + "atomic_fraction": False, + }, + }, } log_shift = 1e-14 diff --git a/espm/cross_sections_utils.py b/espm/cross_sections_utils.py index f6de400..32f6315 100644 --- a/espm/cross_sections_utils.py +++ b/espm/cross_sections_utils.py @@ -1,20 +1,25 @@ import json import re -import numpy as np from pathlib import Path -from espm.conf import DB_PATH, SYMBOLS_PERIODIC_TABLE, SIEGBAHN_TO_IUPAC - -def modify_cross_sections(energy : int | str, - input_type : str = 'new_values', - lines_new_values : dict[str,float] = None, - lines_scaling_factors : dict[str,float] = None, - k_factors : dict[str,float] = None, - reference_line : str = None, - output_filename : str = None) -> None: + +import numpy as np + +from espm.conf import DB_PATH, SIEGBAHN_TO_IUPAC, SYMBOLS_PERIODIC_TABLE + + +def modify_cross_sections( + energy: int | str, + input_type: str = "new_values", + lines_new_values: dict[str, float] = None, + lines_scaling_factors: dict[str, float] = None, + k_factors: dict[str, float] = None, + reference_line: str = None, + output_filename: str = None, +) -> None: r""" Function allowing the user to modify the X-ray emission cross-sections of the input database. The function can be with three diffrent input types: new values, scaling factors and k-factors with a reference line. - + Parameters ---------- energy : int @@ -22,164 +27,204 @@ def modify_cross_sections(energy : int | str, input_type : str The type of input to be used. Can be one of the following: 'new_values', 'scaling_factors', 'k_factors'. lines_new_values : dict - [For the 'new_values' input type] A dictionary containing the new values for the cross-sections. + [For the 'new_values' input type] A dictionary containing the new values for the cross-sections. The keys of the dictionary should be the element symbol and the X-ray line in Siegbahn notation, separated by an underscore. - The correspoding values should be the new cross-section values. + The correspoding values should be the new cross-section values. For example: {'Cu_Ka1' : 6.3e-23}. lines_scaling_factors : dict - [For the 'scaling_factors' input type] A dictionary containing the scaling factors for the cross-sections. + [For the 'scaling_factors' input type] A dictionary containing the scaling factors for the cross-sections. The keys of the dictionary should be the element symbol and the X-ray line family in Siegbahn notation, separated by an underscore. The corresponding values should be the scaling factors. For example: {'Cu_Ka' : 1.2}. k_factors : dict - [For the 'k_factors' input type] A dictionary containing the k-factors for the cross-sections. + [For the 'k_factors' input type] A dictionary containing the k-factors for the cross-sections. The keys of the dictionary should be the element symbol and the X-ray line family in Siegbahn notation, separated by an underscore. The corresponding values should be the Cliff-Lorimer k-factors. For example: {'Cu_Ka' : 1.75}. reference_line : str - [For the 'k_factors' input type] The reference line for the k-factors. + [For the 'k_factors' input type] The reference line for the k-factors. Should be the element symbol and the X-ray line family in Siegbahn notation, separated by an underscore. For example: 'Si_Ka'. output_filename : str The name of the output file. If None, the default name is the same as the input file with '_modified' appended. - + Returns ------- None """ - assert not(lines_new_values is None and lines_scaling_factors is None and k_factors is None), f"You need to input a dict of lines to modify with type : {input_type}." - + assert not ( + lines_new_values is None and lines_scaling_factors is None and k_factors is None + ), f"You need to input a dict of lines to modify with type : {input_type}." + if energy == 100: - cross_sections_json = DB_PATH / '100keV_xrays.json' + cross_sections_json = DB_PATH / "100keV_xrays.json" elif energy == 200: - cross_sections_json = DB_PATH / '200keV_xrays.json' + cross_sections_json = DB_PATH / "200keV_xrays.json" elif energy == 300: - cross_sections_json = DB_PATH / '300keV_xrays.json' + cross_sections_json = DB_PATH / "300keV_xrays.json" - with open(cross_sections_json, 'r') as cross_sections_file: + with open(cross_sections_json, "r") as cross_sections_file: cross_sections_data = json.load(cross_sections_file) - with open(SYMBOLS_PERIODIC_TABLE, 'r') as atomic_symbols_file: + with open(SYMBOLS_PERIODIC_TABLE, "r") as atomic_symbols_file: atomic_symbols_data = json.load(atomic_symbols_file) - with open(SIEGBAHN_TO_IUPAC, 'r') as siegbahn_to_iupac_file: + with open(SIEGBAHN_TO_IUPAC, "r") as siegbahn_to_iupac_file: siegbahn_to_iupac_data = json.load(siegbahn_to_iupac_file) - + iupac_to_siegbahn = { iupac: siegbahn for siegbahn, iupac_list in siegbahn_to_iupac_data.items() for iupac in iupac_list } - - if input_type == 'new_values': + + if input_type == "new_values": for el_line, value in lines_new_values.items(): line_match = re.match(r"([A-Z][a-z]?)_(.*)", el_line) element = line_match.group(1) line_name = line_match.group(2) - - atomic_number = str(atomic_symbols_data['table'][element]['number']) + + atomic_number = str(atomic_symbols_data["table"][element]["number"]) line_iupac = siegbahn_to_iupac_data.get(line_name, [line_name])[0] - - if atomic_number in cross_sections_data['table'] and line_iupac in cross_sections_data['table'][atomic_number]: - cross_sections_data['table'][atomic_number][line_iupac]['cs'] = value + + if ( + atomic_number in cross_sections_data["table"] + and line_iupac in cross_sections_data["table"][atomic_number] + ): + cross_sections_data["table"][atomic_number][line_iupac]["cs"] = value line_siegbahn = iupac_to_siegbahn.get(line_iupac, line_iupac) - print(f"Set new cross-section for Element {element}, Line {line_siegbahn} to {value:.2e}") + print( + f"Set new cross-section for Element {element}, Line {line_siegbahn} to {value:.2e}" + ) else: - print(f"Warning: Element {element} or line {line_iupac} not found in the JSON data.") + print( + f"Warning: Element {element} or line {line_iupac} not found in the JSON data." + ) - elif input_type == 'scaling_factors': + elif input_type == "scaling_factors": for el_line, factor in lines_scaling_factors.items(): line_match = re.match(r"([A-Z][a-z]?)_(.*)", el_line) element = line_match.group(1) line_group = line_match.group(2) - atomic_number = str(atomic_symbols_data['table'][element]['number']) + atomic_number = str(atomic_symbols_data["table"][element]["number"]) line_iupac_list = siegbahn_to_iupac_data.get(line_group, []) for line_iupac in line_iupac_list: - if atomic_number in cross_sections_data['table'] and line_iupac in cross_sections_data['table'][atomic_number]: - current_cs = cross_sections_data['table'][atomic_number][line_iupac]['cs'] - cross_sections_data['table'][atomic_number][line_iupac]['cs'] = current_cs * factor + if ( + atomic_number in cross_sections_data["table"] + and line_iupac in cross_sections_data["table"][atomic_number] + ): + current_cs = cross_sections_data["table"][atomic_number][ + line_iupac + ]["cs"] + cross_sections_data["table"][atomic_number][line_iupac]["cs"] = ( + current_cs * factor + ) line_siegbahn = iupac_to_siegbahn.get(line_iupac, line_iupac) - print(f"Applied scaling factor {factor} to cross-section for element {element}, line {line_siegbahn}") - + print( + f"Applied scaling factor {factor} to cross-section for element {element}, line {line_siegbahn}" + ) + else: - print(f"Skipped element {element}, line {line_iupac}: No matching data in database.") + print( + f"Skipped element {element}, line {line_iupac}: No matching data in database." + ) - elif input_type == 'k_factors' and reference_line is not None: + elif input_type == "k_factors" and reference_line is not None: ref_match = re.match(r"([A-Z][a-z]?)_(.*)", reference_line) ref_element = ref_match.group(1) ref_line_family = ref_match.group(2) - - ref_at_num = str(atomic_symbols_data["table"][ref_element]['number']) - + + ref_at_num = str(atomic_symbols_data["table"][ref_element]["number"]) + ref_lines_iupac = siegbahn_to_iupac_data.get(ref_line_family, []) if not ref_lines_iupac: - raise ValueError(f"Reference line family {ref_line_family} not found in Siegbahn to IUPAC mapping.") - - + raise ValueError( + f"Reference line family {ref_line_family} not found in Siegbahn to IUPAC mapping." + ) + valid_ref_lines = { - line: cross_sections_data['table'][ref_at_num][line]['cs'] - for line in ref_lines_iupac if line in cross_sections_data['table'][ref_at_num] + line: cross_sections_data["table"][ref_at_num][line]["cs"] + for line in ref_lines_iupac + if line in cross_sections_data["table"][ref_at_num] } - + if not valid_ref_lines: - raise ValueError(f"No valid reference lines found for {ref_element}_{ref_line_family} in the cross-section data.") - + raise ValueError( + f"No valid reference lines found for {ref_element}_{ref_line_family} in the cross-section data." + ) + for el_line, factor in k_factors.items(): line_match = re.match(r"([A-Z][a-z]?)_(.*)", el_line) element = line_match.group(1) line_family = line_match.group(2) - - atomic_number = str(atomic_symbols_data['table'][element]['number']) + + atomic_number = str(atomic_symbols_data["table"][element]["number"]) target_lines_iupac = siegbahn_to_iupac_data.get(line_family, []) - + if not target_lines_iupac: - raise ValueError(f"Target line family {line_family} not found in Siegbahn to IUPAC mapping.") + raise ValueError( + f"Target line family {line_family} not found in Siegbahn to IUPAC mapping." + ) for ref_line, target_line in zip(ref_lines_iupac, target_lines_iupac): - if ref_line in valid_ref_lines and target_line in cross_sections_data['table'][atomic_number]: + if ( + ref_line in valid_ref_lines + and target_line in cross_sections_data["table"][atomic_number] + ): ref_cs = valid_ref_lines[ref_line] new_value = ref_cs * factor - cross_sections_data['table'][atomic_number][target_line]['cs'] = new_value - + cross_sections_data["table"][atomic_number][target_line]["cs"] = ( + new_value + ) + ref_line_siegbahn = iupac_to_siegbahn.get(ref_line, ref_line) - target_line_siegbahn = iupac_to_siegbahn.get(target_line, target_line) - - print(f"Set new cross-section for element {element}, line {target_line_siegbahn} " - f"based on k-factor {factor} using {ref_element}, line {ref_line_siegbahn} as reference.") + target_line_siegbahn = iupac_to_siegbahn.get( + target_line, target_line + ) + + print( + f"Set new cross-section for element {element}, line {target_line_siegbahn} " + f"based on k-factor {factor} using {ref_element}, line {ref_line_siegbahn} as reference." + ) else: - print(f"Skipped element {element}, line {target_line}: No matching data in reference or target.") - + print( + f"Skipped element {element}, line {target_line}: No matching data in reference or target." + ) + original_files_list = [ - 'SDD_efficiency.txt', - '200keV_xrays.json', - '__init__.py', - 'default_xrays.json', - 'periodic_table_symbols.json', - 'siegbahn_to_iupac.json', - '300keV_xrays.json', - 'periodic_table_number.json', - '100keV_xrays.json' - ] - + "SDD_efficiency.txt", + "200keV_xrays.json", + "__init__.py", + "default_xrays.json", + "periodic_table_symbols.json", + "siegbahn_to_iupac.json", + "300keV_xrays.json", + "periodic_table_number.json", + "100keV_xrays.json", + ] + if output_filename is None: input_filename = cross_sections_json.name - output_filename = input_filename.replace('.json', '_modified.json') - + output_filename = input_filename.replace(".json", "_modified.json") + if output_filename in original_files_list: - raise ValueError("The output filename cannot be the same as one of the original files.") + raise ValueError( + "The output filename cannot be the same as one of the original files." + ) - with open(DB_PATH / output_filename, 'w') as file: - json.dump(cross_sections_data, file, indent = 4) + with open(DB_PATH / output_filename, "w") as file: + json.dump(cross_sections_data, file, indent=4) -def load_table (db_name : str) -> tuple[dict, dict] : + +def load_table(db_name: str) -> tuple[dict, dict]: r""" Load the table and metadata of a json table generated by emtables. @@ -187,7 +232,7 @@ def load_table (db_name : str) -> tuple[dict, dict] : ---------- db_name : str The file name of the table to load. - + Returns ------- table : dict @@ -200,11 +245,18 @@ def load_table (db_name : str) -> tuple[dict, dict] : Call espm.conf.DB_PATH to get the folder of the tables. """ db_path = DB_PATH / Path(db_name) - with open(db_path,"r") as f : + with open(db_path, "r") as f: json_dict = json.load(f) return json_dict["table"], json_dict["metadata"] -def import_k_factors(table : dict, mdata : dict, k_factors_names : list[str], k_factors_values : list[int], ref_name : str) -> tuple[dict,dict] : + +def import_k_factors( + table: dict, + mdata: dict, + k_factors_names: list[str], + k_factors_values: list[int], + ref_name: str, +) -> tuple[dict, dict]: r""" Modify the X-ray emission cross-sections of the input table using the k-factors input, i.e. imposing cross-sections ratios to correspond to the k-factors. The metadata are modified too to keep track of the modifications. @@ -221,7 +273,7 @@ def import_k_factors(table : dict, mdata : dict, k_factors_names : list[str], k_ The list of the values of the k-factors to import. It has to have the same length and ordering as k_factors_names. ref_name : str The name of the X-ray line to use as a reference for the k-factors. It has to correspond to the nomenclature of the hyperspy X-ray lines. - + Returns ------- new_table : dict @@ -230,39 +282,43 @@ def import_k_factors(table : dict, mdata : dict, k_factors_names : list[str], k_ The modified metadata of the table. """ - with open(SYMBOLS_PERIODIC_TABLE,"r") as f : + with open(SYMBOLS_PERIODIC_TABLE, "r") as f: SPT = json.load(f)["table"] - with open(SIEGBAHN_TO_IUPAC,"r") as f : + with open(SIEGBAHN_TO_IUPAC, "r") as f: STI = json.load(f) - for i,name in enumerate(k_factors_names) : - if name == ref_name : - mr = re.match(r"([A-Z][a-z]?)_(.*)",name) + for i, name in enumerate(k_factors_names): + if name == ref_name: + mr = re.match(r"([A-Z][a-z]?)_(.*)", name) ref_at_num = SPT[mr.group(1)]["number"] - ref_lines = STI[mr.group(2)] + ref_lines = STI[mr.group(2)] ref_sig_vals = [] - for l in ref_lines : - if l in table[str(ref_at_num)] : + for l in ref_lines: + if l in table[str(ref_at_num)]: ref_sig_vals.append(table[str(ref_at_num)][l]["cs"]) ref_sig_val = np.mean(ref_sig_vals) ref_k_val = k_factors_values[i] - for i,name in enumerate(k_factors_names) : - m0 = re.match(r"([A-Z][a-z]?)_(.*)",name) - if m0 : + for i, name in enumerate(k_factors_names): + m0 = re.match(r"([A-Z][a-z]?)_(.*)", name) + if m0: at_num = SPT[m0.group(1)]["number"] - lines = STI[m0.group(2)] - for line in lines : - new_k = k_factors_values[i]/ref_k_val - if line in table[str(at_num)] : + lines = STI[m0.group(2)] + for line in lines: + new_k = k_factors_values[i] / ref_k_val + if line in table[str(at_num)]: sig_val = table[str(at_num)][line]["cs"] - new_value = ref_sig_val*new_k/sig_val - new_table, new_mdata = modify_table_lines(table,mdata,[at_num],line,new_value) - return new_table,new_mdata - + new_value = ref_sig_val * new_k / sig_val + new_table, new_mdata = modify_table_lines( + table, mdata, [at_num], line, new_value + ) + return new_table, new_mdata + -def modify_table_lines (table : dict, mdata : dict, elements : list[str], line : str, coeff : float) -> tuple[dict,dict] : +def modify_table_lines( + table: dict, mdata: dict, elements: list[str], line: str, coeff: float +) -> tuple[dict, dict]: r""" Modify the cross section of the lines of the selected elements in the input table. @@ -278,7 +334,7 @@ def modify_table_lines (table : dict, mdata : dict, elements : list[str], line : The regex of the line to modify. It has to correspond to IUPAC notation. coeff : float The coefficient to multiply the cross section of the selected lines. - + Returns ------- new_table : dict @@ -289,24 +345,25 @@ def modify_table_lines (table : dict, mdata : dict, elements : list[str], line : Notes ----- X-ray line regex examples : input "L" will modify all the L lines, input "L3" will modifiy all the L3 lines, - input "L3M2" will modify the "L3M2" line. - """ - if mdata["lines"] : - for elt in elements : - for key in table[str(elt)].keys() : - if re.match(r"^{}".format(line),key) : - table[str(elt)][key]["cs"] *=coeff - if "modifications" in mdata : + input "L3M2" will modify the "L3M2" line. + """ + if mdata["lines"]: + for elt in elements: + for key in table[str(elt)].keys(): + if re.match(rf"^{line}", key): + table[str(elt)][key]["cs"] *= coeff + if "modifications" in mdata: mdata["modifications"][str(elt) + "_" + key] = coeff - else : + else: mdata["modifications"] = {} mdata["modifications"][str(elt) + "_" + key] = coeff - - else : + + else: print("You need to enable line notation") return table, mdata -def save_table (filename : str, table : dict, mdata : dict) -> None : + +def save_table(filename: str, table: dict, mdata: dict) -> None: r""" Saves a table and its metadata in a json file. The structure of the json file is compliant with espm. @@ -314,10 +371,20 @@ def save_table (filename : str, table : dict, mdata : dict) -> None : d = {} d["table"] = table d["metadata"] = mdata - with open(filename,"w") as f : - json.dump(d,f,indent = 4) - -def get_k_factor (table : dict, mdata : dict, element : int, line : str, range : float = 0.5, ref_elt : str = "14", ref_line : str = "KL3", ref_range : float = 0.5) : + with open(filename, "w") as f: + json.dump(d, f, indent=4) + + +def get_k_factor( + table: dict, + mdata: dict, + element: int, + line: str, + range: float = 0.5, + ref_elt: str = "14", + ref_line: str = "KL3", + ref_range: float = 0.5, +): r""" Obtain the k-factor of a line from an emtables, X-ray emission cross section table. @@ -339,7 +406,7 @@ def get_k_factor (table : dict, mdata : dict, element : int, line : str, range : The regex of the line to use as a reference for the k-factor. It has to correspond to IUPAC notation. ref_range : float The energy range to use for the integration of the cross section of the reference line. - + Returns ------- k_factor : float @@ -347,20 +414,20 @@ def get_k_factor (table : dict, mdata : dict, element : int, line : str, range : """ ref_cs = 0.0 cs = 0.0 - if mdata["lines"] : + if mdata["lines"]: ref_en = table[str(ref_elt)][ref_line]["energy"] - for key in table[str(ref_elt)].keys() : + for key in table[str(ref_elt)].keys(): en = table[str(ref_elt)][key]["energy"] - if (en < ref_en + ref_range) and (en > ref_en - ref_range) : + if (en < ref_en + ref_range) and (en > ref_en - ref_range): ref_cs += table[str(ref_elt)][key]["cs"] elt_en = table[str(element)][line]["energy"] - for key in table[str(element)].keys() : + for key in table[str(element)].keys(): en = table[str(element)][key]["energy"] - if (en < elt_en + range) and (en > elt_en - range) : + if (en < elt_en + range) and (en > elt_en - range): cs += table[str(element)][key]["cs"] - - else : + + else: print("You need to enable line notation") - return cs/ref_cs \ No newline at end of file + return cs / ref_cs diff --git a/espm/datasets/__init__.py b/espm/datasets/__init__.py index 39ff669..fd9adaa 100644 --- a/espm/datasets/__init__.py +++ b/espm/datasets/__init__.py @@ -1,11 +1,11 @@ r""" The :mod:`espm.datasets` module implements the functions that combines the spatial distributions generated from :mod:`espm.weights` and the spectra generated from :mod:`espm.models` into a 3D dataset. -This part of the espm package manages the integration into the hyperspy framework : +This part of the espm package manages the integration into the hyperspy framework : - the datasets and their metadata are stored as hyperspy signals (.hspy). - the :mod:`espm.eds_spim` module implements the :class:`EDSespm` class, which is a subclass of the :class:`hyperspy.signals.Signal1D` class. Using the :class:`EDSespm` class, the user can easily use most of the hyperspy functionalities (e.g. plotting, fitting, decomposition, etc.) as well as the espm functionalites on their experimental and simulated data. - + .. note:: For now espm supports only the signals modeled as EDS data but we aim at implementing the signals corresponding to EELS data too. @@ -13,5 +13,5 @@ """ from espm.datasets.base import generate_dataset +from espm.datasets.built_in_EDXS_datasets import * from espm.datasets.eds_spim import EDSespm -from espm.datasets.built_in_EDXS_datasets import * \ No newline at end of file diff --git a/espm/datasets/base.py b/espm/datasets/base.py index 53a48ea..8c6d4c1 100644 --- a/espm/datasets/base.py +++ b/espm/datasets/base.py @@ -2,15 +2,17 @@ The module :mod:`espm.datasets.base` implements the functions that combines a spatial distribution and associated spectra into a 3D dataset. It also implements the functions to convert the dataset into hyperspy compatible objects. """ -from espm import models -import numpy as np -from espm.conf import DATASETS_PATH from pathlib import Path -from tqdm import tqdm + import hyperspy.api as hs +import numpy as np +from tqdm import tqdm + +from espm import models +from espm.conf import DATASETS_PATH -def generate_spim(phases, weights, densities, N, seed=0,continuous = False): +def generate_spim(phases, weights, densities, N, seed=0, continuous=False): r""" Generate a noiseless spectrum image as tensor product of the phases and weights. Then, if asked for, a noisy spectrum image is generated by drawing from a Poisson distribution. @@ -21,7 +23,7 @@ def generate_spim(phases, weights, densities, N, seed=0,continuous = False): Y^{nl} = N D \otimes ( Diag(d) A ) where :math:`D` is the normalized phases, :math:`A` is the weights, :math:`d` is the density modifier and :math:`N` is the number of counts per pixel. - + To obtain the noisy spectrum image, the noiseless spectrum image is drawn from a Poisson distribution. Parameters @@ -38,7 +40,7 @@ def generate_spim(phases, weights, densities, N, seed=0,continuous = False): Seed for the random number generator. The default is 0. continuous : bool, optional If True, the function returns a noiseless spectrum image. The default is False. - + Returns ------- numpy.ndarray @@ -54,7 +56,7 @@ def generate_spim(phases, weights, densities, N, seed=0,continuous = False): shape_2d = weights.shape[:2] phases = phases / np.sum(phases, axis=1, keepdims=True) - + # n D W A continuous_spim = N * ( weights.reshape(-1, weights.shape[-1]) @@ -64,9 +66,9 @@ def generate_spim(phases, weights, densities, N, seed=0,continuous = False): if continuous: return continuous_spim - else : + else: return np.random.poisson(continuous_spim) - + # # This is probably a very inefficient way to generate the data... # stochastic_spim = np.zeros([*shape_2d, spectral_len]) # for k, w in enumerate(densities): @@ -85,10 +87,10 @@ def generate_spim(phases, weights, densities, N, seed=0,continuous = False): # return stochastic_spim -def sample_to_EDSespm(sample,elements = []) : +def sample_to_EDSespm(sample, elements=[]): # data2spim r"""Convert dataset to a custom hyperspy signal type called EDSespm containing the noisy spectrum image as data, the ground truth as metadata and other useful information. - + Parameters ---------- sample : dict @@ -99,30 +101,38 @@ def sample_to_EDSespm(sample,elements = []) : Returns ------- EDSespm - The hyperspy compatible signal object of the :mod:`espm.eds_spim` module. + The hyperspy compatible signal object of the :mod:`espm.eds_spim` module. """ s = hs.signals.Signal1D(sample["X"]) s.set_signal_type("EDS_espm_Simulated") model_params = sample["model_parameters"] - misc_parameters = sample['misc_parameters'] + misc_parameters = sample["misc_parameters"] s.metadata.Truth = {} - + s.axes_manager[-1].offset = model_params["e_offset"] s.axes_manager[-1].scale = model_params["e_scale"] s.axes_manager[-1].units = "keV" - s.set_microscope_parameters(beam_energy = model_params["E0"]) + s.set_microscope_parameters(beam_energy=model_params["E0"]) s.metadata.Sample = {} s.metadata.Sample.thickness = model_params["params_dict"]["Abs"]["thickness"] s.metadata.Sample.density = model_params["params_dict"]["Abs"]["density"] s.metadata.Sample.elements = elements - s.metadata.Acquisition_instrument.TEM.Detector.EDS.type = model_params["params_dict"]["Det"] - s.metadata.Acquisition_instrument.TEM.Detector.EDS.take_off_angle = model_params["params_dict"]["Abs"]["toa"] - s.metadata.Acquisition_instrument.TEM.Detector.EDS.width_slope = model_params["width_slope"] - s.metadata.Acquisition_instrument.TEM.Detector.EDS.width_intercept = model_params["width_intercept"] + s.metadata.Acquisition_instrument.TEM.Detector.EDS.type = model_params[ + "params_dict" + ]["Det"] + s.metadata.Acquisition_instrument.TEM.Detector.EDS.take_off_angle = model_params[ + "params_dict" + ]["Abs"]["toa"] + s.metadata.Acquisition_instrument.TEM.Detector.EDS.width_slope = model_params[ + "width_slope" + ] + s.metadata.Acquisition_instrument.TEM.Detector.EDS.width_intercept = model_params[ + "width_intercept" + ] s.metadata.xray_db = model_params["db_name"] - + s.metadata.Truth.Data = {} s.metadata.Truth.Data.phases = sample["GW"] s.metadata.Truth.Data.weights = sample["H"] @@ -130,7 +140,8 @@ def sample_to_EDSespm(sample,elements = []) : return s -def sample_to_Signal1D(sample) : + +def sample_to_Signal1D(sample): r""" Same as :func:`espm.datasets.base.sample_to_EDSespm` but for non-EDS data such as the toy dataset. """ @@ -149,7 +160,9 @@ def sample_to_Signal1D(sample) : return s -def generate_spim_sample(phases, weights, model_params,misc_params, seed = 0,g_params = {}): +def generate_spim_sample( + phases, weights, model_params, misc_params, seed=0, g_params={} +): r""" Generate a dictionary containing: the spectrum image (made with the weights and phases), the ground truth, the model parameters and the misc parameters. @@ -167,30 +180,46 @@ def generate_spim_sample(phases, weights, model_params,misc_params, seed = 0,g_p The seed for the random number generator. The default is 0. g_params : dict, optional The parameters for the g matrix. The default is {}. Note that for EDXS data the g matrix is not used during the creation of the data. - + Returns ------- sample : dict A dictionary containing the spectrum image, the ground truth, the model parameters and the misc parameters. """ - assert np.allclose(np.sum(weights,axis = 2),1.0), "The input weights do not sum to one. Please modify it so that they sum to one along axis 2" - Xdot = generate_spim(phases, weights, misc_params["densities"], misc_params["N"], seed = seed,continuous=True) - X = generate_spim(phases, weights, misc_params["densities"], misc_params["N"],seed= seed,continuous=False) + assert np.allclose(np.sum(weights, axis=2), 1.0), ( + "The input weights do not sum to one. Please modify it so that they sum to one along axis 2" + ) + Xdot = generate_spim( + phases, + weights, + misc_params["densities"], + misc_params["N"], + seed=seed, + continuous=True, + ) + X = generate_spim( + phases, + weights, + misc_params["densities"], + misc_params["N"], + seed=seed, + continuous=False, + ) shape_2d = weights.shape[:2] - - if misc_params["model"] == "EDXS" : + + if misc_params["model"] == "EDXS": G = None - else : + else: model_class = getattr(models, misc_params["model"]) model = model_class(**model_params) model.generate_g_matr(**g_params) G = model.G - normed_phases = phases/np.sum(phases, axis=1, keepdims=True) + normed_phases = phases / np.sum(phases, axis=1, keepdims=True) Ns = misc_params["N"] * np.array(misc_params["densities"]) - scaled_phases = normed_phases*Ns[:,np.newaxis] - + scaled_phases = normed_phases * Ns[:, np.newaxis] + sample = {} sample["model_parameters"] = model_params sample["misc_parameters"] = misc_params @@ -204,7 +233,10 @@ def generate_spim_sample(phases, weights, model_params,misc_params, seed = 0,g_p sample["G"] = G return sample -def generate_dataset(*args, base_path = DATASETS_PATH, sample_number = 10, base_seed = 0, elements = [], **kwargs): + +def generate_dataset( + *args, base_path=DATASETS_PATH, sample_number=10, base_seed=0, elements=[], **kwargs +): r""" Generate a set of spectrum images files and save them in the generated dataset folder. Each spectrum image is saved in a separate file and was generated using a different seed. @@ -221,14 +253,19 @@ def generate_dataset(*args, base_path = DATASETS_PATH, sample_number = 10, base_ ------- None. """ - for i in tqdm(range(sample_number)) : - sample = generate_spim_sample(*args, **kwargs, seed = base_seed + i) - if sample["misc_parameters"]["model"] == "EDXS" : - hs_sig = sample_to_EDSespm(sample,elements = elements) - elif sample["misc_parameters"]["model"] == "Toy" : - hs_sig = sample_to_Signal1D(sample) - # ajouter save - else : - raise ValueError("Unknown model. The implemented models are 'EDXS' and 'Toy") - hs_sig.save(base_path / Path(sample["misc_parameters"]["data_folder"]) / Path(f"sample_{i}.hspy")) - + for i in tqdm(range(sample_number)): + sample = generate_spim_sample(*args, **kwargs, seed=base_seed + i) + if sample["misc_parameters"]["model"] == "EDXS": + hs_sig = sample_to_EDSespm(sample, elements=elements) + elif sample["misc_parameters"]["model"] == "Toy": + hs_sig = sample_to_Signal1D(sample) + # ajouter save + else: + raise ValueError( + "Unknown model. The implemented models are 'EDXS' and 'Toy" + ) + hs_sig.save( + base_path + / Path(sample["misc_parameters"]["data_folder"]) + / Path(f"sample_{i}.hspy") + ) diff --git a/espm/datasets/built_in_EDXS_datasets.py b/espm/datasets/built_in_EDXS_datasets.py index 5ad9f73..72177bc 100644 --- a/espm/datasets/built_in_EDXS_datasets.py +++ b/espm/datasets/built_in_EDXS_datasets.py @@ -4,102 +4,99 @@ - A dataset with a linear local accumulation of Sr. """ -from espm.datasets.base import generate_dataset -from espm.conf import DATASETS_PATH +import os from pathlib import Path + import hyperspy.api as hs -import os + +from espm.conf import DATASETS_PATH +from espm.datasets.base import generate_dataset +from espm.models.EDXS_function import elts_list_from_dict_list from espm.models.generate_EDXS_phases import generate_modular_phases from espm.weights.generate_weights import generate_weights -from espm.models.EDXS_function import elts_list_from_dict_list -particles_phases_dict = { - 'elts_dicts' : [ - {'V': 0.04704309583693933, - 'Rb': 0.914954275584854, - 'W': 0.06834271611694454}, - {'N': 0.4517936299777999, - 'Yb': 0.39973013314240835, - 'Pt': 0.08298592142537742}, - {'Al': 0.43306626599937914, - 'Ti': 0.3985896640183708, - 'La': 0.8994030840372912} +particles_phases_dict = { + "elts_dicts": [ + {"V": 0.04704309583693933, "Rb": 0.914954275584854, "W": 0.06834271611694454}, + {"N": 0.4517936299777999, "Yb": 0.39973013314240835, "Pt": 0.08298592142537742}, + {"Al": 0.43306626599937914, "Ti": 0.3985896640183708, "La": 0.8994030840372912}, ], - 'brstlg_pars' : [ - {'b0': 11.408513360414626e-06, - 'b1': 56.606903143185911e-04}, - {'b0': 17.317975736931391e-06, - 'b1': 15.2126092148294355e-04}, - {'b0': 2.2664567599307173e-06, - 'b1': 13.1627208027847766e-04} + "brstlg_pars": [ + {"b0": 11.408513360414626e-06, "b1": 56.606903143185911e-04}, + {"b0": 17.317975736931391e-06, "b1": 15.2126092148294355e-04}, + {"b0": 2.2664567599307173e-06, "b1": 13.1627208027847766e-04}, ], - 'scales' : [1,1,1], - 'model_params': {'e_offset': 0.2, - 'e_size': 1980, - 'e_scale': 0.01, - 'width_slope': 0.01, - 'width_intercept': 0.065, - 'db_name': '200keV_xrays.json', - 'E0': 200, - 'params_dict': {'Abs': {'thickness': 1e-05, - 'toa': 22, - 'density': None, - 'atomic_fraction': False}, - 'Det': 'SDD_efficiency.txt'}} - } + "scales": [1, 1, 1], + "model_params": { + "e_offset": 0.2, + "e_size": 1980, + "e_scale": 0.01, + "width_slope": 0.01, + "width_intercept": 0.065, + "db_name": "200keV_xrays.json", + "E0": 200, + "params_dict": { + "Abs": { + "thickness": 1e-05, + "toa": 22, + "density": None, + "atomic_fraction": False, + }, + "Det": "SDD_efficiency.txt", + }, + }, +} particles_misc_dict = { - 'N': 500, - 'seed' : 91, - 'data_folder': 'built_in_particules', - 'shape_2d': [80, 80], - 'model': 'EDXS', - 'densities': [0.6030107883539217, 0.9870613994765459, 0.8894990661032164] + "N": 500, + "seed": 91, + "data_folder": "built_in_particules", + "shape_2d": [80, 80], + "model": "EDXS", + "densities": [0.6030107883539217, 0.9870613994765459, 0.8894990661032164], } -boundary_phases_dict = { - 'elts_dicts' : [ - {"Ca" : 0.54860348, - "P" : 0.38286879, - "Sr" : 0.03166235, - "Cu" : 0.03686538}, - {"Ca" : 0.54860348, - "P" : 0.38286879, - "Sr" : 0.12166235, - "Cu" : 0.03686538} +boundary_phases_dict = { + "elts_dicts": [ + {"Ca": 0.54860348, "P": 0.38286879, "Sr": 0.03166235, "Cu": 0.03686538}, + {"Ca": 0.54860348, "P": 0.38286879, "Sr": 0.12166235, "Cu": 0.03686538}, ], - 'brstlg_pars' : [ - {"b0" : 5.5367e-4, - "b1" : 0.0192181}, - {"b0" : 5.5367e-4, - "b1" : 0.0192181} + "brstlg_pars": [ + {"b0": 5.5367e-4, "b1": 0.0192181}, + {"b0": 5.5367e-4, "b1": 0.0192181}, ], - 'scales' : [0.05,0.05], - 'model_params': {'e_offset': 1.27, - 'e_size': 3746, - 'e_scale': 0.005, - 'width_slope': 0.01, - 'width_intercept': 0.065, - 'db_name': '200keV_xrays.json', - 'E0': 200, - 'params_dict': {'Abs': {'thickness': 140e-07, - 'toa': 22, - 'density': 3.124, - 'atomic_fraction': False}, - 'Det': 'SDD_efficiency.txt'}} - } + "scales": [0.05, 0.05], + "model_params": { + "e_offset": 1.27, + "e_size": 3746, + "e_scale": 0.005, + "width_slope": 0.01, + "width_intercept": 0.065, + "db_name": "200keV_xrays.json", + "E0": 200, + "params_dict": { + "Abs": { + "thickness": 140e-07, + "toa": 22, + "density": 3.124, + "atomic_fraction": False, + }, + "Det": "SDD_efficiency.txt", + }, + }, +} boundary_misc_dict = { - 'N': 15, - 'seed' : 0, - 'data_folder': 'built_in_grain_boundary', - 'shape_2d': [100, 400], - 'model': 'EDXS', - 'densities': [1, 1] + "N": 15, + "seed": 0, + "data_folder": "built_in_grain_boundary", + "shape_2d": [100, 400], + "model": "EDXS", + "densities": [1, 1], } -def generate_built_in_datasets (seeds_range = 10) : +def generate_built_in_datasets(seeds_range=10): r""" Generate the two built-in datasets if they are not already present in the datasets folder. @@ -112,31 +109,55 @@ def generate_built_in_datasets (seeds_range = 10) : ------- None """ - if not(os.path.isdir(DATASETS_PATH / Path(particles_misc_dict["data_folder"]))) : - print("Generating 2 particles + one matrix built-in dataset. This will take a minute.") + if not (os.path.isdir(DATASETS_PATH / Path(particles_misc_dict["data_folder"]))): + print( + "Generating 2 particles + one matrix built-in dataset. This will take a minute." + ) particle_phases = generate_modular_phases(**particles_phases_dict) - particles_weights = generate_weights("sphere", particles_misc_dict['shape_2d'], n_phases=3, seed=particles_misc_dict['seed'], radius = 20) - particles_elements = elts_list_from_dict_list(particles_phases_dict['elts_dicts']) - generate_dataset(base_seed=particles_misc_dict['seed'], - sample_number=seeds_range, - model_params = particles_phases_dict['model_params'], - misc_params = particles_misc_dict, - phases = particle_phases, - weights = particles_weights, - elements = particles_elements) - if not(os.path.isdir(DATASETS_PATH / Path(boundary_misc_dict["data_folder"]))) : - print("Generating a grain boundary with Sr segregation. This will take a minute.") + particles_weights = generate_weights( + "sphere", + particles_misc_dict["shape_2d"], + n_phases=3, + seed=particles_misc_dict["seed"], + radius=20, + ) + particles_elements = elts_list_from_dict_list( + particles_phases_dict["elts_dicts"] + ) + generate_dataset( + base_seed=particles_misc_dict["seed"], + sample_number=seeds_range, + model_params=particles_phases_dict["model_params"], + misc_params=particles_misc_dict, + phases=particle_phases, + weights=particles_weights, + elements=particles_elements, + ) + if not (os.path.isdir(DATASETS_PATH / Path(boundary_misc_dict["data_folder"]))): + print( + "Generating a grain boundary with Sr segregation. This will take a minute." + ) boundary_phases = generate_modular_phases(**boundary_phases_dict) - boundary_weights = generate_weights("gaussian_ripple", boundary_misc_dict['shape_2d'], n_phases=2, seed=boundary_misc_dict['seed'], width = 10) - boundary_elements = elts_list_from_dict_list(boundary_phases_dict['elts_dicts']) - generate_dataset(base_seed=boundary_misc_dict['seed'], - sample_number=seeds_range, - model_params = boundary_phases_dict['model_params'], - misc_params = boundary_misc_dict, - phases = boundary_phases, - weights = boundary_weights, - elements = boundary_elements) -def load_particules (sample = 0) : + boundary_weights = generate_weights( + "gaussian_ripple", + boundary_misc_dict["shape_2d"], + n_phases=2, + seed=boundary_misc_dict["seed"], + width=10, + ) + boundary_elements = elts_list_from_dict_list(boundary_phases_dict["elts_dicts"]) + generate_dataset( + base_seed=boundary_misc_dict["seed"], + sample_number=seeds_range, + model_params=boundary_phases_dict["model_params"], + misc_params=boundary_misc_dict, + phases=boundary_phases, + weights=boundary_weights, + elements=boundary_elements, + ) + + +def load_particules(sample=0): r""" Load the built-in dataset of particles. @@ -150,11 +171,14 @@ def load_particules (sample = 0) : spim : hyperspy.signals.EDSespm The loaded dataset. """ - filename = DATASETS_PATH / Path("{}/sample_{}.hspy".format(particles_misc_dict["data_folder"],sample)) + filename = DATASETS_PATH / Path( + "{}/sample_{}.hspy".format(particles_misc_dict["data_folder"], sample) + ) spim = hs.load(filename) return spim -def load_grain_boundary (sample = 0) : + +def load_grain_boundary(sample=0): r""" Load the built-in dataset of a grain boundary. @@ -168,6 +192,8 @@ def load_grain_boundary (sample = 0) : spim : hyperspy.signals.EDSespm The loaded dataset. """ - filename = DATASETS_PATH / Path("{}/sample_{}.hspy".format(boundary_misc_dict["data_folder"],sample)) + filename = DATASETS_PATH / Path( + "{}/sample_{}.hspy".format(boundary_misc_dict["data_folder"], sample) + ) spim = hs.load(filename) - return spim \ No newline at end of file + return spim diff --git a/espm/datasets/eds_spim.py b/espm/datasets/eds_spim.py index fe67c01..e6a955e 100644 --- a/espm/datasets/eds_spim.py +++ b/espm/datasets/eds_spim.py @@ -1,6 +1,6 @@ r""" The module :mod:`espm.eds_spim` implements the :class:`EDSespm` class, which is a subclass of the :class:`hyperspy.signals.Signal1D` class. -The main purpose of this class is to provide an easy and clean interface between the hyperspy framework and the espm package: +The main purpose of this class is to provide an easy and clean interface between the hyperspy framework and the espm package: - The metadata are organised to correspond as much as possible to the typical metadata that can be found in hyperspy EDS_TEM object. - The machine learning algorithms of espm can be easily applied to the :class:`EDSespm` object using the standard hyperspy decomposition method. See the notebooks for examples. - The :class:`EDSespm` provides a convinient way to: @@ -9,50 +9,60 @@ - estimate best binning thanks to the method developed by G. Obozinski, N. Perraudin and M. Martinez Ruts. - set fixed W for the :class:`espm.estimators.NMFEstimator` decomposition """ + import json import warnings -import numpy as np -from scipy.optimize import curve_fit -from prettytable import PrettyTable -from tqdm import tqdm +import hyperspy.api as hs +import hyperspy.events import intervaltree -# from functools import wraps +import matplotlib as mpl +import matplotlib.pyplot as plt +import numpy as np +# from functools import wraps from exspy.signals import EDSTEMSpectrum from exspy.utils.eds import take_off_angle +from hyperspy.roi import BaseROI, RectangularROI from hyperspy.signal_tools import Signal1DRangeSelector from hyperspy.ui_registry import get_gui +from prettytable import PrettyTable +from scipy.optimize import curve_fit +from tqdm import tqdm -from hyperspy.roi import RectangularROI, BaseROI -import hyperspy.api as hs -import matplotlib.pyplot as plt -import matplotlib as mpl -import hyperspy.events - -from espm.utils import num_to_symbol -from espm.models import EDXS -from espm.estimators import SmoothNMF, NMFEstimator from espm.conf import NUMBER_PERIODIC_TABLE -from espm.utils import number_to_symbol_list, get_explained_intensity_W, symbol_to_number_list +from espm.estimators import NMFEstimator, SmoothNMF +from espm.models import EDXS +from espm.utils import ( + get_explained_intensity_W, + num_to_symbol, + number_to_symbol_list, + symbol_to_number_list, +) NPT = json.load(open(NUMBER_PERIODIC_TABLE)) -def _check_decomposition(func) -> bool : + +def _check_decomposition(func) -> bool: # @wraps(func) - def inner(instance,*args,**kwargs) : + def inner(instance, *args, **kwargs): est = instance.learning_results.decomposition_algorithm - assert not(est is None), f"No decomposition was performed. Please, use the espm decomposition before running {func}" - assert issubclass(type(est),NMFEstimator), f"To use {func} you need to use an object that inherits NMFEstimator, e.g. SmoothNMF." - return func(instance,*args,**kwargs) + assert est is not None, ( + f"No decomposition was performed. Please, use the espm decomposition before running {func}" + ) + assert issubclass(type(est), NMFEstimator), ( + f"To use {func} you need to use an object that inherits NMFEstimator, e.g. SmoothNMF." + ) + return func(instance, *args, **kwargs) return inner -class EDSespm(EDSTEMSpectrum) : + +class EDSespm(EDSTEMSpectrum): _signal_type = "EDS_espm" - def __init__ (self,*args,**kwargs) : - super().__init__(*args,**kwargs) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self.shape_2d_ = None self._X = None self.G_ = None @@ -65,56 +75,84 @@ def __init__ (self,*args,**kwargs) : # Properties # ############## - def _set_default_analysis_params(self) -> None : - # TODO : make them fetch preferences from the user + def _set_default_analysis_params(self) -> None: + # TODO : make them fetch preferences from the user md = self.metadata md.Signal.signal_type = "EDS_espm" - - if "Acquisition_instrument.TEM.Detector.EDS.width_slope" not in md : + + if "Acquisition_instrument.TEM.Detector.EDS.width_slope" not in md: md.set_item("Acquisition_instrument.TEM.Detector.EDS.width_slope", 0.01) - if "Acquisition_instrument.TEM.Detector.EDS.width_intercept" not in md : - md.set_item("Acquisition_instrument.TEM.Detector.EDS.width_intercept", 0.065) - if "xrays_db" not in md : + if "Acquisition_instrument.TEM.Detector.EDS.width_intercept" not in md: + md.set_item( + "Acquisition_instrument.TEM.Detector.EDS.width_intercept", 0.065 + ) + if "xrays_db" not in md: md.set_item("xray_db", "200keV_xrays.json") - if "Acquisition_instrument.TEM.Detector.EDS.type" not in md : - md.set_item("Acquisition_instrument.TEM.Detector.EDS.type", "SDD_efficiency.txt") - if "Acquisition_instrument.TEM.Stage.tilt_beta" not in md : + if "Acquisition_instrument.TEM.Detector.EDS.type" not in md: + md.set_item( + "Acquisition_instrument.TEM.Detector.EDS.type", "SDD_efficiency.txt" + ) + if "Acquisition_instrument.TEM.Stage.tilt_beta" not in md: md.set_item("Acquisition_instrument.TEM.Stage.tilt_beta", 0.0) - def _check_metadata_G(self) -> None : + def _check_metadata_G(self) -> None: md = self.metadata - if "Sample.elements" not in md : - raise ValueError("The elements of the sample are missing in the metadata. Please use the set_elements method to set the elements.") - if "Acquisition_instrument.TEM.beam_energy" not in md : - raise ValueError("The beam energy is missing in the metadata. Please use the set_microscope_parameters method to set the beam energy.") - if "Sample.density" not in md : - raise ValueError("The density of the sample is missing in the metadata. Please use the set_analysis_parameters method to set the density.") - if "Sample.thickness" not in md : - raise ValueError("The thickness of the sample is missing in the metadata. Please use the set_analysis_parameters method to set the thickness.") - if "Acquisition_instrument.TEM.Detector.EDS.type" not in md : - raise ValueError("The detector type is missing in the metadata. Please use the set_analysis_parameters method to set the detector type.") - if "Acquisition_instrument.TEM.Detector.EDS.take_off_angle" not in md : - raise ValueError("The take-off angle is missing in the metadata. Please use the set_microscope_parameters method to set the take-off angle.") - if "Acquisition_instrument.TEM.Detector.EDS.width_slope" not in md : - raise ValueError("The width slope is missing in the metadata. Please use the set_analysis_parameters method to set the width slope.") - if "Acquisition_instrument.TEM.Detector.EDS.width_intercept" not in md : - raise ValueError("The width intercept is missing in the metadata. Please use the set_analysis_parameters method to set the width intercept.") - if "xray_db" not in md : - raise ValueError("The xray database is missing in the metadata. Please use the set_analysis_parameters method to set the xray database.") - - def _check_metadata_quantification(self) -> None : + if "Sample.elements" not in md: + raise ValueError( + "The elements of the sample are missing in the metadata. Please use the set_elements method to set the elements." + ) + if "Acquisition_instrument.TEM.beam_energy" not in md: + raise ValueError( + "The beam energy is missing in the metadata. Please use the set_microscope_parameters method to set the beam energy." + ) + if "Sample.density" not in md: + raise ValueError( + "The density of the sample is missing in the metadata. Please use the set_analysis_parameters method to set the density." + ) + if "Sample.thickness" not in md: + raise ValueError( + "The thickness of the sample is missing in the metadata. Please use the set_analysis_parameters method to set the thickness." + ) + if "Acquisition_instrument.TEM.Detector.EDS.type" not in md: + raise ValueError( + "The detector type is missing in the metadata. Please use the set_analysis_parameters method to set the detector type." + ) + if "Acquisition_instrument.TEM.Detector.EDS.take_off_angle" not in md: + raise ValueError( + "The take-off angle is missing in the metadata. Please use the set_microscope_parameters method to set the take-off angle." + ) + if "Acquisition_instrument.TEM.Detector.EDS.width_slope" not in md: + raise ValueError( + "The width slope is missing in the metadata. Please use the set_analysis_parameters method to set the width slope." + ) + if "Acquisition_instrument.TEM.Detector.EDS.width_intercept" not in md: + raise ValueError( + "The width intercept is missing in the metadata. Please use the set_analysis_parameters method to set the width intercept." + ) + if "xray_db" not in md: + raise ValueError( + "The xray database is missing in the metadata. Please use the set_analysis_parameters method to set the xray database." + ) + + def _check_metadata_quantification(self) -> None: md = self.metadata - if "Acquisition_instrument.TEM.Detector.EDS.geometric_efficiency" not in md : - raise ValueError("The geometric efficiency of the detector is missing in the metadata. Please use the set_analysis_parameters method to set the geometric efficiency.") - if "Acquisition_instrument.TEM.beam_current" not in md : - raise ValueError("The beam current is missing in the metadata. Please use the set_microscope_parameters method to set the beam current.") - if "Acquisition_instrument.TEM.Detector.EDS.real_time" not in md : - raise ValueError("The acquisition time is missing in the metadata. Please use the set_microscope_parameters method to set the acquisition time.") - + if "Acquisition_instrument.TEM.Detector.EDS.geometric_efficiency" not in md: + raise ValueError( + "The geometric efficiency of the detector is missing in the metadata. Please use the set_analysis_parameters method to set the geometric efficiency." + ) + if "Acquisition_instrument.TEM.beam_current" not in md: + raise ValueError( + "The beam current is missing in the metadata. Please use the set_microscope_parameters method to set the beam current." + ) + if "Acquisition_instrument.TEM.Detector.EDS.real_time" not in md: + raise ValueError( + "The acquisition time is missing in the metadata. Please use the set_microscope_parameters method to set the acquisition time." + ) + @property - def custom_init (self) -> bool : + def custom_init(self) -> bool: r""" Boolean setting whether using the custom_init (see espm.models.EDXS) or not. If True, the custom_init will be used to initialise the decomposition. @@ -122,55 +160,67 @@ def custom_init (self) -> bool : If None, the will be set to False. """ return self.custom_init_ - + @custom_init.setter - def custom_init (self, value : bool) -> None : + def custom_init(self, value: bool) -> None: self.custom_init_ = value @property - def shape_2d (self) -> tuple[int] : + def shape_2d(self) -> tuple[int]: r""" Shape of the data in the spatial dimension. """ - if self.shape_2d_ is None : + if self.shape_2d_ is None: self.shape_2d_ = self.axes_manager[1].size, self.axes_manager[0].size return self.shape_2d_ @property - def X (self) -> np.ndarray : + def X(self) -> np.ndarray: r""" The data in the form of a 2D array of shape (n_samples, n_features). """ - if self._X is None : - shape = self.axes_manager[1].size, self.axes_manager[0].size, self.axes_manager[2].size - self._X = self.data.reshape((shape[0]*shape[1], shape[2])).T + if self._X is None: + shape = ( + self.axes_manager[1].size, + self.axes_manager[0].size, + self.axes_manager[2].size, + ) + self._X = self.data.reshape((shape[0] * shape[1], shape[2])).T return self._X @property - def G(self) -> np.ndarray : + def G(self) -> np.ndarray: r""" The G matrix of the :class:`espm.models.EDXS` model corresponding to the metadata of the :class:`EDSespm` object. """ - if self.G_ is None : - try : - if self.problem_type == "identity" : + if self.G_ is None: + try: + if self.problem_type == "identity": return None - except AttributeError : - warnings.warn("You did not used the build_G method to build the G matrix. In ESpM-NMF, an idenity matrix will be used for decomposition") + except AttributeError: + warnings.warn( + "You did not used the build_G method to build the G matrix. In ESpM-NMF, an idenity matrix will be used for decomposition" + ) return None return self.G_ @property - def model (self) -> EDXS : + def model(self) -> EDXS: r""" The :class:`espm.models.EDXS` model corresponding to the metadata of the :class:`EDSespm` object. - """ - if self.model_ is None : + """ + if self.model_ is None: mod_pars = get_metadata(self) self.model_ = EDXS(**mod_pars, custom_init=self.custom_init_) return self.model_ - def build_G(self, problem_type : str = "bremsstrahlung",ignored_elements : list[str] = ['Cu'],*, elements_dict : dict[str,float] = {}) -> None : + def build_G( + self, + problem_type: str = "bremsstrahlung", + ignored_elements: list[str] = ["Cu"], + *, + elements_dict: dict[str, float] = {}, + ) -> None: r""" Build the G matrix of the :class:`espm.models.EDXS` model corresponding to the metadata of the :class:`EDSespm` object and stores it as an attribute. @@ -186,19 +236,24 @@ def build_G(self, problem_type : str = "bremsstrahlung",ignored_elements : list[ For example elements_dict = {"26",3.0} will separate the characteristic X-rays of the element Fe into two energies ranges and assign them each a column in the G matrix. This is useful to circumvent issues with the absorption. Returns ------- - None + None """ self._check_metadata_G() self.problem_type = problem_type self.separated_lines = elements_dict - g_pars = {"g_type" : problem_type, 'ignored_elements' : ignored_elements, "elements" : self.metadata.Sample.elements, "elements_dict" : elements_dict} + g_pars = { + "g_type": problem_type, + "ignored_elements": ignored_elements, + "elements": self.metadata.Sample.elements, + "elements_dict": elements_dict, + } self.model.generate_g_matr(**g_pars) self.G_ = self.model.G - + # Storing the model parameters in the metadata so that the decomposition does not erase them # Indeed the decomposition re-creates a new object of the same class when it is called - self.metadata.EDS_model= {} + self.metadata.EDS_model = {} self.metadata.EDS_model.problem_type = problem_type self.metadata.EDS_model.separated_lines = elements_dict self.metadata.EDS_model.elements = self.model.model_elts @@ -206,14 +261,14 @@ def build_G(self, problem_type : str = "bremsstrahlung",ignored_elements : list[ def set_analysis_parameters( self, - thickness : float = None, - density : float = None, - detector_type : str | dict = None, - width_slope : float = None, - width_intercept : float = None, - geom_eff : float = None, - xray_db : str = None - ) -> None : + thickness: float = None, + density: float = None, + detector_type: str | dict = None, + width_slope: float = None, + width_intercept: float = None, + geom_eff: float = None, + xray_db: str = None, + ) -> None: r""" Set the relevant parameters for the analysis in the metadata of the :class:`EDSespm` object. @@ -247,31 +302,43 @@ def set_analysis_parameters( if detector_type is not None: md.set_item("Acquisition_instrument.TEM.Detector.EDS.type", detector_type) if width_slope is not None: - md.set_item("Acquisition_instrument.TEM.Detector.EDS.width_slope", width_slope) + md.set_item( + "Acquisition_instrument.TEM.Detector.EDS.width_slope", width_slope + ) if width_intercept is not None: - md.set_item("Acquisition_instrument.TEM.Detector.EDS.width_intercept", width_intercept) + md.set_item( + "Acquisition_instrument.TEM.Detector.EDS.width_intercept", + width_intercept, + ) if geom_eff is not None: - md.set_item("Acquisition_instrument.TEM.Detector.EDS.geometric_efficiency", geom_eff) + md.set_item( + "Acquisition_instrument.TEM.Detector.EDS.geometric_efficiency", geom_eff + ) if xray_db is not None: md.set_item("xray_db", xray_db) - try : - md.set_item("Acquisition_instrument.TEM.Detector.EDS.take_off_angle", - take_off_angle(tilt_stage = md.Acquisition_instrument.TEM.Stage.tilt_alpha, - azimuth_angle = md.Acquisition_instrument.TEM.Detector.EDS.azimuth_angle, - elevation_angle = md.Acquisition_instrument.TEM.Detector.EDS.elevation_angle, - beta_tilt = md.Acquisition_instrument.TEM.Stage.tilt_beta)) - except AttributeError : - print("You need to define the azimuth and elevation of the detector as well as the alpha and beta tilt of the sample holder. Please, use the set_microscope_parameters function.") - + try: + md.set_item( + "Acquisition_instrument.TEM.Detector.EDS.take_off_angle", + take_off_angle( + tilt_stage=md.Acquisition_instrument.TEM.Stage.tilt_alpha, + azimuth_angle=md.Acquisition_instrument.TEM.Detector.EDS.azimuth_angle, + elevation_angle=md.Acquisition_instrument.TEM.Detector.EDS.elevation_angle, + beta_tilt=md.Acquisition_instrument.TEM.Stage.tilt_beta, + ), + ) + except AttributeError: + print( + "You need to define the azimuth and elevation of the detector as well as the alpha and beta tilt of the sample holder. Please, use the set_microscope_parameters function." + ) ############################ # Helper functions for NMF # ############################ - def carto_fixed_W(self, brstlg_comps : int = 1) -> np.ndarray : + def carto_fixed_W(self, brstlg_comps: int = 1) -> np.ndarray: r""" - Helper function to create a fixed_W matrix for chemical mapping. It will output a matrix + Helper function to create a fixed_W matrix for chemical mapping. It will output a matrix It can be used to make a decomposition with as many components as they are chemical elements and then allow each component to have only one of each element. The spectral components are then the characteristic peaks of each element and the spatial components are the associated chemical maps. The bremsstrahlung is calculated separately and added to other components. @@ -285,23 +352,25 @@ def carto_fixed_W(self, brstlg_comps : int = 1) -> np.ndarray : ------- W : numpy.ndarray """ - if self.G_ is None : - raise ValueError("The G matrix has not been built yet. Please use the build_G method.") + if self.G_ is None: + raise ValueError( + "The G matrix has not been built yet. Please use the build_G method." + ) elements = self.metadata.EDS_model.elements - if self.problem_type == "no_brstlg" : - W = np.diag(-1* np.ones((len(elements), ))) - elif self.problem_type == "bremsstrahlung" : - W1 = np.diag(-1* np.ones((len(elements), ))) + if self.problem_type == "no_brstlg": + W = np.diag(-1 * np.ones((len(elements),))) + elif self.problem_type == "bremsstrahlung": + W1 = np.diag(-1 * np.ones((len(elements),))) W2 = np.zeros((2, len(elements))) - W_elts = np.vstack((W1,W2)) - W3 = np.zeros((len(elements),brstlg_comps)) - W4 = -1*np.ones((2,brstlg_comps)) - W_brstlg = np.vstack((W3,W4)) - W = np.hstack((W_elts,W_brstlg)) + W_elts = np.vstack((W1, W2)) + W3 = np.zeros((len(elements), brstlg_comps)) + W4 = -1 * np.ones((2, brstlg_comps)) + W_brstlg = np.vstack((W3, W4)) + W = np.hstack((W_elts, W_brstlg)) return W - def set_fixed_W (self,phases_dict : dict[str,float]) -> np.ndarray : + def set_fixed_W(self, phases_dict: dict[str, float]) -> np.ndarray: r""" Helper function to create a fixed_W matrix. The output matrix will have -1 entries except for the elements (and bremsstrahlung parameters) that are present in the phases_dict dictionary. In the output (fixed_W) matrix, the -1 entries will be ignored during the decomposition using :class:`espm.estimator.NMFEstimator` are normally learned while the non-negative entries will be fixed to the values given in the phases_dict dictionary. @@ -315,46 +384,56 @@ def set_fixed_W (self,phases_dict : dict[str,float]) -> np.ndarray : ------- W : numpy.ndarray """ - if self.G_ is None : - raise ValueError("The G matrix has not been built yet. Please use the build_G method.") + if self.G_ is None: + raise ValueError( + "The G matrix has not been built yet. Please use the build_G method." + ) raw_elts = self.metadata.EDS_model.elements elements = self.model.get_elements() indices = self.model.NMF_simplex() # convert elements to symbols but also omitting splitted lines @number_to_symbol_list - def convert_to_symbols(elements = []) : + def convert_to_symbols(elements=[]): return elements - + conv_elts = convert_to_symbols(elements=elements) - if self.problem_type == "no_brstlg" : - W = -1* np.ones((len(raw_elts), len(phases_dict.keys()))) - elif self.problem_type == "bremsstrahlung" : - W = -1* np.ones((len(raw_elts)+2, len(phases_dict.keys()))) - else : - raise ValueError("problem type should be either no_brstlg or bremsstrahlung") - for p, phase in enumerate(phases_dict) : - for key in phases_dict[phase] : - if key == "b0" : - if self.problem_type == "bremsstrahlung" : - W[-2,p] = phases_dict[phase][key] - else : - warnings.warn("The chosen EDXS modelling does not incorporate the bremsstrahlung. Input bremsstrahlung parameters will be ignored.") - if key == "b1" : - if self.problem_type == "bremsstrahlung" : - W[-1,p] = phases_dict[phase][key] - else : - warnings.warn("The chosen EDXS modelling does not incorporate the bremsstrahlung. Input bremsstrahlung parameters will be ignored.") - if key in conv_elts : - W[indices[conv_elts.index(key)],p] = phases_dict[phase][key] + if self.problem_type == "no_brstlg": + W = -1 * np.ones((len(raw_elts), len(phases_dict.keys()))) + elif self.problem_type == "bremsstrahlung": + W = -1 * np.ones((len(raw_elts) + 2, len(phases_dict.keys()))) + else: + raise ValueError( + "problem type should be either no_brstlg or bremsstrahlung" + ) + for p, phase in enumerate(phases_dict): + for key in phases_dict[phase]: + if key == "b0": + if self.problem_type == "bremsstrahlung": + W[-2, p] = phases_dict[phase][key] + else: + warnings.warn( + "The chosen EDXS modelling does not incorporate the bremsstrahlung. Input bremsstrahlung parameters will be ignored." + ) + if key == "b1": + if self.problem_type == "bremsstrahlung": + W[-1, p] = phases_dict[phase][key] + else: + warnings.warn( + "The chosen EDXS modelling does not incorporate the bremsstrahlung. Input bremsstrahlung parameters will be ignored." + ) + if key in conv_elts: + W[indices[conv_elts.index(key)], p] = phases_dict[phase][key] return W - - def print_concentration_report (self, - selected_elts : list[str] = [], - W_input : np.ndarray = None, - fit_error : bool = True, - disclaimer : bool = True) -> None : + + def print_concentration_report( + self, + selected_elts: list[str] = [], + W_input: np.ndarray = None, + fit_error: bool = True, + disclaimer: bool = True, + ) -> None: r""" Print a report of the chemical concentrations from a fitted W. @@ -380,42 +459,52 @@ def print_concentration_report (self, ----- - This function is only available if the learning results contain a decomposition algorithm that has been fitted. """ - conv_elts, W, errors = self.concentration_report(selected_elts = selected_elts, W_input = W_input, fit_error = fit_error) - + conv_elts, W, errors = self.concentration_report( + selected_elts=selected_elts, W_input=W_input, fit_error=fit_error + ) + table = PrettyTable() field_list = ["Elements"] - for i in range(W.shape[1]) : + for i in range(W.shape[1]): field_list.append("p" + str(i) + " (at.%)") - if fit_error : + if fit_error: field_list.append("p" + str(i) + " std (%)") table.field_names = field_list - for i,j in enumerate(conv_elts) : + for i, j in enumerate(conv_elts): row = [j] - for k in range(W.shape[1]) : - row.append(W[i,k]) - if fit_error : - row.append(errors[i,k]) + for k in range(W.shape[1]): + row.append(W[i, k]) + if fit_error: + row.append(errors[i, k]) table.add_row(row) - table.float_format="0.3" + table.float_format = "0.3" table.align = "r" table.align["Elements"] = "l" - # table.set_style(MSWORD_FRIENDLY) + # table.set_style(MSWORD_FRIENDLY) print(table) - if disclaimer and fit_error: - print("\nDisclaimer : The presented errors correspond to the statistical error on the fitted intensity of the peaks according to a Poisson law.\nIn other words it corresponds to the precision of the measurment.\nThe accuracy of the measurment strongly depends on other factors such as absorption, cross-sections, etc...\nPlease consider these parameters when interpreting the results.") + if disclaimer and fit_error: + print( + "\nDisclaimer : The presented errors correspond to the statistical error on the fitted intensity of the peaks according to a Poisson law.\nIn other words it corresponds to the precision of the measurment.\nThe accuracy of the measurment strongly depends on other factors such as absorption, cross-sections, etc...\nPlease consider these parameters when interpreting the results." + ) ############################ # Bremsstrahlung functions # ############################ - def estimate_mass_thickness(self, ignored_elements : list[str] = ['Cu'], tol : float = 1e-8,*, elements_dict : dict = {}) -> None : + def estimate_mass_thickness( + self, + ignored_elements: list[str] = ["Cu"], + tol: float = 1e-8, + *, + elements_dict: dict = {}, + ) -> None: r""" Based on the complete metadata of the :class:`EDSespm` object, this function estimates the mass thickness of the sample. This function derives the mass-thickness from the characteristic X-rays. Then the bremsstrahlung parameters are estimated using that mass-thickness. The process is then repeated ten times to ensure convergence. The results are plotted on the spectrum. Check the metadata to read the estimated mass-thickness. - + Parameters ---------- elements_dict : dict, optional @@ -439,67 +528,92 @@ def estimate_mass_thickness(self, ignored_elements : list[str] = ['Cu'], tol : f # Let's implement for 1D data first. So we sum over dimensions if needed. self._check_metadata_G() self._check_metadata_quantification() - if len(self.axes_manager.navigation_axes) > 0 : - raise NotImplementedError('For now this function is not fully implemented for spectrum images. Use this on an extracted 1D spectrum.') + if len(self.axes_manager.navigation_axes) > 0: + raise NotImplementedError( + "For now this function is not fully implemented for spectrum images. Use this on an extracted 1D spectrum." + ) curr_X = self.data # First init of fit - self.build_G(ignored_elements= ignored_elements, elements_dict=elements_dict) - estimator = SmoothNMF(n_components = 1, G=self.model) - estimator.fit(curr_X[:,np.newaxis]) + self.build_G(ignored_elements=ignored_elements, elements_dict=elements_dict) + estimator = SmoothNMF(n_components=1, G=self.model) + estimator.fit(curr_X[:, np.newaxis]) H_init = estimator.H_ W_init = estimator.W_ - elts = list(self.model.get_elements(include_ignored = False)) + elts = list(self.model.get_elements(include_ignored=False)) elts_indices = self.model.NMF_simplex() - new_elts_dict = {elts[i] : W_init[elts_indices[i]] for i in range(len(elts))} - + new_elts_dict = {elts[i]: W_init[elts_indices[i]] for i in range(len(elts))} + _ = 0 curr_mt = self.metadata.Sample.thickness * self.metadata.Sample.density - while _ < 5 : + while _ < 5: # first init of the model - brstlg_model, mask = self.model.bremsstrahlung_only_tools(mass_thickness=curr_mt,elements_dict = new_elts_dict, ranges = self.ranges) + brstlg_model, mask = self.model.bremsstrahlung_only_tools( + mass_thickness=curr_mt, elements_dict=new_elts_dict, ranges=self.ranges + ) masked_X = curr_X[mask] - brstlg_estimator = SmoothNMF(n_components = 1, G=brstlg_model, fixed_H = H_init, tol = tol ) - brstlg_estimator.fit(masked_X[:,np.newaxis]) - W_brstlg = np.vstack(( -1 * np.ones((W_init.shape[0] - brstlg_estimator.W_.shape[0], brstlg_estimator.W_.shape[1])),brstlg_estimator.W_)) + brstlg_estimator = SmoothNMF( + n_components=1, G=brstlg_model, fixed_H=H_init, tol=tol + ) + brstlg_estimator.fit(masked_X[:, np.newaxis]) + W_brstlg = np.vstack( + ( + -1 + * np.ones( + ( + W_init.shape[0] - brstlg_estimator.W_.shape[0], + brstlg_estimator.W_.shape[1], + ) + ), + brstlg_estimator.W_, + ) + ) - self.build_G(ignored_elements= ignored_elements, elements_dict=elements_dict) + self.build_G(ignored_elements=ignored_elements, elements_dict=elements_dict) # First estimation of the bremsstrahlung + elts - estimator = SmoothNMF(n_components = 1, G=self.model, fixed_W = W_brstlg, tol = tol) - estimator.fit(curr_X[:,np.newaxis]) - + estimator = SmoothNMF( + n_components=1, G=self.model, fixed_W=W_brstlg, tol=tol + ) + estimator.fit(curr_X[:, np.newaxis]) + # Get the elements, their concentrations and the mass_thickness value W_init = estimator.W_ H_init = estimator.H_ - - elts = list(self.model.get_elements(include_ignored = False)) + + elts = list(self.model.get_elements(include_ignored=False)) elts_indices = self.model.NMF_simplex() - new_elts_dict = {elts[i] : W_init[elts_indices[i]] for i in range(len(elts))} + new_elts_dict = {elts[i]: W_init[elts_indices[i]] for i in range(len(elts))} total_weight = self._elements_dict_to_weights(new_elts_dict) curr_mt = self._extract_mass_thickness(H_init.sum(), total_weight) _ += 1 - print("The current estimated mass-thickness is {} g.cm^-2".format(curr_mt),flush = True) + print( + f"The current estimated mass-thickness is {curr_mt} g.cm^-2", + flush=True, + ) self.plot(True) - self._plot.signal_plot.ax.set_title("Estimated mass-thickness : {} g.cm^-2".format(curr_mt)) - + self._plot.signal_plot.ax.set_title( + f"Estimated mass-thickness : {curr_mt} g.cm^-2" + ) + axis = self.axes_manager.signal_axes[0].axis - self._plot.signal_plot.ax.plot(axis, - estimator.G_@estimator.W_@estimator.H_, - 'b-', - label = 'Full model') - self._plot.signal_plot.ax.plot(axis[mask], - brstlg_estimator.G@brstlg_estimator.W_@brstlg_estimator.H_, - 'g.', - label = 'Bremmstrahlung') + self._plot.signal_plot.ax.plot( + axis, estimator.G_ @ estimator.W_ @ estimator.H_, "b-", label="Full model" + ) + self._plot.signal_plot.ax.plot( + axis[mask], + brstlg_estimator.G @ brstlg_estimator.W_ @ brstlg_estimator.H_, + "g.", + label="Bremmstrahlung", + ) self._plot.signal_plot.ax.legend() self.metadata.Sample.thickness = 1.0 self.metadata.Sample.density = curr_mt - def _elements_dict_to_weights(self,elements_dict) : + def _elements_dict_to_weights(self, elements_dict): """ Convert a dictionary of elements and their quantities to total weight. @@ -514,24 +628,34 @@ def _elements_dict_to_weights(self,elements_dict) : Total weight of the elements in grams. """ total_weight = sum( - quantity * NPT['table'][element]['atomic_mass'] * 1.66053906660e-24 + quantity * NPT["table"][element]["atomic_mass"] * 1.66053906660e-24 for element, quantity in elements_dict.items() ) return total_weight - def _extract_mass_thickness(self,H_value, total_weight) : - Na = 6.02214179e23 # TODO : Check the usefulness of Na. + def _extract_mass_thickness(self, H_value, total_weight): + Na = 6.02214179e23 # TODO : Check the usefulness of Na. # If I am correct the concentrations we guess have no unit. # Since they are not in mole, no need to normalize using Na - Ne = 6.25e18 # Number of electrons in a Coulomb + Ne = 6.25e18 # Number of electrons in a Coulomb # real time shound be the whole acquisition time (without dead time but with all pixels) - return H_value* total_weight/(self.metadata.Acquisition_instrument.TEM.beam_current * 1e-9 * - self.metadata.Acquisition_instrument.TEM.Detector.EDS.real_time * - Ne * self.model.norm[0][0] * - (self.metadata.Acquisition_instrument.TEM.Detector.EDS.geometric_efficiency/(4*np.pi)) - ) - - def select_background_windows(self, num_windows = 4, ranges = None) : + return ( + H_value + * total_weight + / ( + self.metadata.Acquisition_instrument.TEM.beam_current + * 1e-9 + * self.metadata.Acquisition_instrument.TEM.Detector.EDS.real_time + * Ne + * self.model.norm[0][0] + * ( + self.metadata.Acquisition_instrument.TEM.Detector.EDS.geometric_efficiency + / (4 * np.pi) + ) + ) + ) + + def select_background_windows(self, num_windows=4, ranges=None): r""" Select the background windows for the bremsstrahlung estimation. The function will open a window with the spectrum and the user will be able to select the background windows by clicking and dragging the mouse. Click then on 'Apply' to validate the selection. A bremmstrahlung model will be estimated and plotted on the spectrum. @@ -548,36 +672,42 @@ def select_background_windows(self, num_windows = 4, ranges = None) : """ # The code is quite dirty, but it works. # To code a proper gui we need to wait for an update of hyperspy - if self.model_ is None : - raise ValueError("The G matrix has not been built yet. Please use the build_G method.") - if ranges is not None : - self.ranges = ranges - self.model.ranges = self.ranges - else : - if len(self.axes_manager.navigation_axes) > 0 : - raise NotImplementedError('For now this function is not fully implemented for spectrum images. Use this on an extracted 1D spectrum.') + if self.model_ is None: + raise ValueError( + "The G matrix has not been built yet. Please use the build_G method." + ) + if ranges is not None: + self.ranges = ranges + self.model.ranges = self.ranges + else: + if len(self.axes_manager.navigation_axes) > 0: + raise NotImplementedError( + "For now this function is not fully implemented for spectrum images. Use this on an extracted 1D spectrum." + ) cm = self._register_ranges init_ranges = self._generate_ranges(num_windows) self.spans = [] - for i in range(num_windows) : + for i in range(num_windows): self.spans.append(Signal1DRangeSelector(self)) - - for j, span in enumerate(self.spans) : + + for j, span in enumerate(self.spans): span.span_selector.extents = init_ranges[j] span.on_close.append((cm, self)) - get_gui(span, toolkey = "hyperspy.interactive_range_selector") + get_gui(span, toolkey="hyperspy.interactive_range_selector") - def _register_ranges(self,signal, left, right) : + def _register_ranges(self, signal, left, right): # The unused args are required for the event to properly complete coord_list = [[span.ss_left_value, span.ss_right_value] for span in self.spans] - for coords in coord_list : - if np.nan in coords : + for coords in coord_list: + if np.nan in coords: # TODO : needs to be improved in future versions. - raise ValueError("You have to click and drag each area at least so that the calculated bremsstrahlung is displayed.") - coord_list.sort(key = lambda coord : coord[0]) + raise ValueError( + "You have to click and drag each area at least so that the calculated bremsstrahlung is displayed." + ) + coord_list.sort(key=lambda coord: coord[0]) tree = intervaltree.IntervalTree.from_tuples(coord_list) self.ranges = [] - for branch in tree : + for branch in tree: self.ranges.append([branch[0], branch[1]]) self.model.ranges = self.ranges @@ -585,16 +715,18 @@ def _register_ranges(self,signal, left, right) : model = self._compute_bremsstrahlung() self._plot_background(model) - def _compute_bremsstrahlung(self) : + def _compute_bremsstrahlung(self): mt = self.metadata.Sample.density * self.metadata.Sample.thickness - elts_dict = {elt : 1.0 for elt in self.metadata.Sample.elements} - brstlg_model, mask = self.model.bremsstrahlung_only_tools(mass_thickness=mt,elements_dict = elts_dict, ranges= self.ranges) + elts_dict = {elt: 1.0 for elt in self.metadata.Sample.elements} + brstlg_model, mask = self.model.bremsstrahlung_only_tools( + mass_thickness=mt, elements_dict=elts_dict, ranges=self.ranges + ) curr_X = self.data masked_X = curr_X[mask] # Estimate the bremsstrahlung on the partial data - brstlg_estimator = SmoothNMF(n_components = 1, G=brstlg_model) - brstlg_estimator.fit(masked_X[:,np.newaxis]) + brstlg_estimator = SmoothNMF(n_components=1, G=brstlg_model) + brstlg_estimator.fit(masked_X[:, np.newaxis]) # get the fitting results fH = brstlg_estimator.H_ fW = brstlg_estimator.W_ @@ -603,21 +735,23 @@ def _compute_bremsstrahlung(self) : # It is not super efficient but I think it is not an issue. The model can't be easily continued, it is not a function. axis = self.axes_manager.signal_axes[0] full_range = [[axis.low_value, axis.high_value]] - full_brstlg_model, full_mask = self.model.bremsstrahlung_only_tools(mass_thickness=mt,elements_dict = elts_dict, ranges= full_range) + full_brstlg_model, full_mask = self.model.bremsstrahlung_only_tools( + mass_thickness=mt, elements_dict=elts_dict, ranges=full_range + ) - return full_brstlg_model@fW@fH - - def _plot_background(self, model) : + return full_brstlg_model @ fW @ fH + + def _plot_background(self, model): # The full range from self.compute_background misses both ends # The axis needs to be trimmed accordingly - axis = self.axes_manager.signal_axes[0].axis[1:-1] - self._plot.signal_plot.ax.plot(axis,model) - - def _generate_ranges(self, num) : + axis = self.axes_manager.signal_axes[0].axis[1:-1] + self._plot.signal_plot.ax.plot(axis, model) + + def _generate_ranges(self, num): axis = self.axes_manager.signal_axes[0] bounds = (axis.low_value, axis.high_value) - values = np.linspace(bounds[0], bounds[1], num = 2*num + 2) - ranges_list = [(values[2*i-1],values[2*i]) for i in range(1,num+1)] + values = np.linspace(bounds[0], bounds[1], num=2 * num + 2) + ranges_list = [(values[2 * i - 1], values[2 * i]) for i in range(1, num + 1)] return ranges_list def decomposition( @@ -743,7 +877,9 @@ def decomposition( self.model_ = model_ @_check_decomposition - def plot_1D_results(self, xray_lines : bool = True,elements : list[str] = []) -> None : + def plot_1D_results( + self, xray_lines: bool = True, elements: list[str] = [] + ) -> None: r""" Plots each spectrum (component) resulting from an espm decompositions. It shows the contribution of each selected element and the total model. @@ -759,87 +895,105 @@ def plot_1D_results(self, xray_lines : bool = True,elements : list[str] = []) -> """ W = self.learning_results.decomposition_algorithm.W_ G = self.learning_results.decomposition_algorithm.G_ - H = self.learning_results.decomposition_algorithm.H_.mean(axis = 1) + H = self.learning_results.decomposition_algorithm.H_.mean(axis=1) @symbol_to_number_list - def convert_elts(elements = []) : + def convert_elts(elements=[]): return elements - + spectrum_1D = self.mean() - spectrum_1D.plot(xray_lines = xray_lines) - spectrum_1D._plot.signal_plot.ax.plot(spectrum_1D.axes_manager.signal_axes[0].axis, G@W@H, 'b-', label = 'Full model') - - conv_elts = convert_elts(elements = elements) - conv_elts_dict = {conv_elts[i] : elt for i, elt in enumerate(elements)} - line_styles = [ '--', '-.', ':'] - colors = ['g', 'r', 'c', 'm', 'y', 'k'] - + spectrum_1D.plot(xray_lines=xray_lines) + spectrum_1D._plot.signal_plot.ax.plot( + spectrum_1D.axes_manager.signal_axes[0].axis, + G @ W @ H, + "b-", + label="Full model", + ) + + conv_elts = convert_elts(elements=elements) + conv_elts_dict = {conv_elts[i]: elt for i, elt in enumerate(elements)} + line_styles = ["--", "-.", ":"] + colors = ["g", "r", "c", "m", "y", "k"] + _ = 0 for elt in conv_elts: - indices = [[i] for i, mod_elt in enumerate(self.metadata.EDS_model.elements) if str(elt) == mod_elt[:2]] + indices = [ + [i] + for i, mod_elt in enumerate(self.metadata.EDS_model.elements) + if str(elt) == mod_elt[:2] + ] if indices: - component = sum([G[:,idx] @ W[idx,:] @ H for idx in indices]) - spectrum_1D._plot.signal_plot.ax.plot(self.axes_manager.signal_axes[0].axis, component, label=f'{conv_elts_dict[elt]}', linestyle=line_styles[_%len(line_styles)], color=colors[_%len(colors)]) - _+=1 + component = sum([G[:, idx] @ W[idx, :] @ H for idx in indices]) + spectrum_1D._plot.signal_plot.ax.plot( + self.axes_manager.signal_axes[0].axis, + component, + label=f"{conv_elts_dict[elt]}", + linestyle=line_styles[_ % len(line_styles)], + color=colors[_ % len(colors)], + ) + _ += 1 spectrum_1D._plot.signal_plot.ax.legend() - def concentration_report(self, selected_elts = [], W_input = None, fit_error = True) : - if W_input is None : - assert isinstance(self.learning_results.decomposition_algorithm,NMFEstimator), "No espm learning results available, please run a decomposition with an espm algorithm first" - + def concentration_report(self, selected_elts=[], W_input=None, fit_error=True): + if W_input is None: + assert isinstance( + self.learning_results.decomposition_algorithm, NMFEstimator + ), ( + "No espm learning results available, please run a decomposition with an espm algorithm first" + ) + W = self.learning_results.decomposition_algorithm.W_ G = self.learning_results.decomposition_algorithm.G_ H = self.learning_results.decomposition_algorithm.H_ - N = get_explained_intensity_W(G,W,H) + N = get_explained_intensity_W(G, W, H) sqN = np.sqrt(N) - percentages = sqN / N *100 + percentages = sqN / N * 100 - else : + else: W = W_input fit_error = False - @number_to_symbol_list - def convert_elts(elements = []) : + def convert_elts(elements=[]): return elements elts = self.model.get_elements(False) elts_indices = self.model.NMF_simplex() - if selected_elts : + if selected_elts: conv_elts = convert_elts(elements=elts) - conv_elts_dict = {conv_elts[i] : num for i, num in enumerate(elts_indices)} + conv_elts_dict = {conv_elts[i]: num for i, num in enumerate(elts_indices)} new_elts_indices = [] - for elt in selected_elts : - if elt in conv_elts_dict.keys() : + for elt in selected_elts: + if elt in conv_elts_dict: new_elts_indices.append(conv_elts_dict[elt]) - W = W[new_elts_indices,:]*100 /W[new_elts_indices,:].sum(axis = 0) - if fit_error : + W = W[new_elts_indices, :] * 100 / W[new_elts_indices, :].sum(axis=0) + if fit_error: errors = percentages[new_elts_indices, :] errors[errors > 10000] = np.inf - else : + else: errors = np.zeros_like(W) return selected_elts, W, errors - else : + else: conv_elts = convert_elts(elements=elts) - W = W[elts_indices,:]*100 # /W[indices,:].sum(axis = 0) - if fit_error : + W = W[elts_indices, :] * 100 # /W[indices,:].sum(axis = 0) + if fit_error: errors = percentages[elts_indices, :] errors[errors > 10000] = np.inf - else : + else: errors = np.zeros_like(W) return conv_elts, W, errors - def estimate_best_binning(self, inspect = False) : + def estimate_best_binning(self, inspect=False): r""" Estimate the best binning for the dataset based on the method developed by G. Obozinski, N. Perraudin and M. Martinez Ruts. - M. Martinez Ruts has designed an estimator that compares the binned and unbinned data and its minimum gives the best binning factor. + M. Martinez Ruts has designed an estimator that compares the binned and unbinned data and its minimum gives the best binning factor. Parameters ---------- @@ -857,48 +1011,65 @@ def estimate_best_binning(self, inspect = False) : # TODO : Write a document explaining the method L = self.axes_manager[2].size K = self.axes_manager[0].size * self.axes_manager[1].size - - facx = np.arange(1, self.axes_manager[0].size//2+1) - facy = np.arange(1, self.axes_manager[1].size//2+1) - binx = [self.axes_manager[0].size/i for i in facx] - biny = [self.axes_manager[1].size/i for i in facy] + facx = np.arange(1, self.axes_manager[0].size // 2 + 1) + facy = np.arange(1, self.axes_manager[1].size // 2 + 1) + + binx = [self.axes_manager[0].size / i for i in facx] + biny = [self.axes_manager[1].size / i for i in facy] vars_est = np.array([]) biases_est = np.array([]) bprod = [] - for i in zip(binx,biny): - bprod.append((i[0],i[1])) + for i in zip(binx, biny): + bprod.append((i[0], i[1])) for i in tqdm(bprod): # Bin the measurement dataset and upsample to bring it back to its original dimensionality - B = i[0]*i[1] - binned = self.rebin(scale = (i[0], i[1], 1)) - upsampled = binned.rebin(new_shape = (self.axes_manager[0].size, self.axes_manager[1].size, self.axes_manager[2].size)) - upsampled_data = upsampled.data - data = self.data - - # Estimator of variance (Lemma 4.3) - \widehat{Var} (\hat{y}_i) = \alpha ^2 y_{i}+ (1-\alpha)^2 \sum_{k \in \mathcal{K}} (w_k^2 n_{i,k}) - vars_est = np.append(vars_est, np.mean(upsampled_data*1/B)) - - # Estimator of squared bias (Lemma 4.4) - \widehat{Bias^2}(\hat{y}_i) = (1-\alpha)^2\left((y_{n_i} - y_{i})^2 - \sum_{k\in \mathcal{K}} w_k^2y_{i,k} - y_{i} \right) - biases_est = np.append(biases_est, np.mean((data-upsampled_data)**2 - 1/B*upsampled_data - (1-2/B)*data)) - - mprimes_est = vars_est*K/L + biases_est - estimated_binning = (bprod[np.argmin(mprimes_est)][0], bprod[np.argmin(mprimes_est)][1],1) - if inspect : - return mprimes_est, estimated_binning - else : + B = i[0] * i[1] + binned = self.rebin(scale=(i[0], i[1], 1)) + upsampled = binned.rebin( + new_shape=( + self.axes_manager[0].size, + self.axes_manager[1].size, + self.axes_manager[2].size, + ) + ) + upsampled_data = upsampled.data + data = self.data + + # Estimator of variance (Lemma 4.3) - \widehat{Var} (\hat{y}_i) = \alpha ^2 y_{i}+ (1-\alpha)^2 \sum_{k \in \mathcal{K}} (w_k^2 n_{i,k}) + vars_est = np.append(vars_est, np.mean(upsampled_data * 1 / B)) + + # Estimator of squared bias (Lemma 4.4) - \widehat{Bias^2}(\hat{y}_i) = (1-\alpha)^2\left((y_{n_i} - y_{i})^2 - \sum_{k\in \mathcal{K}} w_k^2y_{i,k} - y_{i} \right) + biases_est = np.append( + biases_est, + np.mean( + (data - upsampled_data) ** 2 + - 1 / B * upsampled_data + - (1 - 2 / B) * data + ), + ) + + mprimes_est = vars_est * K / L + biases_est + estimated_binning = ( + bprod[np.argmin(mprimes_est)][0], + bprod[np.argmin(mprimes_est)][1], + 1, + ) + if inspect: + return mprimes_est, estimated_binning + else: return estimated_binning - def define_ROI(self, xray_lines : bool = True) -> RectangularROI : + def define_ROI(self, xray_lines: bool = True) -> RectangularROI: r""" A function to define a rectangular ROI on an HyperSpy EDXS signal. - + Parameters ---------- xray_lines : bool If True, it displays the xray lines markers ok exspy. It might slow down the execution though. - + Returns ------- roi : hs.roi.RectangularROI @@ -906,90 +1077,101 @@ def define_ROI(self, xray_lines : bool = True) -> RectangularROI : """ scale_x = self.axes_manager[0].scale scale_y = self.axes_manager[1].scale - + centre_x = self.data.shape[1] * scale_x / 2 centre_y = self.data.shape[0] * scale_y / 2 dx = self.data.shape[1] * scale_x / 10 dy = self.data.shape[0] * scale_y / 10 - - roi = RectangularROI(left = centre_x - dx, top = centre_y - dy, right = centre_x + dx, bottom = centre_y + dy) + + roi = RectangularROI( + left=centre_x - dx, + top=centre_y - dy, + right=centre_x + dx, + bottom=centre_y + dy, + ) self.plot() - imr = roi.interactive(self, color = 'r') + imr = roi.interactive(self, color="r") selected_spectrum = hs.interactive(imr.mean) selected_spectrum.metadata.General.title = "Spectrum of the selected area" - selected_spectrum.plot(xray_lines = xray_lines) - + selected_spectrum.plot(xray_lines=xray_lines) + return roi - - def generate_part_fixed_H_matrix(self, rois : list[RectangularROI] = None, value : float = 1) -> np.ndarray : + + def generate_part_fixed_H_matrix( + self, rois: list[RectangularROI] = None, value: float = 1 + ) -> np.ndarray: r""" A function to generate a component of the fixed H matrix for one phase. - + Parameters ---------- ROIs : list A list of rectangular ROIs given by the user. value : float Value of the non-negative entries in the partial H matrix. Must be between 0 and 1. - + Returns ------- part_f_H : np.ndarray A fixed H matrix for one phase. """ - part_f_H = (-1) * np.ones(shape = (self.data.shape[0], self.data.shape[1]), dtype = float) - - if value < 0 : + part_f_H = (-1) * np.ones( + shape=(self.data.shape[0], self.data.shape[1]), dtype=float + ) + + if value < 0: raise ValueError("Value must be above 0.") # if value < 1 : # print("The value of some of the fixed_H matrix is below 1.0.", # "In most cases, it means that you want to use simplex_H = True and simplex_W = False in smoothNMF decomposition.") - if rois is None or rois == [] : + if rois is None or rois == []: return part_f_H - + # We don't enter that part of the code if there are no ROIs so it should be fine - if issubclass(type(rois[0]), BaseROI) : + if issubclass(type(rois[0]), BaseROI): for roi in rois: region_parameters = roi.parameters scale_i = self.axes_manager[0].scale scale_j = self.axes_manager[1].scale - j_min = int(region_parameters['left'] // scale_j) - i_min = int(region_parameters['top'] // scale_i) - j_max = int(region_parameters['right'] // scale_j) - i_max = int(region_parameters['bottom'] // scale_i) + j_min = int(region_parameters["left"] // scale_j) + i_min = int(region_parameters["top"] // scale_i) + j_max = int(region_parameters["right"] // scale_j) + i_max = int(region_parameters["bottom"] // scale_i) part_f_H[i_min:i_max, j_min:j_max] = value - + return part_f_H - - def set_fixed_H(self, areas_dict : dict[str,np.ndarray]) -> np.ndarray : + + def set_fixed_H(self, areas_dict: dict[str, np.ndarray]) -> np.ndarray: r""" Helper function to generate a fixed H matrix for the SmoothNMF decomposition algorithm. The output matrix will have -1 entries except for the areas that are specified in the input dictionary. The -1 entries will be ignored during the decomposition and learned normally, while the non-negative entries will be kept fixed. - + Parameters ---------- areas_dict : dict Determines which areas are going to be non-negative. The dictionary has the following structure: areas_dict = {"p0" : part_f_H_0, "p1" : part_f_H_1 ...} where part_f_H_0, part_f_H_1, ... are NumPy arrays with the same dimensions as the input data's spatial dimensions. They are generated using the generate_part_fixed_H_matrix() function. - + Returns ------- H : numpy.ndarray A fixed H matrix for the SmoothNMF decomposition algorithm. """ - - H = (-1) * np.ones(shape = (len(areas_dict), self.data.shape[0], self.data.shape[1]), dtype = float) - + + H = (-1) * np.ones( + shape=(len(areas_dict), self.data.shape[0], self.data.shape[1]), dtype=float + ) + for i, p in enumerate(areas_dict): H[i, :, :] = areas_dict[p] - + return H.reshape((len(areas_dict), self.data.shape[0] * self.data.shape[1])) @_check_decomposition - def elemental_mapping(self,*,skipped_elements : list =[]) -> None: + def elemental_mapping(self, *, skipped_elements: list = []) -> None: r""" Performs pixel-wise elemental quantification using the results of an espm decomposition. Results are stored in self.quantification_signal and self.quantification_list. @@ -1003,80 +1185,99 @@ def elemental_mapping(self,*,skipped_elements : list =[]) -> None: ------- None """ - + est = self.learning_results.decomposition_algorithm _W = est.W_ H = est.H_ @number_to_symbol_list - def sym_elts(elements = []) : + def sym_elts(elements=[]): return elements - + @symbol_to_number_list - def num_elts(elements = []) : + def num_elts(elements=[]): return elements - _elts = sym_elts(elements = self.model.get_elements(False)) + _elts = sym_elts(elements=self.model.get_elements(False)) _elts_indices = self.model.NMF_simplex() - skipped_elts = sym_elts(elements = skipped_elements) + skipped_elts = sym_elts(elements=skipped_elements) elts = [] elts_indices = [] - for i,elt in enumerate(_elts) : - if elt in skipped_elts : + for i, elt in enumerate(_elts): + if elt in skipped_elts: pass - else : + else: elts.append(elt) elts_indices.append(_elts_indices[i]) - W = _W[elts_indices,:] - WH = (W@H) - WH = WH.reshape([W.shape[0]]+list(self.data.shape[:-1])) - - WH/=WH.sum(0)[np.newaxis,...]/100 + W = _W[elts_indices, :] + WH = W @ H + WH = WH.reshape([W.shape[0]] + list(self.data.shape[:-1])) + + WH /= WH.sum(0)[np.newaxis, ...] / 100 if self.axes_manager.navigation_dimension == 2: Signal = hs.signals.Signal2D - elif self.axes_manager.navigation_dimension ==1: + elif self.axes_manager.navigation_dimension == 1: Signal = hs.signals.Signal1D - qs = [Signal(WH[i], - metadata = {"General":{"name":el, - "title":el+" Quantification"}},colorbar_label="A") for i,el in enumerate(elts)] - + qs = [ + Signal( + WH[i], + metadata={"General": {"name": el, "title": el + " Quantification"}}, + colorbar_label="A", + ) + for i, el in enumerate(elts) + ] + for q in qs: for i in range(self.axes_manager.navigation_dimension): - q.axes_manager[i].update_from(self.axes_manager[i],["units","scale","name","offset"]) + q.axes_manager[i].update_from( + self.axes_manager[i], ["units", "scale", "name", "offset"] + ) q.metadata.Signal.quantity = "Atomic %" wh = Signal(WH) for i in range(self.axes_manager.navigation_dimension): - wh.axes_manager[1+i].update_from(self.axes_manager[i],["units","scale","name","offset"]) - + wh.axes_manager[1 + i].update_from( + self.axes_manager[i], ["units", "scale", "name", "offset"] + ) + self.quantification_list = qs self.quantification_signal = wh - for k,m in self.metadata: self.quantification_signal.metadata.set_item(k,m) + for k, m in self.metadata: + self.quantification_signal.metadata.set_item(k, m) self.quantification_signal.metadata.set_item("Sample.elements", elts) - self.quantification_signal.metadata.set_item("Signal.quantity" ,"Atomic %") + self.quantification_signal.metadata.set_item("Signal.quantity", "Atomic %") self.quantification_signal.axes_manager[0].name = "Elements" if self.quantification_signal.metadata.has_item("Sample.xray_lines"): - self.quantification_signal.metadata.set_item("Sample.xray_lines" , [i for i in self.quantification_signal.metadata.Sample.xray_lines if not i.split("_")[0] in skipped_elts]) + self.quantification_signal.metadata.set_item( + "Sample.xray_lines", + [ + i + for i in self.quantification_signal.metadata.Sample.xray_lines + if i.split("_")[0] not in skipped_elts + ], + ) self.quantification_signal_1d = self.quantification_signal.as_signal1D(0) - #hack to label elements. Horrible, I know. + + # hack to label elements. Horrible, I know. def label_elements(): - self.quantification_signal._plot.navigator_plot.ax.set_xticks(list(range(len(elts))),elts) - return + self.quantification_signal._plot.navigator_plot.ax.set_xticks( + list(range(len(elts))), elts + ) - self.quantification_signal.axes_manager[0].events.index_changed.connect(label_elements,[]) + self.quantification_signal.axes_manager[0].events.index_changed.connect( + label_elements, [] + ) hs.plot.plot_images(qs) - return - @_check_decomposition - def plot_comp_model(self,comp_index : int) : + def plot_comp_model(self, comp_index: int): r""" Plots espm model of the component #comp_index, showing contributions of each element and background. @@ -1084,7 +1285,7 @@ def plot_comp_model(self,comp_index : int) : ---------- comp_index : int Component index for which the model plot is built. - + Returns ------- Figure : matplotlib.pyplot.figure @@ -1092,22 +1293,28 @@ def plot_comp_model(self,comp_index : int) : """ idx = comp_index - els = self.get_full_el_list()+["Background 1","Background 2"] - self.GW = self.learning_results.decomposition_algorithm.G_@self.learning_results.decomposition_algorithm.W_ - gs,cs = self.learning_results.decomposition_algorithm.W_.shape - G_idx = self.learning_results.decomposition_algorithm.G_*self.learning_results.decomposition_algorithm.W_[:,idx] - + els = self.get_full_el_list() + ["Background 1", "Background 2"] + self.GW = ( + self.learning_results.decomposition_algorithm.G_ + @ self.learning_results.decomposition_algorithm.W_ + ) + gs, cs = self.learning_results.decomposition_algorithm.W_.shape + G_idx = ( + self.learning_results.decomposition_algorithm.G_ + * self.learning_results.decomposition_algorithm.W_[:, idx] + ) + x = self.axes_manager[-1].axis plt.figure() - plt.plot(x,self.GW[:,idx],"k--",label="Component") + plt.plot(x, self.GW[:, idx], "k--", label="Component") for i in range(gs): - color = list(mpl.colors.TABLEAU_COLORS.values())[i%10] - plt.plot(x,G_idx[:,i],color = color,label = els[i]) - plt.fill_between(x,G_idx[:,i],alpha = 0.4,color = color) + color = list(mpl.colors.TABLEAU_COLORS.values())[i % 10] + plt.plot(x, G_idx[:, i], color=color, label=els[i]) + plt.fill_between(x, G_idx[:, i], alpha=0.4, color=color) plt.legend() ax = plt.gca() - plt.title("Model of component {}".format(str(idx))) - + plt.title(f"Model of component {idx!s}") + return plt.gcf() @_check_decomposition @@ -1120,7 +1327,7 @@ def plot_data_model(self): ---------- WH : np.ndarray WH model. by default is taken from the current decomposition. - + Returns ------- None @@ -1132,37 +1339,44 @@ def plot_data_model(self): if self.learning_results.navigation_mask is not None: H = self.fix_masked_H() - WH = W@H - - contributions = [hs.signals.Signal1D((G[:,[i]]@(WH)[[i],:]).T.reshape(self.data.shape)) for i in range(G.shape[1])] - contributions.append(hs.signals.Signal1D((G@WH).T.reshape(self.data.shape))) - els = self.metadata.EDS_model.elements - titles =self.get_full_el_list()+["Background 1","Background 2","Full Model"] - for i,c in enumerate(contributions): - for a,b in zip(c.axes_manager._axes,self.axes_manager._axes): - a.update_from(b) - c.metadata.General.title = titles[i] - + WH = W @ H + + contributions = [ + hs.signals.Signal1D((G[:, [i]] @ (WH)[[i], :]).T.reshape(self.data.shape)) + for i in range(G.shape[1]) + ] + contributions.append(hs.signals.Signal1D((G @ WH).T.reshape(self.data.shape))) + els = self.metadata.EDS_model.elements + titles = self.get_full_el_list() + [ + "Background 1", + "Background 2", + "Full Model", + ] + for i, c in enumerate(contributions): + for a, b in zip(c.axes_manager._axes, self.axes_manager._axes): + a.update_from(b) + c.metadata.General.title = titles[i] if self.axes_manager.navigation_dimension == 1: nav = self.sum(-1).as_signal1D(0) position = hs.roi.Point1DROI(0) - nav_kwargs={"color":"blue"} + nav_kwargs = {"color": "blue"} elif self.axes_manager.navigation_dimension == 2: - nav = self.sum(-1).as_signal2D((0,1)) - position = hs.roi.Point2DROI(0,0) - nav_kwargs={} + nav = self.sum(-1).as_signal2D((0, 1)) + position = hs.roi.Point2DROI(0, 0) + nav_kwargs = {} nav.plot(**nav_kwargs) - position_interactive = position.interactive(self,nav,color="red") - positions_contribs = [position.interactive(g,None) for g in contributions] - hs.plot.plot_spectra([position_interactive]+positions_contribs,legend="auto", - linestyle=["-"]+["--" for i in positions_contribs], - color = ["k"]+list(mpl.colors.TABLEAU_COLORS.values())*10) - - return - + position_interactive = position.interactive(self, nav, color="red") + positions_contribs = [position.interactive(g, None) for g in contributions] + hs.plot.plot_spectra( + [position_interactive] + positions_contribs, + legend="auto", + linestyle=["-"] + ["--" for i in positions_contribs], + color=["k"] + list(mpl.colors.TABLEAU_COLORS.values()) * 10, + ) + @_check_decomposition def plot_data_model_ROI(self): r""" @@ -1170,14 +1384,14 @@ def plot_data_model_ROI(self): Parameters ---------- - None : + None : The data are taken from a previous decomposition. - + Returns ------- None """ - + W = self.learning_results.decomposition_algorithm.W_ G = self.learning_results.decomposition_algorithm.G_ H = self.learning_results.decomposition_algorithm.H_ @@ -1186,10 +1400,19 @@ def plot_data_model_ROI(self): WH = np.matmul(W, H) - contributions = [hs.signals.Signal1D((G[:, [i]] @ (WH)[[i], :]).T.reshape(self.data.shape)) for i in range(G.shape[1])] - contributions.append(hs.signals.Signal1D((np.matmul(G, WH)).T.reshape(self.data.shape))) + contributions = [ + hs.signals.Signal1D((G[:, [i]] @ (WH)[[i], :]).T.reshape(self.data.shape)) + for i in range(G.shape[1]) + ] + contributions.append( + hs.signals.Signal1D((np.matmul(G, WH)).T.reshape(self.data.shape)) + ) - titles = self.get_full_el_list() + ["Background 1", "Background 2", "Full Model"] + titles = self.get_full_el_list() + [ + "Background 1", + "Background 2", + "Full Model", + ] for i, c in enumerate(contributions): for a, b in zip(c.axes_manager._axes, self.axes_manager._axes): a.update_from(b) @@ -1198,26 +1421,37 @@ def plot_data_model_ROI(self): fig, ax = plt.subplots() self.plot() - roi = hs.roi.RectangularROI(left = 0, top = 0, right = self.axes_manager[1].size, bottom = self.axes_manager[0].size) - - imr = roi.interactive(self, color = 'green').sum(axis = 0).sum(axis = 0) - contributions_roi = [roi.interactive(g, None).sum(axis = 0).sum(axis = 0) for g in contributions] + roi = hs.roi.RectangularROI( + left=0, + top=0, + right=self.axes_manager[1].size, + bottom=self.axes_manager[0].size, + ) + + imr = roi.interactive(self, color="green").sum(axis=0).sum(axis=0) + contributions_roi = [ + roi.interactive(g, None).sum(axis=0).sum(axis=0) for g in contributions + ] spectra = [imr] + contributions_roi lines = [] - - line, = ax.plot(imr.data, label = imr.metadata.General.title, linestyle = "-") + + (line,) = ax.plot(imr.data, label=imr.metadata.General.title, linestyle="-") lines.append(line) - + for spectrum in contributions_roi: - line, = ax.plot(spectrum.data, label = spectrum.metadata.General.title, linestyle = "--") + (line,) = ax.plot( + spectrum.data, label=spectrum.metadata.General.title, linestyle="--" + ) lines.append(line) - + ax.legend() def update_plot(*args, **kwargs): - imr = roi.interactive(self, color = 'green').sum(axis = 0).sum(axis = 0) - contributions_roi = [roi.interactive(g, None).sum(axis = 0).sum(axis = 0) for g in contributions] + imr = roi.interactive(self, color="green").sum(axis=0).sum(axis=0) + contributions_roi = [ + roi.interactive(g, None).sum(axis=0).sum(axis=0) for g in contributions + ] all_data = [imr] + contributions_roi for line, new_data in zip(lines, all_data): @@ -1231,34 +1465,38 @@ def update_plot(*args, **kwargs): update_plot() plt.show() - return - @_check_decomposition - def elemental_profile(self,**kwargs): + def elemental_profile(self, **kwargs): r""" - Plots quantification profiles of all elements. + Plots quantification profiles of all elements. Parameters ---------- **kwargs are passed to Line2DROI - + Returns ------- Quantification profiles """ line = hs.roi.Line2DROI(**kwargs) p1 = hs.signals.Signal2D(self.data.sum(-1)) - if not self.learning_results.navigation_mask is None: - p1.data[self.learning_results.navigation_mask.reshape(self.data.shape[:-1])]=np.nan + if self.learning_results.navigation_mask is not None: + p1.data[ + self.learning_results.navigation_mask.reshape(self.data.shape[:-1]) + ] = np.nan for i in range(2): - p1.axes_manager[i].update_from(self.axes_manager[i],["units","scale","name","offset"]) + p1.axes_manager[i].update_from( + self.axes_manager[i], ["units", "scale", "name", "offset"] + ) p1.plot() - line.interactive(p1,color="red") - p_contrib = [line.interactive(g,None) for g in self.quantification_list]#here are the profiles stored + line.interactive(p1, color="red") + p_contrib = [ + line.interactive(g, None) for g in self.quantification_list + ] # here are the profiles stored for p in p_contrib: p.axes_manager[0].name = "Profile" p.metadata.Signal.quantity = "Atomic %" - hs.plot.plot_spectra(p_contrib,legend = "auto") + hs.plot.plot_spectra(p_contrib, legend="auto") ax = plt.gca() ax.set_ylabel("Atomic %") return p_contrib @@ -1268,35 +1506,59 @@ def calibrate_from_lines(self) -> None: Selects two Xray lines from a 1D EDXS spectrum for further calibration by the apply_interactive_calibration method. """ print("Instructions : ") - print("1. Select two X-ray lines (ideally far apart). The energies of the fitted peaks appears nearby.") - print("2. Note down those values and use : apply_interactive_calibration(e1,e2), where e1 and e2 are the energy values in keV.") - self._gauss_means=np.zeros(2) - a = self.sum((0,1)) + print( + "1. Select two X-ray lines (ideally far apart). The energies of the fitted peaks appears nearby." + ) + print( + "2. Note down those values and use : apply_interactive_calibration(e1,e2), where e1 and e2 are the energy values in keV." + ) + self._gauss_means = np.zeros(2) + a = self.sum((0, 1)) b = a.deepcopy() - b.data=np.zeros(b.data.shape) + b.data = np.zeros(b.data.shape) a.plot() - hs.plot.plot_spectra([b,b],fig=plt.gcf(),ax=plt.gca(),color="k",linewidth=2) + hs.plot.plot_spectra( + [b, b], fig=plt.gcf(), ax=plt.gca(), color="k", linewidth=2 + ) eax = self.axes_manager[-1].axis ne = self.axes_manager[-1].axis.shape[0] - roi1 = hs.roi.SpanROI(left=eax[ne//5],right=eax[2*ne//5]) - roi_signal1 = roi1.interactive(a,color="blue") + roi1 = hs.roi.SpanROI(left=eax[ne // 5], right=eax[2 * ne // 5]) + roi_signal1 = roi1.interactive(a, color="blue") - roi2 = hs.roi.SpanROI(left=eax[3*ne//5],right=eax[4*ne//5]) + roi2 = hs.roi.SpanROI(left=eax[3 * ne // 5], right=eax[4 * ne // 5]) roi_signal2 = roi2.interactive(a) - hs.interactive(self.fit_plot_gauss,event = roi1.events.changed,roi_signal = roi_signal1,a=a,roi = roi1,i=1) - hs.interactive(self.fit_plot_gauss,event = roi2.events.changed,roi_signal = roi_signal2,a=a,roi = roi2,i=2) + hs.interactive( + self.fit_plot_gauss, + event=roi1.events.changed, + roi_signal=roi_signal1, + a=a, + roi=roi1, + i=1, + ) + hs.interactive( + self.fit_plot_gauss, + event=roi2.events.changed, + roi_signal=roi_signal2, + a=a, + roi=roi2, + i=2, + ) # print("When ready, run self.apply_interactive_calibration(enery_left_peak,energy_right_peak)") - def apply_interactive_calibration(self,energy_left_peak : float,energy_right_peak : float) -> None : + def apply_interactive_calibration( + self, energy_left_peak: float, energy_right_peak: float + ) -> None: r""" Applies the calibration on two peaks previously selected using calibrate_from_lines. It modifies the axes of the object. """ - #Dumbass hyperspy keeps events linked and has no "remove events" method - self.axes_manager.events.any_axis_changed.trigger = hyperspy.events.Event().trigger - self.axes_manager.events.any_axis_changed._connected_some={} + # Dumbass hyperspy keeps events linked and has no "remove events" method + self.axes_manager.events.any_axis_changed.trigger = ( + hyperspy.events.Event().trigger + ) + self.axes_manager.events.any_axis_changed._connected_some = {} current_e1 = min(self._gauss_means) current_e2 = max(self._gauss_means) @@ -1305,56 +1567,64 @@ def apply_interactive_calibration(self,energy_left_peak : float,energy_right_pea old_scale = eax.scale old_offset = eax.offset - new_scale = old_scale*(energy_right_peak-energy_left_peak)/(current_e2-current_e1) - new_offset = energy_right_peak-(current_e2-old_offset)*new_scale/old_scale + new_scale = ( + old_scale + * (energy_right_peak - energy_left_peak) + / (current_e2 - current_e1) + ) + new_offset = ( + energy_right_peak - (current_e2 - old_offset) * new_scale / old_scale + ) print(new_scale) print(new_offset) with self.axes_manager.events.any_axis_changed.suppress(): self.axes_manager[-1].scale = new_scale self.axes_manager[-1].offset = new_offset - return - def fit_plot_gauss(self,roi_signal,a,roi,i): + def fit_plot_gauss(self, roi_signal, a, roi, i): x = a.axes_manager[-1].axis y = np.zeros(a.data.shape) eax = a.axes_manager[-1] - y[eax.value2index(roi.left):eax.value2index(roi.right)]=roi_signal.data - - mean = (x*y).sum()/y.sum() + y[eax.value2index(roi.left) : eax.value2index(roi.right)] = roi_signal.data + + mean = (x * y).sum() / y.sum() - sigma = np.sqrt((y * (x - mean)**2).sum() / y.sum()) - popt,_pcov = curve_fit(Gauss, x, y, p0=[max(y),mean ,sigma]) - self._gauss_means[i-1]=popt[1] + sigma = np.sqrt((y * (x - mean) ** 2).sum() / y.sum()) + popt, _pcov = curve_fit(Gauss, x, y, p0=[max(y), mean, sigma]) + self._gauss_means[i - 1] = popt[1] fit_mean = popt[1] - + fig = plt.gcf() ax = fig.axes[0] l = fig.axes[0].lines[i] - l.set_ydata(Gauss(x,*popt)) + l.set_ydata(Gauss(x, *popt)) # --- Interactive Text Handling --- text_label = f"Peak energy: {fit_mean:.2f} keV" - + # Look for an existing text object belonging to this specific index/plot text_obj = None for txt in ax.texts: if txt.get_gid() == f"gauss_text_{i}": text_obj = txt break - + if text_obj: # Update the existing text and reposition it dynamically if needed text_obj.set_text(text_label) - text_obj.set_position((fit_mean, max(y) * 0.9)) + text_obj.set_position((fit_mean, max(y) * 0.9)) else: # Create a new text object and give it a unique Group ID (gid) # transform=ax.transData places it relative to your data coordinates ax.text( - fit_mean, max(y) * 0.9, text_label, - color=l.get_color(), fontweight='bold', - bbox=dict(facecolor='white', alpha=0.6, edgecolor='none'), - gid=f"gauss_text_{i}" + fit_mean, + max(y) * 0.9, + text_label, + color=l.get_color(), + fontweight="bold", + bbox=dict(facecolor="white", alpha=0.6, edgecolor="none"), + gid=f"gauss_text_{i}", ) fig.canvas.draw() @@ -1368,48 +1638,62 @@ def get_full_el_list(self): els_names = [num_to_symbol(el) for el in els] return els_names + ####################### # Auxiliary functions # ####################### -def get_metadata(spim) : + +def get_metadata(spim): r""" Get the metadata of the :class:`EDSespm` object and format it as a model parameters dictionary. """ mod_pars = {} - try : + try: mod_pars["E0"] = spim.metadata.Acquisition_instrument.TEM.beam_energy mod_pars["e_offset"] = spim.axes_manager[-1].offset - assert mod_pars["e_offset"] > 0.01, "The energy scale can't include 0, it will produce errors elsewhere. Please crop your data." + assert mod_pars["e_offset"] > 0.01, ( + "The energy scale can't include 0, it will produce errors elsewhere. Please crop your data." + ) mod_pars["e_scale"] = spim.axes_manager[-1].scale mod_pars["e_size"] = spim.axes_manager[-1].size mod_pars["db_name"] = spim.metadata.xray_db - mod_pars["width_slope"] = spim.metadata.Acquisition_instrument.TEM.Detector.EDS.width_slope - mod_pars["width_intercept"] = spim.metadata.Acquisition_instrument.TEM.Detector.EDS.width_intercept - + mod_pars["width_slope"] = ( + spim.metadata.Acquisition_instrument.TEM.Detector.EDS.width_slope + ) + mod_pars["width_intercept"] = ( + spim.metadata.Acquisition_instrument.TEM.Detector.EDS.width_intercept + ) + pars_dict = {} pars_dict["Abs"] = { - "thickness" : spim.metadata.Sample.thickness, - "toa" : spim.metadata.Acquisition_instrument.TEM.Detector.EDS.take_off_angle, - "density" : spim.metadata.Sample.density + "thickness": spim.metadata.Sample.thickness, + "toa": spim.metadata.Acquisition_instrument.TEM.Detector.EDS.take_off_angle, + "density": spim.metadata.Sample.density, } - try : - pars_dict["Det"] = spim.metadata.Acquisition_instrument.TEM.Detector.EDS.type.as_dictionary() - except AttributeError : - pars_dict["Det"] = spim.metadata.Acquisition_instrument.TEM.Detector.EDS.type + try: + pars_dict["Det"] = ( + spim.metadata.Acquisition_instrument.TEM.Detector.EDS.type.as_dictionary() + ) + except AttributeError: + pars_dict["Det"] = ( + spim.metadata.Acquisition_instrument.TEM.Detector.EDS.type + ) mod_pars["params_dict"] = pars_dict - except AttributeError : - print("You need to define the relevant parameters for the analysis. Use the set_analysis_parameters function.") + except AttributeError: + print( + "You need to define the relevant parameters for the analysis. Use the set_analysis_parameters function." + ) return mod_pars -def build_G(model, g_params) : + +def build_G(model, g_params): model.generate_g_matr(**g_params) return model.G - def Gauss(x, a, x0, sigma): - return a * np.exp(-(x - x0)**2 / (2 * sigma**2)) + return a * np.exp(-((x - x0) ** 2) / (2 * sigma**2)) diff --git a/espm/datasets/eds_spim_sim.py b/espm/datasets/eds_spim_sim.py index d587431..d0cb004 100644 --- a/espm/datasets/eds_spim_sim.py +++ b/espm/datasets/eds_spim_sim.py @@ -1,11 +1,11 @@ from espm.datasets.eds_spim import EDSespm -class EDSespmSimulated(EDSespm) : +class EDSespmSimulated(EDSespm): _signal_type = "EDS_espm_Simulated" - def __init__ (self,*args,**kwargs) : - super().__init__(*args,**kwargs) + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) self._phases = None self._maps = None self._maps_2d = None @@ -15,57 +15,56 @@ def __init__ (self,*args,**kwargs) : # Properties # ############## - def _set_default_analysis_params(self) : - # TODO : make them fetch preferences from the user + def _set_default_analysis_params(self): + # TODO : make them fetch preferences from the user super()._set_default_analysis_params() md = self.metadata md.Signal.signal_type = "EDS_espm_Simulated" @property - def maps (self) : + def maps(self): r""" Ground truth of the spatial distribution of the phases in the form of a 3D array of shape (shape_2d[0],shape_2d[1],n_phases), if available. - """ - if self._maps is None : + """ + if self._maps is None: self._maps = self.build_ground_truth()[1] return self._maps @property - def phases (self) : + def phases(self): r""" Ground truth of the spectra of the phases in the form of a 2D array of shape (n_phases,n_features), if available. """ - if self._phases is None : + if self._phases is None: self._phases = self.build_ground_truth()[0] return self._phases @property - def maps_2d (self) : + def maps_2d(self): r""" Ground truth of the spatial distribution of the phases in the form of a 2D array of shape (shape_2d[0]*shape_2d[1],n_phases), if available. """ - if self._maps_2d is None : - self._maps_2d = self.build_ground_truth(reshape = False)[1] + if self._maps_2d is None: + self._maps_2d = self.build_ground_truth(reshape=False)[1] return self._maps_2d - - + @property - def Xdot (self) : + def Xdot(self): r""" The ground truth in the form of a 3D array of shape (shape_2d[0],shape_2d[1],n_features), if available. """ - if self._Xdot is None : - try : + if self._Xdot is None: + try: self._Xdot = self.phases @ self.maps - except AttributeError : + except AttributeError: print("This dataset contains no ground truth. Nothing was done.") return self._Xdot - + ###################################### # Modelling and simulation functions # ###################################### - def build_ground_truth(self,reshape = True) : + def build_ground_truth(self, reshape=True): r""" Get the ground truth stored in the metadata of the :class:`EDSespm` object, if available. The reshape arguments can be used to get the ground truth in a form easier to use for machine learning algorithms. @@ -73,7 +72,7 @@ def build_ground_truth(self,reshape = True) : ---------- reshape : bool, optional If False, the ground truth is returned in the form of a 3D array of shape (shape_2d[0],shape_2d[1],n_phases) and a 2D array of shape (n_phases,n_features). - + Returns ------- phases : numpy.ndarray @@ -81,13 +80,14 @@ def build_ground_truth(self,reshape = True) : weights : numpy.ndarray The ground truth of the spatial distribution of the phases. """ - if "phases" in self.metadata.Truth.Data : + if "phases" in self.metadata.Truth.Data: phases = self.metadata.Truth.Data.phases weights = self.metadata.Truth.Data.weights - if reshape : + if reshape: phases = phases.T - weights = weights.reshape((weights.shape[0]*weights.shape[1], weights.shape[2])).T - else : + weights = weights.reshape( + (weights.shape[0] * weights.shape[1], weights.shape[2]) + ).T + else: raise AttributeError("There is no ground truth contained in this dataset") return phases, weights - diff --git a/espm/estimators/__init__.py b/espm/estimators/__init__.py index 40c2fb0..8d67555 100644 --- a/espm/estimators/__init__.py +++ b/espm/estimators/__init__.py @@ -1,13 +1,12 @@ -# -*- coding: utf-8 -*- - -r""" NMF Estimators. +r"""NMF Estimators. The :mod:`espm.estimators` module implements different NMF algorithms. The class :mod:`espm.estimators` is an abstract class for all NMF algorithms. It implements the fit and transform methods. -The fit method is implemented in the abstract class and calls the `_iteration` method which is implemented in the child classes. +The fit method is implemented in the abstract class and calls the `_iteration` method which is implemented in the child classes. The transform method is implemented in the child classes. """ + from espm.estimators.base import NMFEstimator -from espm.estimators.smooth_nmf import SmoothNMF \ No newline at end of file +from espm.estimators.smooth_nmf import SmoothNMF diff --git a/espm/estimators/base.py b/espm/estimators/base.py index 33e86c4..04fdb2c 100644 --- a/espm/estimators/base.py +++ b/espm/estimators/base.py @@ -1,24 +1,25 @@ -import numpy as np -from sklearn.base import BaseEstimator, TransformerMixin -from sklearn.utils.validation import check_is_fitted -from espm.estimators.updates import initialize_algorithms -from espm.measures import KLdiv_loss, Frobenius_loss, find_min_angle, find_min_MSE -from espm.conf import log_shift -from espm.utils import rescaled_DH import time from abc import ABC, abstractmethod -from espm.utils import create_laplacian_matrix + +import numpy as np from scipy.sparse import lil_matrix +from sklearn.base import BaseEstimator, TransformerMixin +from sklearn.utils.validation import check_is_fitted, validate_data + +from espm.conf import log_shift +from espm.estimators.updates import initialize_algorithms +from espm.measures import Frobenius_loss, KLdiv_loss, find_min_angle, find_min_MSE from espm.models.base import PhysicalModel -from sklearn.utils.validation import validate_data, check_array +from espm.utils import create_laplacian_matrix, rescaled_DH -def normalization_factor (X, nc) : +def normalization_factor(X, nc): m = np.mean(X) - return nc/(m*X.shape[0]) + return nc / (m * X.shape[0]) + class NMFEstimator(ABC, TransformerMixin, BaseEstimator): - r""" Abstract class for NMF algorithms. + r"""Abstract class for NMF algorithms. This abstract class `espm.estimators.NMFEstimator` is used to implement the different NMF algorithms. It solves problems of the form: @@ -26,11 +27,11 @@ class NMFEstimator(ABC, TransformerMixin, BaseEstimator): \dot{W}, \dot{H} = \arg \min_{W \geq \epsilon, H \geq \epsilon} \frac{1}{2} L(X, GWH) + R(W, H) - where - :math:`X` is the data matrix, - :math:`G` is a matrix of known values, - :math:`W` and :math:`H` are the matrices to be learned, - :math:`R` is a regularization term and + where + :math:`X` is the data matrix, + :math:`G` is a matrix of known values, + :math:`W` and :math:`H` are the matrices to be learned, + :math:`R` is a regularization term and :math:`L` is a loss function. that represent the generalized KL divergence. As a reminder, the generalized KL divergence is defined as: .. math:: @@ -44,7 +45,7 @@ class NMFEstimator(ABC, TransformerMixin, BaseEstimator): L(X, Y) = - \sum_{i,j} X_{ij} \log \frac{GWH_{ij}} + GWH_{ij} The Generalized KL divergence has the advantage of being zero when :math:`X = Y`, which is not the case for our loss. - Therefore, we shift the loss function by a constant :math:`C` such that it equals the Generalized KL divergence. + Therefore, we shift the loss function by a constant :math:`C` such that it equals the Generalized KL divergence. This constant is stored in the attribute :attr:`espm.estimators.NMFEstimator.const_KL_`. The loss function can also be selected to be the Frobenius norm. In this case, the loss function is: @@ -71,11 +72,11 @@ class NMFEstimator(ABC, TransformerMixin, BaseEstimator): Number of components, i.e. dimensionality of the latent space. init : str Method used to initialize the procedure. Default is None - The method use the initialization of :mod:`sklearn.decomposition`. + The method use the initialization of :mod:`sklearn.decomposition`. It can be imported using: .. code-block::python >>> from sklearn.decomposition._nmf import _initialize_nmf - + tol : float, default=1e-4 Tolerance of the stopping condition. max_iter : int, default=200 @@ -88,8 +89,8 @@ class NMFEstimator(ABC, TransformerMixin, BaseEstimator): l2 : bool, default=False If True, the algorithm will use the l2 norm instead of the KL divergence. G : np.array, function or None, default=None - If np.array, it is the known matrix of the data. - If function, it is a function that takes as input the data matrix and returns the known matrix (np.array). + If np.array, it is the known matrix of the data. + If function, it is a function that takes as input the data matrix and returns the known matrix (np.array). If None, it is assumed that G is the identity matrix. shape_2d : tuple or None, default=None If not None, it is the image shape of the columns of the matrices :math:`X` and :math:`H`. @@ -104,7 +105,7 @@ class NMFEstimator(ABC, TransformerMixin, BaseEstimator): true_H : np.array or None, default=None Ground truth for the matrix :math:`H`. Used for evaluation purposes. fixed_H : np.array or None, default=None - If not None, it fixes the non-zero values of the matrix :math:`H`. + If not None, it fixes the non-zero values of the matrix :math:`H`. Note that convergence is not guaranteed with fixed_H enabled. fixed_W : np.array or None, default=None If not None, it fixes the non-zero values of the matrix :math:`W`. @@ -120,15 +121,34 @@ class NMFEstimator(ABC, TransformerMixin, BaseEstimator): >>> out = spim.decomposition(algorithm = est, return_info=True) """ + loss_names_ = ["KL_div_loss"] const_KL_ = None - - def __init__(self, n_components=2, init=None, tol=1e-4, max_iter=200, - random_state=None, verbose=1, debug=False, - l2=False, G=None, shape_2d = None, normalize = False, log_shift=log_shift, - eval_print=10, true_D = None, true_H = None, fixed_H = None, fixed_W = None, hspy_comp = False, - no_stop_criterion = False, simplex_H=False, simplex_W = True - ): + + def __init__( + self, + n_components=2, + init=None, + tol=1e-4, + max_iter=200, + random_state=None, + verbose=1, + debug=False, + l2=False, + G=None, + shape_2d=None, + normalize=False, + log_shift=log_shift, + eval_print=10, + true_D=None, + true_H=None, + fixed_H=None, + fixed_W=None, + hspy_comp=False, + no_stop_criterion=False, + simplex_H=False, + simplex_W=True, + ): self.n_components = n_components self.init = init self.tol = tol @@ -157,14 +177,13 @@ def __sklearn_tags__(self): return tags def _more_tags(self): - return {'requires_positive_X': True} + return {"requires_positive_X": True} @abstractmethod - def _iteration(self, W, H): + def _iteration(self, W, H): pass - - def loss(self, W, H, average=True, X = None): + def loss(self, W, H, average=True, X=None): """Loss function. Compute the loss function for the given matrices W and H. @@ -187,20 +206,22 @@ def loss(self, W, H, average=True, X = None): """ GW = self.G_ @ W - if X is None : + if X is None: X = self.X_ - assert(X.shape == (self.G_.shape[0],H.shape[1])) + assert X.shape == (self.G_.shape[0], H.shape[1]) self.GWH_numel_ = self.G_.shape[0] * H.shape[1] - + if self.l2: - loss_ = 0.5*Frobenius_loss(X, GW, H, average=False) + loss_ = 0.5 * Frobenius_loss(X, GW, H, average=False) else: if self.const_KL_ is None: - self.const_KL_ = np.sum(X*np.log(np.maximum(self.X_, self.log_shift))) - np.sum(X) + self.const_KL_ = np.sum( + X * np.log(np.maximum(self.X_, self.log_shift)) + ) - np.sum(X) - loss_ = KLdiv_loss(X, GW, H, self.log_shift, average=False) + self.const_KL_ + loss_ = KLdiv_loss(X, GW, H, self.log_shift, average=False) + self.const_KL_ if average: loss_ = loss_ / self.GWH_numel_ self.detailed_loss_ = [loss_] @@ -234,62 +255,67 @@ def fit_transform(self, X, y=None, W=None, H=None): ------- GW : ndarrays Transformed data. - + """ ############################ # Initialize the algorithm # ############################ - if self.hspy_comp : - self.X_ = validate_data(self,X.T, dtype=[np.float64, np.float32]) + if self.hspy_comp: + self.X_ = validate_data(self, X.T, dtype=[np.float64, np.float32]) - else : - self.X_ = validate_data(self,X, dtype=[np.float64, np.float32]) + else: + self.X_ = validate_data(self, X, dtype=[np.float64, np.float32]) - if self.hspy_comp==False: + if self.hspy_comp == False: try: import inspect + curframe = inspect.currentframe() calframe = inspect.getouterframes(curframe, 2) functions_called = [i.function for i in calframe] if "decomposition" in functions_called: - print("Are you calling the function decomposition from Hyperspy?\n" + - "If so, please set the compatibility argument 'hspy_comp' to True.\n\n" + - "If this argument is not set correctly, the function will not work properly!!!") + print( + "Are you calling the function decomposition from Hyperspy?\n" + + "If so, please set the compatibility argument 'hspy_comp' to True.\n\n" + + "If this argument is not set correctly, the function will not work properly!!!" + ) except: pass # The algorithm does not work when full columns or lines of X are zero self.X_ = self.remove_zeros_lines(self.X_, self.log_shift) self.const_KL_ = None - if self.normalize : + if self.normalize: # We normalize the data so that the strength of the regularization is somewhat the same for all datasets - self.norm_factor_ = normalization_factor(self.X_,self.n_components) + self.norm_factor_ = normalization_factor(self.X_, self.n_components) self.X_ = self.norm_factor_ * self.X_ - + if isinstance(self.G, PhysicalModel): self.physics_model_ = self.G G = self.physics_model_.NMF_update() else: self.physics_model_ = None G = self.G - - self.G_, self.W_, self.H_ = initialize_algorithms(X = self.X_, - G = G, - W = W, - H = H, - n_components = self.n_components, - init = self.init, - random_state = self.random_state, - simplex_H = self.simplex_H, - simplex_W = self.simplex_W, - physics_model = self.physics_model_) - - if not(self.shape_2d is None) : + + self.G_, self.W_, self.H_ = initialize_algorithms( + X=self.X_, + G=G, + W=W, + H=H, + n_components=self.n_components, + init=self.init, + random_state=self.random_state, + simplex_H=self.simplex_H, + simplex_W=self.simplex_W, + physics_model=self.physics_model_, + ) + + if self.shape_2d is not None: self.L_ = create_laplacian_matrix(*self.shape_2d) - else : - self.L_ =lil_matrix((self.X_.shape[1],self.X_.shape[1]),dtype=np.float32) - self.L_.setdiag([1]*self.X_.shape[1]) + else: + self.L_ = lil_matrix((self.X_.shape[1], self.X_.shape[1]), dtype=np.float32) + self.L_.setdiag([1] * self.X_.shape[1]) algo_start = time.time() eval_before = np.inf @@ -299,15 +325,19 @@ def fit_transform(self, X, y=None, W=None, H=None): self.losses_ = [] self.rel_ = [] self.detailed_losses_ = [] - if not(self.true_D is None) and not(self.true_H is None) : - if (self.true_D.shape[1] == self.n_components) and (self.true_H.shape[0] == self.n_components) : + if self.true_D is not None and self.true_H is not None: + if (self.true_D.shape[1] == self.n_components) and ( + self.true_H.shape[0] == self.n_components + ): self.angles_ = [] self.mse_ = [] self.true_losses_ = [] true_DH = self.true_D @ self.true_H - else : - print("The chosen number of components does not match the number of components of the provided truth. The ground truth will be ignored.") - + else: + print( + "The chosen number of components does not match the number of components of the provided truth. The ground truth will be ignored." + ) + ############# # Main loop # ############# @@ -315,59 +345,65 @@ def fit_transform(self, X, y=None, W=None, H=None): while True: # Take one step in W, H old_W, old_H = self.W_.copy(), self.H_.copy() - - self.W_, self.H_ = self._iteration(self.W_, self.H_ ) + + self.W_, self.H_ = self._iteration(self.W_, self.H_) eval_after = self.loss(self.W_, self.H_) - self.n_iter_ +=1 - - rel_W = np.max(np.abs((self.W_ - old_W))/(self.W_ + self.tol*np.mean(self.W_) )) - rel_H = np.max(np.abs((self.H_ - old_H))/(self.H_ + self.tol*np.mean(self.H_) )) + self.n_iter_ += 1 + + rel_W = np.max( + np.abs(self.W_ - old_W) / (self.W_ + self.tol * np.mean(self.W_)) + ) + rel_H = np.max( + np.abs(self.H_ - old_H) / (self.H_ + self.tol * np.mean(self.H_)) + ) # store some information for assessing the convergence # for debugging purposes - - # We need to store this value as + + # We need to store this value as # loss = self.loss(self.P_,self.A_, X = true_DA ) - # might destroy it. Furthermore, saving the data before the if, might cause - # an error if the optimization is stoped with a keyboard interrupt. + # might destroy it. Furthermore, saving the data before the if, might cause + # an error if the optimization is stoped with a keyboard interrupt. detailed_loss_ = self.detailed_loss_ - if not(self.true_D is None) and not(self.true_H is None) : - if (self.true_D.shape[1] == self.n_components) and (self.true_H.shape[0] == self.n_components) : + if self.true_D is not None and self.true_H is not None: + if (self.true_D.shape[1] == self.n_components) and ( + self.true_H.shape[0] == self.n_components + ): if self.simplex_H or self.simplex_W: - W, H = self.W_, self.H_ + W, H = self.W_, self.H_ else: - W, H = rescaled_DH(self.W_, self.H_ ) + W, H = rescaled_DH(self.W_, self.H_) GW = self.G_ @ W - angles = find_min_angle(self.true_D.T,GW.T, unique=True) - mse = find_min_MSE(self.true_H, H,unique=True) - loss = self.loss(self.W_,H, X = true_DH ) + angles = find_min_angle(self.true_D.T, GW.T, unique=True) + mse = find_min_MSE(self.true_H, H, unique=True) + loss = self.loss(self.W_, H, X=true_DH) self.angles_.append(angles) self.mse_.append(mse) self.true_losses_.append(loss) - + self.losses_.append(eval_after) self.detailed_losses_.append(detailed_loss_) - self.rel_.append([rel_W,rel_H]) - + self.rel_.append([rel_W, rel_H]) + # check convergence criterions if self.n_iter_ >= self.max_iter: print("exits because max_iteration was reached") break - + if not self.no_stop_criterion: # If there is no regularization the algorithm stops with this criterion # Otherwise it goes to the data fitting step - if max(rel_H,rel_W) < self.tol: + if max(rel_H, rel_W) < self.tol: print( - "exits because of relative change rel_A {} and rel_P {} < tol ".format( - rel_H,rel_W - ) + f"exits because of relative change rel_A {rel_H} and rel_P {rel_W} < tol " ) break - elif abs((eval_before - eval_after)/eval_init) < self.tol: - print("exits because of relative change < tol: {}".format((eval_before - eval_after)/eval_init)) + elif abs((eval_before - eval_after) / eval_init) < self.tol: + print( + f"exits because of relative change < tol: {(eval_before - eval_after) / eval_init}" + ) break elif np.isnan(eval_after): @@ -375,21 +411,22 @@ def fit_transform(self, X, y=None, W=None, H=None): break elif (eval_before - eval_after) < 0: - print("exit because of negative decrease {}: {}, {}".format((eval_before - eval_after), eval_before, eval_after)) + print( + f"exit because of negative decrease {eval_before - eval_after}: {eval_before}, {eval_after}" + ) break - + if self.verbose > 0 and np.mod(self.n_iter_, self.eval_print) == 0: # I added a log shift since check_estimator seems to run so fast that elapsed time is 0 print( - f"It {self.n_iter_} / {self.max_iter}: loss {eval_after:3e}, {self.n_iter_/(time.time()-algo_start + log_shift):0.3f} it/s", + f"It {self.n_iter_} / {self.max_iter}: loss {eval_after:3e}, {self.n_iter_ / (time.time() - algo_start + log_shift):0.3f} it/s", ) - pass # Update G might increase the loss so we reevaluate the loss to avoid artificial negative decrease # We do this update every 3 iterations, but it is arbitrary. - if self.physics_model_ != None and self.n_iter_%3 == 0: + if self.physics_model_ != None and self.n_iter_ % 3 == 0: self.G_ = self.physics_model_.NMF_update(self.W_) eval_before = self.loss(self.W_, self.H_) - else : + else: eval_before = eval_after except KeyboardInterrupt: pass @@ -397,31 +434,31 @@ def fit_transform(self, X, y=None, W=None, H=None): ################### # End of the loop # ################### - if not(self.simplex_H) and not(self.simplex_W): - self.W_, self.H_ = rescaled_DH(self.W_, self.H_ ) - + if not (self.simplex_H) and not (self.simplex_W): + self.W_, self.H_ = rescaled_DH(self.W_, self.H_) + algo_time = time.time() - algo_start print( - f"Stopped after {self.n_iter_} iterations in {algo_time//60} minutes " + f"Stopped after {self.n_iter_} iterations in {algo_time // 60} minutes " f"and {np.round(algo_time) % 60} seconds." ) self.reconstruction_err_ = self.loss(self.W_, self.H_) - if self.normalize : + if self.normalize: self.W_ = self.W_ / self.norm_factor_ - + GW = self.G_ @ self.W_ self.n_components_ = self.H_.shape[0] - - if self.hspy_comp : + + if self.hspy_comp: self.components_ = GW.T return self.H_.T - else : + else: self.components_ = self.H_ return GW def fit(self, X, y=None, **params): - """ Learn a NMF model for the data X. + """Learn a NMF model for the data X. Parameters ---------- @@ -430,7 +467,7 @@ def fit(self, X, y=None, **params): y : Ignored params : dict Parameters passed to the `fit_transform` method. - + Returns ------- @@ -460,71 +497,87 @@ def fit(self, X, y=None, **params): # return self.P_ def inverse_transform(self, W): - """ Transform data back to its original space. + """Transform data back to its original space. Parameters ---------- W : {ndarray, sparse matrix} of shape (n_samples, n_components) Transformed data matrix. - + Returns ------- - + X : {ndarray, sparse matrix} of shape (n_samples, n_features) Data matrix of original shape. - + """ check_is_fitted(self) return self.G_ @ W @ self.H_ - + def get_losses(self): """ - For debug purposes : return the evolution of losses. - """ - if not(self.true_D is None) and not(self.true_H is None) : + For debug purposes : return the evolution of losses. + """ + if self.true_D is not None and self.true_H is not None: mse_list = [] angles_list = [] - for i in range(self.n_components) : - angles_list.append("ang_p{}".format(i)) - mse_list.append("mse_p{}".format(i)) - names = ["full_loss"] + self.loss_names_ + ["rel_W","rel_H"] + angles_list + mse_list + ["true_KL_loss"] + for i in range(self.n_components): + angles_list.append(f"ang_p{i}") + mse_list.append(f"mse_p{i}") + names = ( + ["full_loss"] + + self.loss_names_ + + ["rel_W", "rel_H"] + + angles_list + + mse_list + + ["true_KL_loss"] + ) dt_list = [] - for elt in names : - dt_list.append((elt,"float64")) + for elt in names: + dt_list.append((elt, "float64")) dt = np.dtype(dt_list) tup_list = [] - for i in range(len(self.losses_)) : - tup_list.append((self.losses_[i],) + tuple(self.detailed_losses_[i]) + tuple(self.rel_[i]) \ - + tuple(self.angles_[i]) + tuple(self.mse_[i]) + (self.true_losses_[i],) ) - - array = np.array(tup_list,dtype=dt) - - #TODO : Check this part for optimization - else : - names = ["full_loss"] + self.loss_names_ + ["rel_W","rel_H"] + for i in range(len(self.losses_)): + tup_list.append( + (self.losses_[i],) + + tuple(self.detailed_losses_[i]) + + tuple(self.rel_[i]) + + tuple(self.angles_[i]) + + tuple(self.mse_[i]) + + (self.true_losses_[i],) + ) + + array = np.array(tup_list, dtype=dt) + + # TODO : Check this part for optimization + else: + names = ["full_loss"] + self.loss_names_ + ["rel_W", "rel_H"] dt_list = [] - for elt in names : - dt_list.append((elt,"float64")) + for elt in names: + dt_list.append((elt, "float64")) dt = np.dtype(dt_list) tup_list = [] - for i in range(len(self.losses_)) : - tup_list.append((self.losses_[i],) + tuple(self.detailed_losses_[i]) + tuple(self.rel_[i])) - - array = np.array(tup_list,dtype=dt) + for i in range(len(self.losses_)): + tup_list.append( + (self.losses_[i],) + + tuple(self.detailed_losses_[i]) + + tuple(self.rel_[i]) + ) + + array = np.array(tup_list, dtype=dt) return array - def remove_zeros_lines (self, X, epsilon) : - if np.all(X >= 0) : + def remove_zeros_lines(self, X, epsilon): + if np.all(X >= 0): new_X = X.copy() - sum_cols = X.sum(axis = 0) - sum_rows = X.sum(axis = 1) - new_X[:,np.where(sum_cols == 0)] = epsilon - new_X[np.where(sum_rows == 0),:] = epsilon + sum_cols = X.sum(axis=0) + sum_rows = X.sum(axis=1) + new_X[:, np.where(sum_cols == 0)] = epsilon + new_X[np.where(sum_rows == 0), :] = epsilon return new_X - else : + else: raise ValueError("Negative values in data") - diff --git a/espm/estimators/dicotomy.py b/espm/estimators/dicotomy.py index 9cffa49..6e27cc9 100644 --- a/espm/estimators/dicotomy.py +++ b/espm/estimators/dicotomy.py @@ -1,52 +1,57 @@ import numpy as np + from espm.conf import dicotomy_tol, log_shift, maxit_dichotomy -def dichotomy_simplex(num, denum, log_shift=log_shift, tol=dicotomy_tol, maxit=maxit_dichotomy): + +def dichotomy_simplex( + num, denum, log_shift=log_shift, tol=dicotomy_tol, maxit=maxit_dichotomy +): """ Function to solve the num/(x+denum) -1 = 0 equation. Here, x is the Lagragian multiplier which is used to apply the simplex constraint. - The first part consists in finding a and b such that num/(a+denum) -1 > 0 and num/(b+denum) -1 < 0. + The first part consists in finding a and b such that num/(a+denum) -1 > 0 and num/(b+denum) -1 < 0. The second part applies the dichotomy algorithm to solve the equation. """ # The function has exactly one root at the right of the first singularity (the singularity at min(denum)) - + # num = num.astype("float64") # denum = denum.astype("float64") # do some test - assert((num>=0).all()) - assert((denum>=0).all()) - assert((np.sum(num, axis=0)>0).all()) - if log_shift>0: + assert (num >= 0).all() + assert (denum >= 0).all() + assert (np.sum(num, axis=0) > 0).all() + if log_shift > 0: # Check that a solution is possible if denum.shape[0] * log_shift >= 1: raise ValueError("No solution exists!") - denum_max = num/log_shift + denum_max = num / log_shift else: denum_max = np.inf # Ideally we want to do this, but we have to exclude the case where num==0. # a = np.max(num/2 - denum, axis=0) - if denum.shape[1]>1: + if denum.shape[1] > 1: a = [] - for n,d in zip(num.T, denum.T): - m = n>0 + for n, d in zip(num.T, denum.T): + m = n > 0 # The divided by 2 is just a factor to help a bit. - a.append(np.max(n[m]/2 - d[m])) + a.append(np.max(n[m] / 2 - d[m])) a = np.array(a) - else: + else: # This else is just to preserve the size and make it work in any case... # There might be a possiblity to write this more elegantly - d = denum[:,0] + d = denum[:, 0] + def max_masked(n): - m = n>0 - return np.max(n[m]/2-d[m]) + m = n > 0 + return np.max(n[m] / 2 - d[m]) + a = np.apply_along_axis(max_masked, 0, num) - - + # r = np.sum(num/denum, axis=0) # b = np.zeros(r.shape) # b[r>=1] = (len(num) * np.max(num, axis=0)/0.5 - np.min(denum, axis=0))[r>=1] - b = len(num) * np.max(num, axis=0)/0.5 - np.min(denum, axis=0) + b = len(num) * np.max(num, axis=0) / 0.5 - np.min(denum, axis=0) def func(x): new_x = x + denum @@ -54,58 +59,67 @@ def func(x): return dicotomy(a, b, func, maxit, tol) -def dichotomy_simplex_acc(a, b, minus_c, log_shift=log_shift, tol=dicotomy_tol, maxit=maxit_dichotomy): + +def dichotomy_simplex_acc( + a, b, minus_c, log_shift=log_shift, tol=dicotomy_tol, maxit=maxit_dichotomy +): """ Function to solve the dicotomy for the function: f(nu) = n_p * nu_k + 2a - sum_p sqrt ( (b_p + nu)^2 - 4 a c_p) + sum_p b_p - The first part consists in finding nu_max and nu_min such that f(nu_max) > 0 and f(nu_min) < 0. + The first part consists in finding nu_max and nu_min such that f(nu_max) > 0 and f(nu_min) < 0. The second part applies the dichotomy algorithm to solve the equation. """ # do some test - assert(a>=0) - assert((minus_c>=0).all()) + assert a >= 0 + assert (minus_c >= 0).all() - if log_shift>0: + if log_shift > 0: # Check that a solution is possible if b.shape[0] * log_shift >= 1: raise ValueError("No solution exists!") - n_p = len(b) - nu_max = n_p * np.max(b**2/a+2*a+2*(b+ minus_c), axis=0) * 1.5 + 1e-3 - nu_min = - (2 * a + np.sum(b, axis=0))/ n_p * 1.1 - 1e-3 - + n_p = len(b) + nu_max = n_p * np.max(b**2 / a + 2 * a + 2 * (b + minus_c), axis=0) * 1.5 + 1e-3 + nu_min = -(2 * a + np.sum(b, axis=0)) / n_p * 1.1 - 1e-3 + def func(x): - return 2*a - np.sum( np.maximum(np.sqrt( (b + x)**2 + 4*a*minus_c) - x - b, log_shift*2*a), axis=0) + return 2 * a - np.sum( + np.maximum( + np.sqrt((b + x) ** 2 + 4 * a * minus_c) - x - b, log_shift * 2 * a + ), + axis=0, + ) return dicotomy(nu_max, nu_min, func, maxit, tol) -def dichotomy_simplex_projected_gradient(a, log_shift=log_shift, tol=dicotomy_tol, maxit=maxit_dichotomy): + +def dichotomy_simplex_projected_gradient( + a, log_shift=log_shift, tol=dicotomy_tol, maxit=maxit_dichotomy +): r""" Function to solve the dicotomy for the function: - .. :math:: + .. :math:: - f(\nu) = \sum_p \max \left( \alpha_p+\nu, \epsilon right) - 1 = 0 + f(\nu) = \sum_p \max \left( \alpha_p+\nu, \epsilon right) - 1 = 0 - The first part consists in finding nu_max and nu_min such that f(nu_max) > 0 and f(nu_min) < 0. + The first part consists in finding nu_max and nu_min such that f(nu_max) > 0 and f(nu_min) < 0. The second part applies the dichotomy algorithm to solve the equation. """ - if log_shift>0: + if log_shift > 0: # Check that a solution is possible if a.shape[0] * log_shift >= 1: raise ValueError("No solution exists!") - nu_min = -np.max(a, axis=0) - nu_max = 1/a.shape[0] - np.min(a, axis=0) - - def func(x): - return np.sum( np.maximum(a + x, log_shift), axis=0) -1 + nu_max = 1 / a.shape[0] - np.min(a, axis=0) + def func(x): + return np.sum(np.maximum(a + x, log_shift), axis=0) - 1 - return dicotomy(nu_max, nu_min, func, maxit, tol) + return dicotomy(nu_max, nu_min, func, maxit, tol) def dicotomy(a, b, func, maxit, tol): @@ -125,7 +139,7 @@ def dicotomy(a, b, func, maxit, tol): Maximum number of iterations - the algorithm stops if |func(sol)| < tol tol : float Tolerance - the algorithm stops if |func(sol)| < tol - + Returns ------- @@ -138,26 +152,25 @@ def dicotomy(a, b, func, maxit, tol): func_max = func(a) func_min = func(b) - assert(np.sum(func_min>=0)==0) - assert(np.sum(func_max<=0)==0) - assert(np.sum(np.isnan(func_max))==0) - assert(np.sum(np.isnan(func_min))==0) + assert np.sum(func_min >= 0) == 0 + assert np.sum(func_max <= 0) == 0 + assert np.sum(np.isnan(func_max)) == 0 + assert np.sum(np.isnan(func_min)) == 0 # Dichotomy algorithm to solve the equation it = 0 - new = (a + b)/2 + new = (a + b) / 2 func_new = func(new) # print("A : {}, B: {}, new : {}, fA : {}, fB : {}, fnew : {}".format(np.min(np.abs(a)),np.min(np.abs(b)),np.min(np.abs(new)),np.min(np.abs(func(a))),np.min(np.abs(func(b))),np.min(np.abs(func(new))))) # print("A : {}, B: {}, new : {}, fA : {}, fB : {}, fnew : {}".format(np.max(a),np.max(b),np.max(new),np.max(func(a)),np.max(func(b)),np.max(func(new)))) while np.max(np.abs(func_new)) > tol: - - it=it+1 + it = it + 1 func_a = func(a) # func_b = func(b) # if f(a)*f(new) <0 then f(new) < 0 --> store in b minus_bool = func_a * func_new <= 0 - + # if f(a)*f(new) > 0 then f(new) > 0 --> store in a # plus_bool = func_a * func_new > 0 plus_bool = np.logical_not(minus_bool) @@ -166,8 +179,10 @@ def dicotomy(a, b, func, maxit, tol): a[plus_bool] = new[plus_bool] new = (a + b) / 2 func_new = func(new) - if it>=maxit: - print("Dicotomy stopped for maximum number of iterations with an error of : {}".format(np.max(np.abs(func_new)))) + if it >= maxit: + print( + f"Dicotomy stopped for maximum number of iterations with an error of : {np.max(np.abs(func_new))}" + ) break - + return new diff --git a/espm/estimators/smooth_nmf.py b/espm/estimators/smooth_nmf.py index 6093ba2..15da68a 100644 --- a/espm/estimators/smooth_nmf.py +++ b/espm/estimators/smooth_nmf.py @@ -1,19 +1,32 @@ +from copy import deepcopy + import numpy as np -from espm.estimators.updates import multiplicative_step_h, multiplicative_step_w, multiplicative_step_hq, proj_grad_step_h, proj_grad_step_w, gradH, gradW, estimate_Lipschitz_bound_h, estimate_Lipschitz_bound_w -from espm.measures import trace_xtLx, log_reg + +from espm.conf import dicotomy_tol, log_shift, sigmaL from espm.estimators import NMFEstimator from espm.estimators.surrogates import diff_surrogate, quadratic_surrogate -from espm.conf import dicotomy_tol, sigmaL -from copy import deepcopy -from espm.conf import log_shift +from espm.estimators.updates import ( + estimate_Lipschitz_bound_h, + estimate_Lipschitz_bound_w, + gradH, + gradW, + multiplicative_step_h, + multiplicative_step_hq, + multiplicative_step_w, + proj_grad_step_h, + proj_grad_step_w, +) +from espm.measures import log_reg, trace_xtLx + # from espm.measures import KL_loss_surrogate, KLdiv_loss, log_reg, log_surrogate + class SmoothNMF(NMFEstimator): r"""SmoothNMF - NMF with a smooth regularization term - We encourage to read the example available in the documentation: - https://espm.readthedocs.io/en/latest/introduction/notebooks/toy-problem.html - + We encourage to read the example available in the documentation: + https://espm.readthedocs.io/en/latest/introduction/notebooks/toy-problem.html + The corresponding notebook is available on github: https://github.com/adriente/espm/blob/main/notebooks/toy-ML.ipynb @@ -26,7 +39,7 @@ class SmoothNMF(NMFEstimator): \lambda_L tr(H \Delta H^\top) + \mu \sum_{ij} \log(H_{ij} + \epsilon_{reg}) where: - + - `D_{GKL}` is the Generalized KL divergence loss function defined as: .. math:: @@ -34,13 +47,13 @@ class SmoothNMF(NMFEstimator): D_{GKL}(X || Y) = \sum_{i,j} X_{ij} \log \frac{X_{ij}}{Y_{ij}} - X_{ij} + Y_{ij} See the documentation of the class :mod:`espm.estimators.NMFEstimator` for more details. - + - `\Delta` is the Laplacian operator (it can be created using the function `create_laplacian_matrix` from the `utils` module). - + - `\epsilon_{reg}` is the slope of the log regularization/sparsity at 0 (you probably want to leave this to 1). - `\lambda_L` is a regularization parameter, which encourages smoothness in the columns of `H`. - + - `\mu` is a regularization parameter, which is similar to an L1 sparsity penalty. The size of: @@ -52,7 +65,7 @@ class SmoothNMF(NMFEstimator): The columns of the matrices `H` and `X` are assumed to be images, typically for the smoothness regularization. The parameter `shape_2d` defines the shape of the images, i.e., `shape_2d[0]*shape_2d[1] = p`. - + Parameters ---------- lambda_L : float, default=1.0 @@ -76,63 +89,68 @@ class SmoothNMF(NMFEstimator): **kwargs : dict Additional parameters for the `NMFEstimator` class. - """ + """ - loss_names_ = NMFEstimator.loss_names_ + ["log_reg_loss"] + ["Lapl_reg_loss"] + ["gamma"] + loss_names_ = ( + NMFEstimator.loss_names_ + ["log_reg_loss"] + ["Lapl_reg_loss"] + ["gamma"] + ) # args and kwargs are copied from the init to the super instead of capturing them in *args and **kwargs to be scikit-learn compliant. - def __init__(self, - lambda_L = 0.0, - linesearch=False, - mu=0, - epsilon_reg=1, - algo="log_surrogate", - dicotomy_tol=dicotomy_tol, - gamma=None, - n_components=2, - init=None, - tol=1e-4, - max_iter=200, - random_state=None, - verbose=1, - debug=False, - l2=False, - G=None, - shape_2d = None, - normalize = False, - log_shift=log_shift, - eval_print=10, - true_D = None, - true_H = None, - fixed_H = None, - fixed_W = None, - hspy_comp = False, - no_stop_criterion = False, - simplex_H=False, - simplex_W = True - ): - - super().__init__(n_components=n_components, - init=init, - tol=tol, - max_iter=max_iter, - random_state=random_state, - verbose=verbose, - debug=debug, - l2=l2, - G=G, - shape_2d = shape_2d, - normalize = normalize, - log_shift=log_shift, - eval_print=eval_print, - true_D = true_D, - true_H = true_H, - fixed_H = fixed_H, - fixed_W = fixed_W, - hspy_comp = hspy_comp, - no_stop_criterion = no_stop_criterion, - simplex_H=simplex_H, - simplex_W = simplex_W) + def __init__( + self, + lambda_L=0.0, + linesearch=False, + mu=0, + epsilon_reg=1, + algo="log_surrogate", + dicotomy_tol=dicotomy_tol, + gamma=None, + n_components=2, + init=None, + tol=1e-4, + max_iter=200, + random_state=None, + verbose=1, + debug=False, + l2=False, + G=None, + shape_2d=None, + normalize=False, + log_shift=log_shift, + eval_print=10, + true_D=None, + true_H=None, + fixed_H=None, + fixed_W=None, + hspy_comp=False, + no_stop_criterion=False, + simplex_H=False, + simplex_W=True, + ): + + super().__init__( + n_components=n_components, + init=init, + tol=tol, + max_iter=max_iter, + random_state=random_state, + verbose=verbose, + debug=debug, + l2=l2, + G=G, + shape_2d=shape_2d, + normalize=normalize, + log_shift=log_shift, + eval_print=eval_print, + true_D=true_D, + true_H=true_H, + fixed_H=fixed_H, + fixed_W=fixed_W, + hspy_comp=hspy_comp, + no_stop_criterion=no_stop_criterion, + simplex_H=simplex_H, + simplex_W=simplex_W, + ) self.lambda_L = lambda_L self.linesearch = linesearch self.mu = mu @@ -142,7 +160,7 @@ def __init__(self, self.gamma = gamma self.check_params() - def check_params(self) : + def check_params(self): """Check the parameters of the model.""" # Type checking of the parameters # TODO: Add typing from the __init__ method. I don't know why it didn't work ... @@ -199,40 +217,58 @@ def check_params(self) : print("The n_components parameter is set to 2") self.n_components = 2 # Value checking of the parameters - if not(self.algo in ["l2_surrogate", "log_surrogate", "projected_gradient", "bmd"]) : - print("The algorithm must be 'l2_surrogate', 'log_surrogate', 'bmd' or 'projected_gradient'") + if self.algo not in [ + "l2_surrogate", + "log_surrogate", + "projected_gradient", + "bmd", + ]: + print( + "The algorithm must be 'l2_surrogate', 'log_surrogate', 'bmd' or 'projected_gradient'" + ) print("The algorithm is set to 'log_surrogate'") self.algo = "log_surrogate" - if not(self.lambda_L >= 0) : + if not (self.lambda_L >= 0): print("The regularization parameter lambda_L must be non-negative") print("The regularization parameter lambda_L is set to 0") self.lambda_L = 0 - if not(self.epsilon_reg > 0.0) : + if not (self.epsilon_reg > 0.0): print("The regularization parameter epsilon_reg must be positive") print("The regularization parameter epsilon_reg is set to 1") - self.epsilon_reg = 1.0 - if not(np.all(np.array(self.mu)>=0)) : + self.epsilon_reg = 1.0 + if not (np.all(np.array(self.mu) >= 0)): print("The regularization parameter mu must be non-negative") print("The regularization parameter mu is set to 0") self.mu = 0 - if not((self.simplex_H and not(self.simplex_W)) or (not(self.simplex_H) and self.simplex_W) or (not(self.simplex_H) and not(self.simplex_W))) : - print("The simplex constraint must be applied to either W or H or none of them") + if not ( + (self.simplex_H and not (self.simplex_W)) + or (not (self.simplex_H) and self.simplex_W) + or (not (self.simplex_H) and not (self.simplex_W)) + ): + print( + "The simplex constraint must be applied to either W or H or none of them" + ) print("The simplex constraint is applied to W and not to H") self.simplex_W = True self.simplex_H = False if self.linesearch: - if self.l2 : + if self.l2: print("The l2 parameter must be False when using linesearch") print("The l2 parameter is set to False") - self.l2 = False - if not(self.lambda_L > 0) : - print("The regularization parameter lambda_L must be non-zero when using linesearch") + self.l2 = False + if not (self.lambda_L > 0): + print( + "The regularization parameter lambda_L must be non-zero when using linesearch" + ) print("The regularization parameter lambda_L is set to 1") self.lambda_L = 1 - if not(self.algo=="l2_surrogate") : - if self.l2 : - print("The l2 parameter must be False when using the algorithm "+self.algo) + if not (self.algo == "l2_surrogate"): + if self.l2: + print( + "The l2 parameter must be False when using the algorithm " + + self.algo + ) print("The l2 parameter is set to False") self.l2 = False @@ -244,8 +280,8 @@ def fit_transform(self, X, y=None, W=None, H=None): * :math:`X` is :math:`(n, p)`, * :math:`W` is :math:`(m, k)`, * :math:`H` is :math:`(k, p)`, - * :math:`G` is :math:`(n, m)`. - + * :math:`G` is :math:`(n, m)`. + Parameters ---------- X : array-like, shape (n, p) @@ -261,7 +297,7 @@ def fit_transform(self, X, y=None, W=None, H=None): ------- GW : ndarrays Transformed data. - + """ # To be remove in future versions. In this commented version below, G_ is called before intialisation which obviously causes issues. # I'll move it to the _iteration method. Adrien @@ -289,18 +325,21 @@ def _iteration(self, W, H): if self.n_iter_ == 0: if self.gamma is None: - if self.algo in ["l2_surrogate", "log_surrogate", "bmd"]: self.gamma_ = sigmaL else: - gamma_W = estimate_Lipschitz_bound_w(self.log_shift, self.X_, self.G_, k=self.n_components) - gamma_H = estimate_Lipschitz_bound_h(self.log_shift, - self.X_, - self.G_, - k=self.n_components, - lambda_L=self.lambda_L, - mu=self.mu, - epsilon_reg=self.epsilon_reg) + gamma_W = estimate_Lipschitz_bound_w( + self.log_shift, self.X_, self.G_, k=self.n_components + ) + gamma_H = estimate_Lipschitz_bound_h( + self.log_shift, + self.X_, + self.G_, + k=self.n_components, + lambda_L=self.lambda_L, + mu=self.mu, + epsilon_reg=self.epsilon_reg, + ) self.gamma_ = [gamma_H, gamma_W] else: self.gamma_ = deepcopy(self.gamma) @@ -308,143 +347,163 @@ def _iteration(self, W, H): # 1. Update for H if self.linesearch: Hold = H.copy() - if self.algo=="l2_surrogate": - H = multiplicative_step_hq(self.X_, - self.G_, - W, - H, - simplex_H=self.simplex_H, - log_shift=self.log_shift, - safe=self.debug, - dicotomy_tol=self.dicotomy_tol, - lambda_L=self.lambda_L, - L=self.L_, - sigmaL=self.gamma_, - fixed_H=self.fixed_H) - elif self.algo=="log_surrogate": - H = multiplicative_step_h(self.X_, - self.G_, - W, - H, - simplex_H=self.simplex_H, - mu=self.mu, - log_shift=self.log_shift, - epsilon_reg=self.epsilon_reg, - safe=self.debug, - dicotomy_tol=self.dicotomy_tol, - lambda_L=self.lambda_L, - L=self.L_, - l2=self.l2, - fixed_H=self.fixed_H, - sigmaL=self.gamma_) - elif self.algo=="projected_gradient": - H = proj_grad_step_h(self.X_, - self.G_, - W, - H, - simplex_H=self.simplex_H, - mu=self.mu, - log_shift=self.log_shift, - epsilon_reg=self.epsilon_reg, - safe=self.debug, - dicotomy_tol=self.dicotomy_tol, - lambda_L=self.lambda_L, - L=self.L_, - l2=self.l2, - fixed_H=self.fixed_H, - gamma=self.gamma_[0]) - elif self.algo=="bmd": - H = multiplicative_step_h(self.X_, - self.G_, - W, - H, - simplex_H=self.simplex_H, - mu=self.mu, - log_shift=self.log_shift, - epsilon_reg=self.epsilon_reg, - safe=self.debug, - dicotomy_tol=self.dicotomy_tol, - lambda_L=self.lambda_L, - L=self.L_, - l2=self.l2, - fixed_H=self.fixed_H, - sigmaL=self.gamma_, - use_bregman=True) + if self.algo == "l2_surrogate": + H = multiplicative_step_hq( + self.X_, + self.G_, + W, + H, + simplex_H=self.simplex_H, + log_shift=self.log_shift, + safe=self.debug, + dicotomy_tol=self.dicotomy_tol, + lambda_L=self.lambda_L, + L=self.L_, + sigmaL=self.gamma_, + fixed_H=self.fixed_H, + ) + elif self.algo == "log_surrogate": + H = multiplicative_step_h( + self.X_, + self.G_, + W, + H, + simplex_H=self.simplex_H, + mu=self.mu, + log_shift=self.log_shift, + epsilon_reg=self.epsilon_reg, + safe=self.debug, + dicotomy_tol=self.dicotomy_tol, + lambda_L=self.lambda_L, + L=self.L_, + l2=self.l2, + fixed_H=self.fixed_H, + sigmaL=self.gamma_, + ) + elif self.algo == "projected_gradient": + H = proj_grad_step_h( + self.X_, + self.G_, + W, + H, + simplex_H=self.simplex_H, + mu=self.mu, + log_shift=self.log_shift, + epsilon_reg=self.epsilon_reg, + safe=self.debug, + dicotomy_tol=self.dicotomy_tol, + lambda_L=self.lambda_L, + L=self.L_, + l2=self.l2, + fixed_H=self.fixed_H, + gamma=self.gamma_[0], + ) + elif self.algo == "bmd": + H = multiplicative_step_h( + self.X_, + self.G_, + W, + H, + simplex_H=self.simplex_H, + mu=self.mu, + log_shift=self.log_shift, + epsilon_reg=self.epsilon_reg, + safe=self.debug, + dicotomy_tol=self.dicotomy_tol, + lambda_L=self.lambda_L, + L=self.L_, + l2=self.l2, + fixed_H=self.fixed_H, + sigmaL=self.gamma_, + use_bregman=True, + ) else: raise ValueError("Unknown algorithm") if self.linesearch: if self.algo in ["l2_surrogate", "log_surrogate", "bmd"]: - d = diff_surrogate(Hold, H, L=self.L_, sigmaL=self.gamma_, algo=self.algo) - if d>0: - self.gamma_ = self.gamma_ / 1.05 + d = diff_surrogate( + Hold, H, L=self.L_, sigmaL=self.gamma_, algo=self.algo + ) + if d > 0: + self.gamma_ = self.gamma_ / 1.05 else: - self.gamma_ = self.gamma_ * 1.5 + self.gamma_ = self.gamma_ * 1.5 else: - gradf_xt = gradH(self.X_, - self.G_, - W, - Hold, - mu= self.mu, - lambda_L=self.lambda_L, - L=self.L_, - epsilon_reg=self.epsilon_reg, - log_shift=self.log_shift, - safe=self.debug) - f_xt = self.loss(W, Hold, X = self.X_, average=False) - f_x = self.loss(W, H, X = self.X_, average=False) + gradf_xt = gradH( + self.X_, + self.G_, + W, + Hold, + mu=self.mu, + lambda_L=self.lambda_L, + L=self.L_, + epsilon_reg=self.epsilon_reg, + log_shift=self.log_shift, + safe=self.debug, + ) + f_xt = self.loss(W, Hold, X=self.X_, average=False) + f_x = self.loss(W, H, X=self.X_, average=False) g_xxt = quadratic_surrogate(H, Hold, f_xt, gradf_xt, self.gamma_[0]) d = g_xxt - f_x - if d>0: - self.gamma_[0] = self.gamma_[0] / 1.05 + if d > 0: + self.gamma_[0] = self.gamma_[0] / 1.05 else: - self.gamma_[0] = self.gamma_[0] * 1.5 + self.gamma_[0] = self.gamma_[0] * 1.5 # 2. Update for W if self.algo in ["l2_surrogate", "log_surrogate"]: - W = multiplicative_step_w(self.X_, - self.G_, - W, - H, - log_shift=self.log_shift, - safe=self.debug, - l2=self.l2, - simplex_W=self.simplex_W, - fixed_W=self.fixed_W, - physics_model=self.physics_model_) - elif self.algo=="bmd": - W = multiplicative_step_w(self.X_, - self.G_, - W, - H, - log_shift=self.log_shift, - safe=self.debug, - l2=self.l2, - simplex_W=self.simplex_W, - fixed_W=self.fixed_W, - use_bregman=True, - physics_model=self.physics_model_) + W = multiplicative_step_w( + self.X_, + self.G_, + W, + H, + log_shift=self.log_shift, + safe=self.debug, + l2=self.l2, + simplex_W=self.simplex_W, + fixed_W=self.fixed_W, + physics_model=self.physics_model_, + ) + elif self.algo == "bmd": + W = multiplicative_step_w( + self.X_, + self.G_, + W, + H, + log_shift=self.log_shift, + safe=self.debug, + l2=self.l2, + simplex_W=self.simplex_W, + fixed_W=self.fixed_W, + use_bregman=True, + physics_model=self.physics_model_, + ) else: if self.linesearch: Wold = W.copy() - W = proj_grad_step_w(self.X_, - self.G_, - W, - H, - log_shift=self.log_shift, - safe=self.debug, - gamma=self.gamma_[1], - simplex_W=self.simplex_W) + W = proj_grad_step_w( + self.X_, + self.G_, + W, + H, + log_shift=self.log_shift, + safe=self.debug, + gamma=self.gamma_[1], + simplex_W=self.simplex_W, + ) if self.linesearch: - gradf_xt = gradW(self.X_, self.G_, Wold, H, log_shift=self.log_shift, safe=self.debug) - f_xt = self.loss(Wold, H, X = self.X_, average=False) - f_x = self.loss(W, H, X = self.X_, average=False) + gradf_xt = gradW( + self.X_, self.G_, Wold, H, log_shift=self.log_shift, safe=self.debug + ) + f_xt = self.loss(Wold, H, X=self.X_, average=False) + f_x = self.loss(W, H, X=self.X_, average=False) g_xxt = quadratic_surrogate(W, Wold, f_xt, gradf_xt, self.gamma_[1]) d = g_xxt - f_x - if d>0: - self.gamma_[1] = self.gamma_[1] / 1.05 + if d > 0: + self.gamma_[1] = self.gamma_[1] / 1.05 else: - self.gamma_[1] = self.gamma_[1] * 1.5 + self.gamma_[1] = self.gamma_[1] * 1.5 # KL_surr = KL_loss_surrogate(self.X_, W, H, Hold, eps=0) # log_surr = log_surrogate(H, Hold, mu=self.mu, epsilon=self.epsilon_reg) @@ -452,12 +511,12 @@ def _iteration(self, W, H): # KL_surr = KL_loss_surrogate(self.X_, W, H, H, eps=0) # log_surr = log_surrogate(H, H, mu=self.mu, epsilon=self.epsilon_reg) # print("loss after:", KL_surr, log_surr, log_surr+KL_surr) - return W, H + return W, H - def loss(self, W, H, average=True, X = None): + def loss(self, W, H, average=True, X=None): """Compute the loss function.""" - lkl = super().loss(W, H, average=average, X = X) - + lkl = super().loss(W, H, average=average, X=X) + reg = log_reg(H, self.mu, self.epsilon_reg, average=False) if average: reg = reg / self.GWH_numel_ @@ -472,4 +531,4 @@ def loss(self, W, H, average=True, X = None): else: self.detailed_loss_.append(self.gamma_) - return lkl + reg + l2 \ No newline at end of file + return lkl + reg + l2 diff --git a/espm/estimators/surrogates.py b/espm/estimators/surrogates.py index 52ccd1d..303415a 100644 --- a/espm/estimators/surrogates.py +++ b/espm/estimators/surrogates.py @@ -1,9 +1,9 @@ import numpy as np + from espm.conf import sigmaL from espm.measures import trace_xtLx - def smooth_l2_surrogate(Ht, L, H=None, sigmaL=sigmaL, lambda_L=1): r"""Compute the smooth L2 surrogate of the Laplacian regularizer :math:`\lambda_L/2 tr(H \Delta H^\top)` at :math:`H^t` @@ -13,12 +13,12 @@ def smooth_l2_surrogate(Ht, L, H=None, sigmaL=sigmaL, lambda_L=1): g(H,H^t ) = \frac{\lambda_L}{2} \left( tr\left( H^t \Delta H^{t\top} \right) + 2 tr \left(H^t \Delta (H - H^t)\top \right) + \sigma_L \| H - H^t \|_F^2 ) - where - + where + * :math:`H^t` is the current estimate of :math:`\dot{H}`, - * :math:`H` is the main variable of :math:`g`, + * :math:`H` is the main variable of :math:`g`, * :math:`\Delta` is the Laplacian matrix, - * :math:`\lambda_L` is the regularization parameter for the Laplacian regularizer,and + * :math:`\lambda_L` is the regularization parameter for the Laplacian regularizer,and * :math:`\sigma_L` is the maximum eigenvalue of the Laplacian :math:`\Delta`. Parameters @@ -33,7 +33,7 @@ def smooth_l2_surrogate(Ht, L, H=None, sigmaL=sigmaL, lambda_L=1): Maximum eigenvalue of the Laplacian :math:`\Delta`, by default sigmaL lambda_L : float, optional Regularization parameter for the Laplacian regularizer, by default 1 - + Returns ------- @@ -47,21 +47,23 @@ def smooth_l2_surrogate(Ht, L, H=None, sigmaL=sigmaL, lambda_L=1): t2 = t1 t3 = 0 else: - t2 = np.sum(HtTL * H ) - t3 = np.sum((Ht-H)**2) - return lambda_L / 2 * (2*t2 - t1 + sigmaL * t3) + t2 = np.sum(HtTL * H) + t3 = np.sum((Ht - H) ** 2) + return lambda_L / 2 * (2 * t2 - t1 + sigmaL * t3) + # def smooth_l2_surrogate_alt(Ht, L, H=None, sigmaL=sigmaL, lambda_L=1): # HtTL = Ht @ L # t1 = np.sum(HtTL * Ht) # if H is None: # return lambda_L / 2 * t1 - + # t2 = 2 * np.sum(HtTL * (H - Ht) ) # t3 = sigmaL * np.sum((Ht-H)**2) - + # return lambda_L / 2 * (t1 + t2 + t3) + def smooth_dgkl_surrogate(Ht, L, H=None, sigmaL=sigmaL, lambda_L=1): r"""Compute the smooth KL surrogate of the Laplacian regularizer :math:`\lambda_L/2 tr(H \Delta H^\top)` at :math:`H^t` @@ -71,8 +73,8 @@ def smooth_dgkl_surrogate(Ht, L, H=None, sigmaL=sigmaL, lambda_L=1): g(H,H^t ) = \frac{\lambda_L}{2} \left( tr\left( H^t \Delta H^{t\top} \right) + 2 tr \left(H^t \Delta (H - H^t)\top \right) + \sigma_L \sum_{i=1}^n \max_{j} \left( H_{ij} \log \frac{H_{ij}}{H^t_{ij}} - H_{ij} + H^t_{ij} \right) ) - where - + where + * :math:`H^t` is the current estimate of :math:`\dot{H}`, * :math:`H` is the main variable of :math:`g`, * :math:`\Delta` is the Laplacian matrix, @@ -100,22 +102,23 @@ def smooth_dgkl_surrogate(Ht, L, H=None, sigmaL=sigmaL, lambda_L=1): """ HtTL = Ht @ L t1 = np.sum(HtTL * Ht) - + def dgkl(p, q): - return p * np.log(p/q) - p + q - + return p * np.log(p / q) - p + q + if H is None: t2 = t1 t3 = 0 else: - t2 = np.sum(HtTL * H ) + t2 = np.sum(HtTL * H) maxH = np.max(H, axis=1) t3 = np.sum(maxH * np.sum(dgkl(Ht, H), axis=1)) - return lambda_L / 2 * (2*t2 - t1 + sigmaL * t3) + return lambda_L / 2 * (2 * t2 - t1 + sigmaL * t3) + def diff_surrogate(Ht, H, L, sigmaL=sigmaL, lambda_L=1, algo="log_surrogate"): r"""Compute the difference between the surrogate and the true value of the Laplacian regularizer at :math:`H^t`. - + Parameters ---------- Ht : np.ndarray @@ -142,22 +145,21 @@ def diff_surrogate(Ht, H, L, sigmaL=sigmaL, lambda_L=1, algo="log_surrogate"): b_inf = trace_xtLx(L, H.T) * lambda_L / 2 if algo in ["log_surrogate", "bmd"]: b_supp = smooth_dgkl_surrogate(Ht, L=L, H=H, sigmaL=sigmaL, lambda_L=lambda_L) - elif algo=="l2_surrogate": + elif algo == "l2_surrogate": b_supp = smooth_l2_surrogate(Ht, L=L, H=H, sigmaL=sigmaL, lambda_L=lambda_L) - else: + else: raise "Unknown algorithm" return b_supp - b_inf - def quadratic_surrogate(x, xt, f_xt, gradf_xt, sigma): r"""Compute the quadratic surrogate function of :math:`f` at :math:`x^t` This function essentially computes: .. math:: - - g(x,x^t) = f(x^t) + \left< x - x^t , \nabla f (x^t) \right> + \sigma \| x - x^t \|_2^2 + + g(x,x^t) = f(x^t) + \left< x - x^t , \nabla f (x^t) \right> + \sigma \| x - x^t \|_2^2 :param np.array x: variable :param np.array xt: variable @@ -168,5 +170,4 @@ def quadratic_surrogate(x, xt, f_xt, gradf_xt, sigma): :returns: the answer """ - return f_xt + np.sum((x-xt) * gradf_xt) + sigma * np.sum((x-xt)**2) - + return f_xt + np.sum((x - xt) * gradf_xt) + sigma * np.sum((x - xt) ** 2) diff --git a/espm/estimators/updates.py b/espm/estimators/updates.py index c7e0511..e2bec65 100644 --- a/espm/estimators/updates.py +++ b/espm/estimators/updates.py @@ -1,27 +1,35 @@ import numpy as np -from espm.conf import log_shift, dicotomy_tol, sigmaL -from sklearn.decomposition._nmf import _initialize_nmf as initialize_nmf -from espm.estimators.dicotomy import dichotomy_simplex, dichotomy_simplex_acc, dichotomy_simplex_projected_gradient - -def multiplicative_step_w(X, - G, - W, - H, - simplex_W = False, - log_shift=log_shift, - safe=True, - l2=False, - fixed_W = None, - physics_model=None, - use_bregman=False): +from sklearn.decomposition._nmf import _initialize_nmf as initialize_nmf + +from espm.conf import dicotomy_tol, log_shift, sigmaL +from espm.estimators.dicotomy import ( + dichotomy_simplex, + dichotomy_simplex_acc, + dichotomy_simplex_projected_gradient, +) + + +def multiplicative_step_w( + X, + G, + W, + H, + simplex_W=False, + log_shift=log_shift, + safe=True, + l2=False, + fixed_W=None, + physics_model=None, + use_bregman=False, +): """ Multiplicative step in W. """ if safe: # Allow for very small negative values! - assert(np.sum(H<-log_shift/2)==0) - assert(np.sum(W<-log_shift/2)==0) - assert(np.sum(G<-log_shift/2)==0) + assert np.sum(H < -log_shift / 2) == 0 + assert np.sum(W < -log_shift / 2) == 0 + assert np.sum(G < -log_shift / 2) == 0 H = np.maximum(H, log_shift) W = np.maximum(W, log_shift) @@ -44,7 +52,11 @@ def multiplicative_step_w(X, else: sigmaR = np.sum(X) num = sigmaR * W - gradg = - G.T @ (X / GWH) @ H.T + np.sum(G, axis=0, keepdims=True).T @ np.sum(H, axis=1, keepdims=True).T + gradg = ( + -G.T @ (X / GWH) @ H.T + + np.sum(G, axis=0, keepdims=True).T + @ np.sum(H, axis=1, keepdims=True).T + ) denum = gradg * W + sigmaR else: @@ -54,17 +66,26 @@ def multiplicative_step_w(X, if np.any(np.isnan(op1)): GWH = np.maximum(GWH, log_shift) op1 = X / GWH - + mult1 = G.T @ op1 - num = W*(mult1 @ H.T) - denum = np.sum(G, axis=0, keepdims=True).T @ np.sum(H, axis=1, keepdims=True).T + num = W * (mult1 @ H.T) + denum = ( + np.sum(G, axis=0, keepdims=True).T @ np.sum(H, axis=1, keepdims=True).T + ) if simplex_W: if physics_model != None: indices = physics_model.NMF_simplex() - nu = dichotomy_simplex(num[indices,:], denum[indices,:], log_shift=log_shift, tol=dicotomy_tol) - denum[indices,:] = denum[indices,:] + nu - else : - nu = dichotomy_simplex(num, denum, log_shift=log_shift, tol=dicotomy_tol) + nu = dichotomy_simplex( + num[indices, :], + denum[indices, :], + log_shift=log_shift, + tol=dicotomy_tol, + ) + denum[indices, :] = denum[indices, :] + nu + else: + nu = dichotomy_simplex( + num, denum, log_shift=log_shift, tol=dicotomy_tol + ) denum = denum + nu new_W = num / denum @@ -72,46 +93,61 @@ def multiplicative_step_w(X, new_W = np.maximum(new_W, log_shift) # TODO: exclude the fixed values in the update process. It is not straightforward - if fixed_W is not None: - new_W[fixed_W >= 0] = fixed_W[fixed_W >=0] + if fixed_W is not None: + new_W[fixed_W >= 0] = fixed_W[fixed_W >= 0] return new_W - - -def multiplicative_step_h(X, G, W, H, simplex_H =False, mu=0, log_shift=log_shift, epsilon_reg=1, safe=True, dicotomy_tol=dicotomy_tol, lambda_L=0, L=None, l2=False, sigmaL=sigmaL, fixed_H = None, use_bregman=False): +def multiplicative_step_h( + X, + G, + W, + H, + simplex_H=False, + mu=0, + log_shift=log_shift, + epsilon_reg=1, + safe=True, + dicotomy_tol=dicotomy_tol, + lambda_L=0, + L=None, + l2=False, + sigmaL=sigmaL, + fixed_H=None, + use_bregman=False, +): """ Multiplicative step in A. The main terms are calculated first. - With mu_sparse = 0, the steps in A are calculated once. For mu_sparse != 0, the steps - in A are calculated first with particle regularization. Then only the entries allowed + With mu_sparse = 0, the steps in A are calculated once. For mu_sparse != 0, the steps + in A are calculated first with particle regularization. Then only the entries allowed by the mask are calculaed, without particle regularization. Note that mu can be passed as a vector to regularize the different phase of A differently. To calculate the regularized step, we make a linear approximation of the log. """ - if not(lambda_L==0): + if not (lambda_L == 0): if L is None: raise ValueError("Please provide the laplacian") - HL = H@L + HL = H @ L if safe: # Allow for very small negative values! # TODO: update this - assert(np.sum(H<-log_shift/2)==0) - assert(np.sum(W<-log_shift/2)==0) - assert(np.sum(G<-log_shift/2)==0) + assert np.sum(H < -log_shift / 2) == 0 + assert np.sum(W < -log_shift / 2) == 0 + assert np.sum(G < -log_shift / 2) == 0 H = np.maximum(H, log_shift) W = np.maximum(W, log_shift) - GW = G @ W # Also called D - + GW = G @ W # Also called D + if l2: assert lambda_L == 0 if np.isscalar(mu): assert mu == 0 else: - assert (mu==0).all() + assert (mu == 0).all() WGGW = GW.T @ GW WGX = GW.T @ X num = WGX @@ -121,7 +157,7 @@ def multiplicative_step_h(X, G, W, H, simplex_H =False, mu=0, log_shift=log_shif GWH = GW @ H sigmaR = np.sum(X, axis=0, keepdims=True) num = sigmaR / H - gradg = - GW.T @ (X / GWH) + np.sum(GW, axis=0, keepdims=True).T + gradg = -GW.T @ (X / GWH) + np.sum(GW, axis=0, keepdims=True).T denum = gradg + sigmaR / H else: GWH = GW @ H @@ -129,43 +165,54 @@ def multiplicative_step_h(X, G, W, H, simplex_H =False, mu=0, log_shift=log_shif if np.any(np.isnan(num)): GWH = np.maximum(GWH, log_shift) num = GW.T @ (X / GWH) - denum = np.sum(GW, axis=0, keepdims=True).T + denum = np.sum(GW, axis=0, keepdims=True).T - if not(np.isscalar(mu) and mu==0): - if len(np.shape(mu))==1: + if not (np.isscalar(mu) and mu == 0): + if len(np.shape(mu)) == 1: mu = np.expand_dims(mu, axis=1) denum = denum + mu / (H + epsilon_reg) - if not(lambda_L==0): + if not (lambda_L == 0): maxH = np.max(H, axis=1, keepdims=True) num = num + lambda_L * sigmaL * maxH - denum = denum + lambda_L * sigmaL * maxH + lambda_L * HL + denum = denum + lambda_L * sigmaL * maxH + lambda_L * HL num = H * num if simplex_H: nu = dichotomy_simplex(num, denum, log_shift=log_shift, tol=dicotomy_tol) else: nu = 0 if safe: - assert np.sum(denum<0)==0 - assert np.sum(num<0)==0 + assert np.sum(denum < 0) == 0 + assert np.sum(num < 0) == 0 # Add the shift... - new_H = np.maximum(num/(denum+nu), log_shift) + new_H = np.maximum(num / (denum + nu), log_shift) - if fixed_H is not None: + if fixed_H is not None: new_H[fixed_H >= 0] = fixed_H[fixed_H >= 0] return new_H - -def initialize_algorithms(X, G, W, H, n_components, init, random_state, simplex_H, simplex_W, logshift=log_shift, physics_model = None): +def initialize_algorithms( + X, + G, + W, + H, + n_components, + init, + random_state, + simplex_H, + simplex_W, + logshift=log_shift, + physics_model=None, +): # Handle initialization - if G is None : + if G is None: skip_second = True - # G = sparse.diags(np.ones(X.shape[0]).astype(X.dtype)) + # G = sparse.diags(np.ones(X.shape[0]).astype(X.dtype)) G = np.diag(np.ones(X.shape[0]).astype(X.dtype)) - # elif callable(G) : + # elif callable(G) : # assert not(model_params is None), "You need to input model_parameters" # assert not(g_params is None), "You need to input g_parameters" # G = G(model_params,g_params) @@ -176,60 +223,66 @@ def initialize_algorithms(X, G, W, H, n_components, init, random_state, simplex_ if W is None: if H is None: - D, H = initialize_nmf(X, n_components=n_components, init=init, random_state=random_state) + D, H = initialize_nmf( + X, n_components=n_components, init=init, random_state=random_state + ) # D, A = u.rescaled_DA(D,A) if simplex_H: - H = np.nan_to_num(H, nan = 1.0/H.shape[0]) + H = np.nan_to_num(H, nan=1.0 / H.shape[0]) scale = np.sum(H, axis=0, keepdims=True) - H = H/scale - + H = H / scale + # D = np.abs(np.linalg.lstsq(H.T, X.T,rcond=None)[0].T) - D = D*np.mean(scale) + D = D * np.mean(scale) else: - D = np.abs(np.linalg.lstsq(H.T, X.T,rcond=None)[0].T) + D = np.abs(np.linalg.lstsq(H.T, X.T, rcond=None)[0].T) if skip_second: W = D else: - if physics_model != None: - # [np.where(G[:,:-2].sum(axis=1)<(np.max(G[:,:-2].sum(axis=1))*0.001))[0],:] - # Divide in two parts the initial fitting, otherwise the bremsstrahlung (which has a low intensity) tends to be poorly learned - # First fit the caracteristic Xrays, then subtract that contribution to obtain a rough estimate of the bremsstralung parameters + if physics_model != None: + # [np.where(G[:,:-2].sum(axis=1)<(np.max(G[:,:-2].sum(axis=1))*0.001))[0],:] + # Divide in two parts the initial fitting, otherwise the bremsstrahlung (which has a low intensity) tends to be poorly learned + # First fit the caracteristic Xrays, then subtract that contribution to obtain a rough estimate of the bremsstralung parameters W = physics_model.NMF_initialize_W(D) # Wbrem = (np.linalg.lstsq(G[:,-2:],D - G[:,:-2]@Wcarac,rcond = None)[0]).clip(min = 0) if simplex_W: indices = physics_model.NMF_simplex() - W = np.nan_to_num(W, nan = 1.0/W.shape[0]) - scale = np.sum(W[indices,:], axis=0, keepdims=True) - W[indices,:] = W[indices,:]/scale + W = np.nan_to_num(W, nan=1.0 / W.shape[0]) + scale = np.sum(W[indices, :], axis=0, keepdims=True) + W[indices, :] = W[indices, :] / scale # P = np.abs(np.linalg.lstsq(G, D,rcond=None)[0]) - else : - W = np.abs(np.linalg.lstsq(G, D,rcond=None)[0]) + else: + W = np.abs(np.linalg.lstsq(G, D, rcond=None)[0]) if simplex_W: - W = np.nan_to_num(W, nan = 1.0/W.shape[0]) + W = np.nan_to_num(W, nan=1.0 / W.shape[0]) scale = np.sum(W, axis=0, keepdims=True) - W = W/scale + W = W / scale elif H is None: D = G @ W H = np.abs(np.linalg.lstsq(D, X, rcond=None)[0]) if simplex_H: scale = np.sum(H, axis=0, keepdims=True) - H = H/scale + H = H / scale W = np.maximum(W, log_shift) H = np.maximum(H, log_shift) return G, W, H + def update_q(D, H, log_shift=log_shift): """Perform a Q step.""" Htmp = np.expand_dims(H.T, axis=0) Dtmp = np.expand_dims(D, axis=1) - Ntmp = np.expand_dims(D @ H, axis=2) - return Htmp * (Dtmp / (Ntmp+log_shift)) - -def multiplicative_step_wq(X, G, W, H, simplex_W = True, log_shift=log_shift, safe=True, physics_model = None): + Ntmp = np.expand_dims(D @ H, axis=2) + return Htmp * (Dtmp / (Ntmp + log_shift)) + + +def multiplicative_step_wq( + X, G, W, H, simplex_W=True, log_shift=log_shift, safe=True, physics_model=None +): """ Multiplicative step in W using the WQ technique. @@ -238,56 +291,77 @@ def multiplicative_step_wq(X, G, W, H, simplex_W = True, log_shift=log_shift, sa if safe: # Allow for very small negative values! - assert np.sum(H<-log_shift/2)==0 - assert np.sum(W<-log_shift/2)==0 - assert np.sum(G<-log_shift/2)==0 + assert np.sum(H < -log_shift / 2) == 0 + assert np.sum(W < -log_shift / 2) == 0 + assert np.sum(G < -log_shift / 2) == 0 GW = G @ W Q = update_q(GW, H, log_shift=log_shift) XQ = np.sum(np.expand_dims(X, axis=2) * Q, axis=1) - term1 = G.T @ (XQ / (GW + log_shift)) + term1 = G.T @ (XQ / (GW + log_shift)) - term2 = np.sum(G, axis=0, keepdims=True).T @ np.sum(H, axis=1, keepdims=True).T - if simplex_W : + term2 = np.sum(G, axis=0, keepdims=True).T @ np.sum(H, axis=1, keepdims=True).T + if simplex_W: if physics_model != None: indices = physics_model.NMF_simplex() - nu = dichotomy_simplex(term1[indices,:], term2[indices,:], log_shift=log_shift, tol=dicotomy_tol) - term2[indices,:] = term2[indices,:] + nu - else : + nu = dichotomy_simplex( + term1[indices, :], + term2[indices, :], + log_shift=log_shift, + tol=dicotomy_tol, + ) + term2[indices, :] = term2[indices, :] + nu + else: nu = dichotomy_simplex(term1, term2, log_shift=log_shift, tol=dicotomy_tol) term2 = term2 + nu return W / term2 * term1 -def multiplicative_step_hq(X, G, W, H, simplex_H=True, log_shift=log_shift, safe=True, dicotomy_tol=dicotomy_tol, lambda_L=0, L=None, sigmaL=sigmaL, fixed_H = None): + +def multiplicative_step_hq( + X, + G, + W, + H, + simplex_H=True, + log_shift=log_shift, + safe=True, + dicotomy_tol=dicotomy_tol, + lambda_L=0, + L=None, + sigmaL=sigmaL, + fixed_H=None, +): """ Multiplicative step in H. """ - if not lambda_L==0: + if not lambda_L == 0: if L is None: raise ValueError("Please provide the laplacian") if safe: # Allow for very small negative values! - assert np.sum(H<-log_shift/2)==0 - assert np.sum(W<-log_shift/2)==0 - assert np.sum(G<-log_shift/2)==0 + assert np.sum(H < -log_shift / 2) == 0 + assert np.sum(W < -log_shift / 2) == 0 + assert np.sum(G < -log_shift / 2) == 0 - GW = G @ W # Also called D + GW = G @ W # Also called D GWH = GW @ H - minus_c = H * (GW.T @ (X / (GWH+log_shift))) + minus_c = H * (GW.T @ (X / (GWH + log_shift))) - b = np.sum(GW, axis=0, keepdims=True).T - if not lambda_L==0 : - b = b + lambda_L * H @ L - lambda_L * sigmaL * H + b = np.sum(GW, axis=0, keepdims=True).T + if not lambda_L == 0: + b = b + lambda_L * H @ L - lambda_L * sigmaL * H a = lambda_L * sigmaL if simplex_H: - nu = dichotomy_simplex_acc(a, b, minus_c, log_shift=log_shift, tol=dicotomy_tol) + nu = dichotomy_simplex_acc( + a, b, minus_c, log_shift=log_shift, tol=dicotomy_tol + ) b = b + nu - new_H = (-b + np.sqrt(b**2 + 4* a *minus_c)) / (2*a) - else: # We recover the classic case: multiplicative_step_a + new_H = (-b + np.sqrt(b**2 + 4 * a * minus_c)) / (2 * a) + else: # We recover the classic case: multiplicative_step_a if simplex_H: nu = dichotomy_simplex(minus_c, b, log_shift=log_shift, tol=dicotomy_tol) b = b + nu @@ -296,52 +370,77 @@ def multiplicative_step_hq(X, G, W, H, simplex_H=True, log_shift=log_shift, safe # Add the shift... new_H = np.maximum(new_H, log_shift) - if fixed_H is not None: + if fixed_H is not None: new_H[fixed_H >= 0] = fixed_H[fixed_H >= 0] return new_H + def gradW(X, G, W, H, log_shift=log_shift, safe=False, l2=False): if safe: H = np.maximum(H, log_shift) W = np.maximum(W, log_shift) if l2: - grad = 2*G.T @ ( (G @ W) @ H - X ) @ H.T + grad = 2 * G.T @ ((G @ W) @ H - X) @ H.T else: D = G @ W DH = D @ H - grad = G.T @ (- (X / DH) @ H.T + np.sum(H, axis=1, keepdims=True).T) + grad = G.T @ (-(X / DH) @ H.T + np.sum(H, axis=1, keepdims=True).T) return grad -def gradH(X, G, W, H, mu=0, lambda_L=0, L=None, epsilon_reg=1, log_shift=log_shift, safe=False, l2=False): - if not(lambda_L==0): + +def gradH( + X, + G, + W, + H, + mu=0, + lambda_L=0, + L=None, + epsilon_reg=1, + log_shift=log_shift, + safe=False, + l2=False, +): + if not (lambda_L == 0): if L is None: raise ValueError("Please provide the laplacian") - HL = H@L - + HL = H @ L + if safe: H = np.maximum(H, log_shift) W = np.maximum(W, log_shift) - if l2: D = G @ W grad = D.T @ (D @ H - X) else: D = G @ W DH = D @ H - grad = - D.T @ (X / DH) + np.sum(D, axis=0, keepdims=True).T + grad = -D.T @ (X / DH) + np.sum(D, axis=0, keepdims=True).T - if not(np.isscalar(mu) and mu==0): - if len(np.shape(mu))==1: + if not (np.isscalar(mu) and mu == 0): + if len(np.shape(mu)) == 1: mu = np.expand_dims(mu, axis=1) grad += mu / (H + epsilon_reg) - if not(lambda_L==0): + if not (lambda_L == 0): grad += (lambda_L * L @ H.T).T return grad -# -def proj_grad_step_w(X, G, W, H, gamma, simplex_W = True, log_shift=log_shift, safe=True, l2=False, fixed_W = None): + + +def proj_grad_step_w( + X, + G, + W, + H, + gamma, + simplex_W=True, + log_shift=log_shift, + safe=True, + l2=False, + fixed_W=None, +): """Projected gradient step for the variable W.""" if safe: @@ -351,18 +450,37 @@ def proj_grad_step_w(X, G, W, H, gamma, simplex_W = True, log_shift=log_shift, s grad = gradW(X, G, W, H, log_shift=log_shift, safe=safe, l2=l2) # gradient step - new_W = W - 1/gamma * grad + new_W = W - 1 / gamma * grad # projection new_W = np.maximum(new_W, log_shift) - if fixed_W is not None: - new_W[fixed_W >= 0] = fixed_W[fixed_W >=0] + if fixed_W is not None: + new_W[fixed_W >= 0] = fixed_W[fixed_W >= 0] - if simplex_W : - raise NotImplementedError("Simplex constraint not implemented for W using the projected gradient method") + if simplex_W: + raise NotImplementedError( + "Simplex constraint not implemented for W using the projected gradient method" + ) return new_W -def proj_grad_step_h(X, G, W, H, gamma, simplex_H=True, mu=0, log_shift=log_shift, epsilon_reg=1, safe=True, dicotomy_tol=dicotomy_tol, lambda_L=0, L=None, l2=False, fixed_H = None): + +def proj_grad_step_h( + X, + G, + W, + H, + gamma, + simplex_H=True, + mu=0, + log_shift=log_shift, + epsilon_reg=1, + safe=True, + dicotomy_tol=dicotomy_tol, + lambda_L=0, + L=None, + l2=False, + fixed_H=None, +): """Projected gradient step for the variable H.""" if safe: @@ -370,22 +488,37 @@ def proj_grad_step_h(X, G, W, H, gamma, simplex_H=True, mu=0, log_shift=log_shif W = np.maximum(W, log_shift) # gradient step - grad = gradH(X, G, W, H, log_shift=log_shift, safe=safe, mu=mu, epsilon_reg=epsilon_reg, lambda_L=lambda_L, L=L, l2=l2) - new_H = H - 1/gamma * grad + grad = gradH( + X, + G, + W, + H, + log_shift=log_shift, + safe=safe, + mu=mu, + epsilon_reg=epsilon_reg, + lambda_L=lambda_L, + L=L, + l2=l2, + ) + new_H = H - 1 / gamma * grad # Dichotomy if simplex_H: - nu = dichotomy_simplex_projected_gradient(new_H, log_shift=log_shift, tol=dicotomy_tol) + nu = dichotomy_simplex_projected_gradient( + new_H, log_shift=log_shift, tol=dicotomy_tol + ) else: nu = 0 # projection new_H = np.maximum(new_H + nu, log_shift) - if fixed_H is not None: - new_H[fixed_H >= 0] = fixed_H[fixed_H >=0] + if fixed_H is not None: + new_H[fixed_H >= 0] = fixed_H[fixed_H >= 0] return new_H + def estimate_Lipschitz_bound_w(log_shift, X, G, k): if G is None: G = np.eye(X.shape[0]) @@ -393,9 +526,10 @@ def estimate_Lipschitz_bound_w(log_shift, X, G, k): Hlim = np.ones([k, X.shape[1]]) * log_shift D = G @ Wlim DH = D @ Hlim - gamma = np.max((np.sum(Hlim,axis=0, keepdims=True) * X/(DH**2))@ Hlim.T) + gamma = np.max((np.sum(Hlim, axis=0, keepdims=True) * X / (DH**2)) @ Hlim.T) return gamma + def estimate_Lipschitz_bound_h(log_shift, X, G, k, lambda_L=0, mu=0, epsilon_reg=1): if G is None: G = np.eye(X.shape[0]) @@ -403,7 +537,11 @@ def estimate_Lipschitz_bound_h(log_shift, X, G, k, lambda_L=0, mu=0, epsilon_reg Hlim = np.ones([k, X.shape[1]]) * log_shift D = G @ Wlim DH = D @ Hlim - - gamma = np.max(D.T @ (np.sum(D,axis=1, keepdims=True) * X/(DH**2)) ) + 2*lambda_L+mu*epsilon_reg - return gamma \ No newline at end of file + gamma = ( + np.max(D.T @ (np.sum(D, axis=1, keepdims=True) * X / (DH**2))) + + 2 * lambda_L + + mu * epsilon_reg + ) + + return gamma diff --git a/espm/measures.py b/espm/measures.py index f4f85df..9e00892 100644 --- a/espm/measures.py +++ b/espm/measures.py @@ -1,30 +1,33 @@ -r""" The :mod:`espm.measures` module implements different measures for the matrix factorisation problem. -In particular it contains the different losses and regularizers used in :mod:`espm.estimator` module. +r"""The :mod:`espm.measures` module implements different measures for the matrix factorisation problem. +In particular it contains the different losses and regularizers used in :mod:`espm.estimator` module. It also contains different metrics to evaluate the results. """ -import numpy as np -from espm.conf import log_shift import warnings as w from itertools import permutations + +import numpy as np from sklearn.metrics import r2_score +from espm.conf import log_shift + + def spectral_angle(v1, v2): r"""Spectral angle Calculate the angle between two spectra of the same dimension. - + :param np.array 1D v1: first spectrum :param np.array 1D v2: second spectrum :returns: the answer - + :rtype: float Examples -------- - + >>> import numpy as np >>> from espm.measures import spectral_angle >>> v1 = np.array([0, 1, 0]) @@ -34,7 +37,7 @@ def spectral_angle(v1, v2): """ - if len(v1.shape)==1: + if len(v1.shape) == 1: if v1.shape != v2.shape: raise ValueError("v1 and v2 should have the same shape.") v1_u = v1 / np.linalg.norm(v1) @@ -60,7 +63,7 @@ def mse(map1, map2): Examples -------- - + >>> import numpy as np >>> from espm.measures import mse >>> map1 = np.array([[0, 1][0, 1]]) @@ -69,7 +72,7 @@ def mse(map1, map2): 0.5 """ - return np.mean((map1-map2)**2) + return np.mean((map1 - map2) ** 2) def mae(map1, map2): @@ -79,12 +82,12 @@ def mae(map1, map2): :param np.array 2D map1: first array :param np.array 2D map2: second array - + :returns: the answer Examples -------- - + >>> import numpy as np >>> from espm.measures import mae >>> map1 = np.array([[0, 1][0, 1]]) @@ -93,36 +96,37 @@ def mae(map1, map2): 0.5 """ - return np.mean(np.abs(map1-map2)) + return np.mean(np.abs(map1 - map2)) def r2(map_true, map_pred): r""":math:`R^2` - Coefficient of determination Calculates the coefficient of determination between two 2D arrays of the same dimension. - This is also called regression score function. + This is also called regression score function. See `wikipedia `_. This function is a wrapper for the function :mod:`sklearn.metrics.r2_score` of Scikit Learn. - + :param np.array 2D map1: first array :param np.array 2D map2: second array - + :returns: the answer :rtype: float """ + def reshape_2d(x): return x.reshape(x.shape[0], -1) - return r2_score(reshape_2d(map_true), reshape_2d(map_pred)) + return r2_score(reshape_2d(map_true), reshape_2d(map_pred)) # This function will find the best matching endmember for each true spectrum. # This is useful since the A and P matrice are initialized at random. # This function works but can probably greatly improved -def find_min_angle(true_vectors, algo_vectors, get_ind = False, unique=False): +def find_min_angle(true_vectors, algo_vectors, get_ind=False, unique=False): # This function calculates all the possible angles between endmembers and true spectra # For each true spectrum a best matching endmember is found # The function returns the angles of the corresponding pairs @@ -132,49 +136,50 @@ def find_min_angle(true_vectors, algo_vectors, get_ind = False, unique=False): :param np.array 2D algo_vectors: NMF spectra with shape (number of phases, number of energy channels) :param boolean get_ind: If True, the function will also return the indices of the NMF spectra corresponding to the ground truth :param boolean unique: If False it will find the global minimum but several spectra can be associated to the same ground truth - + :returns: list of angles, (optionally) tuple of indices :rtype: (list[float],list[int]) - ..warning:: + ..warning:: The output being either a list or a tuple of list isn't a great idea. It has to change. """ - angle_matr = spectral_angle(true_vectors,algo_vectors) - if unique : + angle_matr = spectral_angle(true_vectors, algo_vectors) + if unique: ordered_angles = unique_min(angle_matr) - else : + else: ordered_angles = global_min(angle_matr) - #unique minimum angles are ordered - if get_ind : + # unique minimum angles are ordered + if get_ind: # if unique : # print("Impossible to get indices when searching with unique minima.") - # else : + # else : return ordered_angles - else : + else: return ordered_angles[0] -def global_min (matr) : + +def global_min(matr): res = [] ind_res = [] - for i in matr : + for i in matr: ind_min = np.argmin(i) min = np.min(i) res.append(min) ind_res.append(ind_min) - if any(ind_res.count(x) > 1 for x in ind_res) : + if any(ind_res.count(x) > 1 for x in ind_res): w.warn("Several results share the same truth") return res, ind_res - -def unique_min (matrix) : - ''' - From a square matrix of float values, finds the combination of elements with +def unique_min(matrix): + """ + + From a square matrix of float values, finds the combination of elements with different lines which mimises the sum of elements. - + It is a brute force algorithm, it is not recommended to input a matrix bigger than 20 x 20 :param np.array 2D matrix: square matrix - + :returns: list of unique min values and corresponding indices in the same order :rtype: (list, list[int]) @@ -187,28 +192,28 @@ def unique_min (matrix) : >>> matrix = np.array([[1.2, 1.3, 3.5], [4.9, 2.2, 6.5], [9.0, 4.1, 1.8]]) - >>> unique_min(v1, v2) + >>> unique_min(v1, v2) ([1.2, 2.2, 1.8], (0, 1, 2)) - ''' + """ shape = matrix.shape[0] - perms = list(permutations(range(shape),shape)) + perms = list(permutations(range(shape), shape)) list_sum_angles = [] - for perm in perms : + for perm in perms: sum_angles = 0 - for i in range(shape) : - sum_angles += matrix[perm[i],i] + for i in range(shape): + sum_angles += matrix[perm[i], i] list_sum_angles.append(sum_angles) - + min_ind = list_sum_angles.index(min(list_sum_angles)) mins = [] - for i in range(shape) : - mins.append(matrix[perms[min_ind][i],i]) + for i in range(shape): + mins.append(matrix[perms[min_ind][i], i]) return mins, perms[min_ind] - -# def unique_min (matr) : + +# def unique_min (matr) : # mins= [] # ind_mins = [] # for vec in matr : @@ -219,8 +224,9 @@ def unique_min (matrix) : # return mins, ind_mins -def find_min_config(true_maps,true_spectra, algo_maps, algo_spectra,angles = True) : - r''' + +def find_min_config(true_maps, true_spectra, algo_maps, algo_spectra, angles=True): + r""" Determines the best match between the true and the NMF spectra and maps by finding the configuration that minimizes either the sum of angles or the sum of MSE. The function returns a warning (boolean) if the MSE and the angles disagree on the best configuration. @@ -231,31 +237,34 @@ def find_min_config(true_maps,true_spectra, algo_maps, algo_spectra,angles = Tru :param boolean angles: If True, the function will minimize the sum of angles between true and NMF spectra. If False, it will minimize the sum of MSE between true and NMF maps. :returns: list of angles, list of MSE, configuration of the best match, warning - ''' - min_MSE_config = find_min_MSE(true_maps,algo_maps,get_ind=True,unique=True)[1] - min_angle_config = find_min_angle(true_spectra,algo_spectra,get_ind=True,unique=True)[1] + """ + min_MSE_config = find_min_MSE(true_maps, algo_maps, get_ind=True, unique=True)[1] + min_angle_config = find_min_angle( + true_spectra, algo_spectra, get_ind=True, unique=True + )[1] warning = False - if min_MSE_config != min_angle_config : + if min_MSE_config != min_angle_config: print("WARNING : angles and mse disagree there's probably an issue") warning = True - if angles : - corresponding_angles = find_min_angle(true_spectra,algo_spectra,unique=True) + if angles: + corresponding_angles = find_min_angle(true_spectra, algo_spectra, unique=True) min_config = min_angle_config - corresponding_mse = ordered_mse(true_maps,algo_maps,min_angle_config) + corresponding_mse = ordered_mse(true_maps, algo_maps, min_angle_config) - else : - corresponding_mse = find_min_MSE(true_maps,algo_maps,unique=True) + else: + corresponding_mse = find_min_MSE(true_maps, algo_maps, unique=True) min_config = min_MSE_config - corresponding_angles = ordered_angles(true_spectra,algo_spectra,min_MSE_config) + corresponding_angles = ordered_angles( + true_spectra, algo_spectra, min_MSE_config + ) return corresponding_angles, corresponding_mse, min_config, warning - # This function works but can probably greatly improved -def find_min_MSE(true_maps, algo_maps, get_ind = False, unique=False): +def find_min_MSE(true_maps, algo_maps, get_ind=False, unique=False): # This function calculates all the possible MSE between abundances and true maps # For each true map a best matching abundance is found # The function returns the MSE of the corresponding pairs @@ -265,75 +274,78 @@ def find_min_MSE(true_maps, algo_maps, get_ind = False, unique=False): :param np.array 2D algo_maps: NMF maps with shape (number of phases, number of pixels) :param boolean get_ind: If True, the function will also return the indices of the NMF maps corresponding to the ground truth :param boolean unique: If False it will find the global minimum but several maps can be associated to the same ground truth - + :returns: list of mse, (optionally) tuple of indices :rtype: (list[float],list[int]) - ..warning:: + ..warning:: The output being either a list or a tuple of list isn't a great idea. It has to change. """ mse_matr = squared_distance(true_maps, algo_maps) - if unique : + if unique: ordered_maps = unique_min(mse_matr) - else : + else: ordered_maps = global_min(mse_matr) - if get_ind : + if get_ind: # if unique : # print("Impossible to get indices when searching with unique minima.") - # else : + # else : return ordered_maps - else : + else: return ordered_maps[0] -def ordered_mse (true_maps, algo_maps, input_inds) : - ''' + +def ordered_mse(true_maps, algo_maps, input_inds): + """ input : p x Npx matrix of floats, p x Npx matrix of floats, list of integers output : list of floats %-------------------------% Takes true maps of p phases and Npx pixels, reconstructed maps of the same size and indices of the correspondance between true phases and reconstructed phases returns the mean squared errors of each phase in truth order. - ''' + """ ordered_maps = [] - for i,j in enumerate(input_inds) : + for i, j in enumerate(input_inds): ordered_maps.append(float(mse(true_maps[j], algo_maps[i]))) return ordered_maps -def ordered_mae (true_maps, algo_maps, input_inds) : - ''' + +def ordered_mae(true_maps, algo_maps, input_inds): + """ input : p x Npx matrix of floats, p x Npx matrix of floats, list of integers output : list of floats %-------------------------% Takes true maps of p phases and Npx pixels, reconstructed maps of the same size and indices of the correspondance between true phases and reconstructed phases returns the mean average errors of each phase in truth order. - ''' + """ ordered_maps = [] - for i,j in enumerate(input_inds) : + for i, j in enumerate(input_inds): ordered_maps.append(float(mae(true_maps[j], algo_maps[i]))) return ordered_maps -def ordered_r2(true_maps, algo_maps, input_inds) : - ''' + +def ordered_r2(true_maps, algo_maps, input_inds): + """ input : p x Npx matrix of floats, p x Npx matrix of floats, list of integers output : list of floats %-------------------------% Takes true maps of p phases and Npx pixels, reconstructed maps of the same size and indices of the correspondance between true phases and reconstructed phases returns the coefficient of determination of each phase in truth order. - ''' + """ ordered_maps = [] - for i,j in enumerate(input_inds) : + for i, j in enumerate(input_inds): ordered_maps.append(float(r2(true_maps[j], algo_maps[i]))) return ordered_maps -def ordered_angles (true_spectra, algo_spectra, input_inds) : - ''' +def ordered_angles(true_spectra, algo_spectra, input_inds): + """ See ordered mse - ''' + """ ordered_angles = [] - for i,j in enumerate(input_inds) : - ordered_angles.append(spectral_angle(true_spectra[j],algo_spectra[i])) + for i, j in enumerate(input_inds): + ordered_angles.append(spectral_angle(true_spectra[j], algo_spectra[i])) return ordered_angles @@ -347,9 +359,10 @@ def residuals(data, model): ) return X_sum - model_sum + def Frobenius_loss(X, W, H, average=False): r"""Frobenius norm of the difference between X and WH. - + Compute the Froebenius norm (elementwise L2 norm of a matrix) given :math:`X,W,H`: .. math:: @@ -363,10 +376,10 @@ def Frobenius_loss(X, W, H, average=False): divide the result by n*m (default False) :returns: the answer - + Examples -------- - + >>> import numpy as np >>> from espm.measures import Frobenius_loss >>> X = np.array([[1, 1, -1], [2, 4, 5]]) @@ -380,15 +393,16 @@ def Frobenius_loss(X, W, H, average=False): DH = W @ H if average: - return np.mean((DH - X)**2) + return np.mean((DH - X) ** 2) else: - return np.sum((DH - X)**2) + return np.sum((DH - X) ** 2) + def KLdiv(X, D, H, log_shift=log_shift, average=False): r"""Generalized KL (Kullback–Leibler) divergence Compute the generalized KL divergence given :math:`X,W,H`: - + .. math:: D_KL(X || WH) = \sum_{ji} X_{ij} \log (X / D A)_{ij} + (D A - X)_{ij} @@ -397,7 +411,7 @@ def KLdiv(X, D, H, log_shift=log_shift, average=False): :param np.array 2D X: n x m matrix :param np.array 2D W: n x k matrix :param np.array 2D H: k x m matrix - :param float log_shift: small constant to ensure the KL divergence does + :param float log_shift: small constant to ensure the KL divergence does not explode (default value set in module :mod:`esppy.conf`) :param boolean average: replace the sum with a mean, i.e., divide the result by n*m (default False) @@ -405,10 +419,10 @@ def KLdiv(X, D, H, log_shift=log_shift, average=False): :returns: the answer :rtype: float - + Examples -------- - + >>> import numpy as np >>> from espm.measures import KLdiv >>> X = np.array([[1, 1, 1], [2, 4, 5]]) @@ -419,11 +433,12 @@ def KLdiv(X, D, H, log_shift=log_shift, average=False): """ D = np.maximum(D, log_shift) - H = np.maximum(H, log_shift) + H = np.maximum(H, log_shift) X = np.maximum(X, log_shift) DH = D @ H return KL(X, DH, log_shift, average) + def KL(X, Y, log_shift=log_shift, average=False): r"""Generalized KL (Kullback–Leibler) divergence for two matrices @@ -434,7 +449,7 @@ def KL(X, Y, log_shift=log_shift, average=False): :param np.array 2D X: n x m matrix :param np.array 2D W: n x k matrix :param np.array 2D H: k x m matrix - :param float log_shift: small constant to ensure the KL divergence does + :param float log_shift: small constant to ensure the KL divergence does not explode (default value set in module :mod:`esppy.conf`) :param boolean average: replace the sum with a mean, i.e., divide the result by n*m (default False) @@ -447,14 +462,15 @@ def KL(X, Y, log_shift=log_shift, average=False): Y = np.maximum(Y, log_shift) if average: x_lin = np.mean(Y) - np.mean(X) - x_log = np.mean(X*np.log(X)) - np.mean(X*np.log(Y)) + x_log = np.mean(X * np.log(X)) - np.mean(X * np.log(Y)) else: x_lin = np.sum(Y) - np.sum(X) - x_log = np.sum(X*np.log(X)) - np.sum(X*np.log(Y)) + x_log = np.sum(X * np.log(X)) - np.sum(X * np.log(Y)) return x_lin + x_log + def KLdiv_loss(X, W, H, log_shift=log_shift, average=False): - r""" Generalized Generalized KL (Kullback–Leibler) divergence loss + r"""Generalized Generalized KL (Kullback–Leibler) divergence loss Compute the loss based on the generalized KL divergence given :math: `X,W,H`: @@ -467,7 +483,7 @@ def KLdiv_loss(X, W, H, log_shift=log_shift, average=False): :param np.array 2D X: n x m matrix :param np.array 2D Y: n x m matrix - :param float log_shift: small constant to ensure the KL divergence does + :param float log_shift: small constant to ensure the KL divergence does not explode (default value set in module :mod:`esppy.conf`) :param boolean average: replace the sum with a mean, i.e., divide the result by n*m (default False) @@ -475,7 +491,7 @@ def KLdiv_loss(X, W, H, log_shift=log_shift, average=False): :returns: the answer :rtype: float - + Examples -------- @@ -497,14 +513,15 @@ def KLdiv_loss(X, W, H, log_shift=log_shift, average=False): Y = W @ H if average: x_lin = np.mean(Y) - x_log = np.mean(X*np.log(Y)) + x_log = np.mean(X * np.log(Y)) else: x_lin = np.sum(Y) - x_log = np.sum(X*np.log(Y)) + x_log = np.sum(X * np.log(Y)) return x_lin - x_log + def KL_loss_surrogate(X, W, H, Ht, log_shift=log_shift, average=False): - r""" Surrogate loss for the KL divergence.""" + r"""Surrogate loss for the KL divergence.""" W = np.maximum(W, log_shift) H = np.maximum(H, log_shift) @@ -513,21 +530,22 @@ def KL_loss_surrogate(X, W, H, Ht, log_shift=log_shift, average=False): WHT = np.expand_dims(W, axis=2) * np.expand_dims(Ht, axis=0) - U = WHT/(np.sum(WHT, axis=1, keepdims=True)) - + U = WHT / (np.sum(WHT, axis=1, keepdims=True)) + DH = np.expand_dims(W, axis=2) * np.expand_dims(H, axis=0) if average: - return np.mean(X * np.sum(U * np.log(U/ DH), axis=1) + np.sum(WHT, axis=1)) + return np.mean(X * np.sum(U * np.log(U / DH), axis=1) + np.sum(WHT, axis=1)) else: - return np.sum(X * np.sum(U * np.log(U/ DH), axis=1)) + np.sum(WHT) + return np.sum(X * np.sum(U * np.log(U / DH), axis=1)) + np.sum(WHT) + def log_reg(H, mu, epsilon=1, average=False): - r""" Log regularisation + r"""Log regularisation + + Compute the regularization loss: + + .. math:: - Compute the regularization loss: - - .. math:: - R(\mu, H, \epsilon) = \sum_{ij} \mu_i \log \left( H_{ij} + \epsilon \right). :param np.array 2D H: n x m matrix @@ -540,25 +558,27 @@ def log_reg(H, mu, epsilon=1, average=False): :rtype: float """ - if not(np.isscalar(mu)): + if not (np.isscalar(mu)): mu = np.expand_dims(mu, axis=1) if average: - return np.mean(mu* np.log(H+epsilon)) + return np.mean(mu * np.log(H + epsilon)) else: - return np.sum(mu* np.log(H+epsilon)) + return np.sum(mu * np.log(H + epsilon)) + def log_surrogate(H, Ht, mu, epsilon, average=False): r"""Surrogate loss for the log function.""" - if not(np.isscalar(mu)): + if not (np.isscalar(mu)): mu = np.expand_dims(mu, axis=1) - tmp = np.log(Ht+epsilon) + (H-Ht) / (epsilon + Ht) + tmp = np.log(Ht + epsilon) + (H - Ht) / (epsilon + Ht) if average: - return np.mean(mu* tmp) + return np.mean(mu * tmp) else: - return np.sum(mu* tmp) + return np.sum(mu * tmp) + def trace_xtLx(L, x, average=False): - r""" Trace of :math:`X^T L X` + r"""Trace of :math:`X^T L X` Compute the following expression :math:`\text{Tr} (X^T L X)`. @@ -576,8 +596,9 @@ def trace_xtLx(L, x, average=False): else: return np.sum(x * (L @ x)) + def squared_distance(x, y=None): - r""" Squared distance between two between all colon vectors matrices. + r"""Squared distance between two between all colon vectors matrices. Calculate the squared L2 distance between all pairs of vectors of two matrices. If only one matrix is given, the function uses each pair of vector of this matrix. @@ -588,10 +609,10 @@ def squared_distance(x, y=None): :returns: the answer :rtype: np.array (m x m) - + Examples -------- - + >>> import numpy as np >>> from espm.measures import square_distance >>> x = np.arange(3) @@ -605,24 +626,22 @@ def squared_distance(x, y=None): try: x.shape[1] except IndexError: - x = x.reshape(1, x.shape[0]) + x = x.reshape(1, x.shape[0]) if y is None: - y = x + y = x else: try: - y.shape[1] except IndexError: - y = y.reshape(1, y.shape[0]) + y = y.reshape(1, y.shape[0]) rx, cx = x.shape - ry, cy = y.shape + ry, cy = y.shape # Size verification if cx != cy: - raise ValueError("The sizes of x and y do not fit") + raise ValueError("The sizes of x and y do not fit") xx = (x * x).sum(axis=1) yy = (y * y).sum(axis=1) - xy = np.dot(x, y.T) - d = abs(np.kron(np.ones((ry, 1)), xx).T +np.kron(np.ones((rx, 1)), yy) - 2 * xy) - - return d / cx + xy = np.dot(x, y.T) + d = abs(np.kron(np.ones((ry, 1)), xx).T + np.kron(np.ones((rx, 1)), yy) - 2 * xy) + return d / cx diff --git a/espm/models/EDXS_function.py b/espm/models/EDXS_function.py index b50fdc6..4017727 100644 --- a/espm/models/EDXS_function.py +++ b/espm/models/EDXS_function.py @@ -7,16 +7,18 @@ """ +from collections import Counter + import numpy as np from scipy.special import erfc -from espm.models.absorption_edxs import det_efficiency_from_curve,det_efficiency,absorption_correction -from espm.models import edxs as e -from collections import Counter -from espm.conf import SYMBOLS_PERIODIC_TABLE -import json -from espm.utils import number_to_symbol_list - +from espm.models.absorption_edxs import ( + absorption_correction, + det_efficiency, + det_efficiency_from_curve, +) + + def gaussian(x, mu, sigma): r""" Calculate the gaussian function according to the following formula: @@ -33,7 +35,7 @@ def gaussian(x, mu, sigma): Mean of the gaussian. sigma : float Standard deviation of the gaussian. - + Returns ------- gaussian : np.array 1D @@ -45,71 +47,74 @@ def gaussian(x, mu, sigma): * np.exp(-np.power(x - mu, 2) / (2 * np.power(sigma, 2))) ) -def read_lines_db (elt,db_dict) : + +def read_lines_db(elt, db_dict): r""" Read the energy and cross section of each line of a chemical element for explicit databases. - + Parameters ---------- - elt : + elt : :string: Atomic number. - db_dict : + db_dict : :dict: Dictionnary extracted from the json database containing the emission lines and their energies. - + Returns ------- - energies : + energies : :list float: List of energies associated to each emission line of the given element - cross-sections : + cross-sections : :list float: List of emission cross-sections of each line of the given element """ energies = [] cs = [] - for line in db_dict[str(elt)] : + for line in db_dict[str(elt)]: energies.append(db_dict[str(elt)][line]["energy"]) cs.append(db_dict[str(elt)][line]["cs"]) return energies, cs -def read_compact_db (elt,db_dict) : + +def read_compact_db(elt, db_dict): r""" Read the energy and cross section of each line of a chemical element for compact databases. - + Parameters ---------- - elt : + elt : :string: Atomic number. - db_dict : + db_dict : :dict: Dictionnary extracted from the json database containing the emission lines and their energies. - + Returns ------- - energies : + energies : :list float: List of energies associated to each emission line of the given element - cross-sections : + cross-sections : :list float: List of emission cross-sections of each line of the given element """ energies = db_dict[str(elt)]["energies"] - cs = db_dict[str(elt)]["cs"] + cs = db_dict[str(elt)]["cs"] return energies, cs + def chapman_bremsstrahlung(x, b0, b1, b2): r""" - Calculate the bremsstrahlung as parametrized by chapman et al. - + Calculate the bremsstrahlung as parametrized by chapman et al. + Parameters ---------- - x : + x : :np.array 1D: Energy scale. - b0 : + b0 : :float: First parameter, corresponding to the inverse of the energy. - b1 : + b1 : :float: Second parameter, corresponding to the energy. - b2 : + b2 : :float: Third parameter, corresponding to the square of the energy. - + Returns ------- - bremsstrahlung : + bremsstrahlung : :np.array 1D: Bremsstrahlung model Notes @@ -117,8 +122,9 @@ def chapman_bremsstrahlung(x, b0, b1, b2): For details see :cite:p:`chapman1984understanding` """ return b0 / x + b1 + b2 * x - -def lifshin_bremsstrahlung(x, b0, b1, E0 = 200): + + +def lifshin_bremsstrahlung(x, b0, b1, E0=200): r""" Calculate the custom parametrized bremsstrahlung inspired by the model of L.Lifshin. @@ -130,46 +136,53 @@ def lifshin_bremsstrahlung(x, b0, b1, E0 = 200): f(\varepsilon) = b_0 \frac{\varepsilon_0 - \varepsilon}{\varepsilon_0 \varepsilon} \left(1 - \frac{\varepsilon_0 - \varepsilon}{\varepsilon_0}\right) + b_1 \frac{\left( \varepsilon_0 - \varepsilon \right) ^2}{\varepsilon_0^2 \varepsilon} - where :math:`\varepsilon_0` is the energy of the incident beam, :math:`\varepsilon` is the energy of the emitted photon and :math:`b_0` and :math:`b_1` are the parameters of the model. - + where :math:`\varepsilon_0` is the energy of the incident beam, :math:`\varepsilon` is the energy of the emitted photon and :math:`b_0` and :math:`b_1` are the parameters of the model. + Parameters ---------- - x : + x : :np.array 1D: Energy scale. - b0 : + b0 : :float: First parameter. - b1 : + b1 : :float: Second parameter. - E0 : + E0 : :float: Energy of the incident beam in keV. - + Returns ------- - bremsstrahlung : + bremsstrahlung : :np.array 1D: Bremsstrahlung model - + Notes ----- For details see L.Lifshin, Ottawa, Proc.9.Ann.Conf.Microbeam Analysis Soc. 53. (1974) """ - lbb0 = lifshin_bremsstrahlung_b0(x,b0,E0) - lbb1 = lifshin_bremsstrahlung_b1(x,b1,E0) + lbb0 = lifshin_bremsstrahlung_b0(x, b0, E0) + lbb1 = lifshin_bremsstrahlung_b1(x, b1, E0) return lbb0 + lbb1 -def lifshin_bremsstrahlung_b0(x, b0, E0 = 200): + +def lifshin_bremsstrahlung_b0(x, b0, E0=200): r""" Calculate the first part of our bremsstrahlung model. """ - assert np.inf not in 1/x, "You have 0.0 in your energy scale. Retry with a cropped energy scale" + assert np.inf not in 1 / x, ( + "You have 0.0 in your energy scale. Retry with a cropped energy scale" + ) # return b0*((E0 - x)/x - np.power(E0 - x, 2)/(E0*x)) - return b0*(E0 -x)/(E0*x)*(1 - (E0 - x)/E0) + return b0 * (E0 - x) / (E0 * x) * (1 - (E0 - x) / E0) -def lifshin_bremsstrahlung_b1(x, b1, E0 = 200): + +def lifshin_bremsstrahlung_b1(x, b1, E0=200): r""" Calculate the second part of our bremsstrahlung model. """ - assert np.inf not in 1/x, "You have 0.0 in your energy scale. Retry with a cropped energy scale" - return b1*np.power((E0 - x),2)/(E0*E0*x) + assert np.inf not in 1 / x, ( + "You have 0.0 in your energy scale. Retry with a cropped energy scale" + ) + return b1 * np.power((E0 - x), 2) / (E0 * E0 * x) + def shelf(x, height, length): r""" @@ -177,46 +190,49 @@ def shelf(x, height, length): Parameters ---------- - x : + x : :np.array 1D: Energy scale. - height : + height : :float: Height in intensity of shelf contribution of the detector. - length : + length : :float: Length in energy of shelf contribution of the detector. - + Returns ------- - shelf : + shelf : :np.array 1D: SDD shelf model. - + Notes ----- For details see :cite:p:`scholze2009modelling` """ return height * erfc(x - length) -def continuum_xrays(x,params_dict={},b0= 0, b1 = 0, E0 = 200,*,elements_dict = {"Si" : 1.0} ): + +def continuum_xrays( + x, params_dict={}, b0=0, b1=0, E0=200, *, elements_dict={"Si": 1.0} +): r""" Computes the continuum X-rays, i.e. the bremsstrahlung multiplied by the absorption and the detection efficiency. Parameters ---------- - x : + x : :np.array 1D: Energy scale. - params_dict : + params_dict : :dict: Dictionnary containing the absorption and detection parameters. - b0 : + b0 : :float: First parameter. - b1 : + b1 : :float: Second parameter. - E0 : + E0 : :float: Energy of the incident beam in keV. - elements_dict : + elements_dict : :dict: Composition of the studied sample. It is required for absorption calculation. - + Returns ------- - continuum_xrays : + continuum_xrays : :np.array 1D: Continuum X-rays model. Notes @@ -225,84 +241,73 @@ def continuum_xrays(x,params_dict={},b0= 0, b1 = 0, E0 = 200,*,elements_dict = { * For an example structure of the params_dict parameter, check the DEFAULT_EDXS_PARAMS espm.conf. * For a custom detection efficiency, check the spectrum fit notebook. """ - if len(params_dict) == 0 : - return 0*x - - B = lifshin_bremsstrahlung( - x, - b0 = b0, - b1 = b1, - E0 = E0 - ) - - A = absorption_correction(x,**params_dict["Abs"],elements_dict=elements_dict) - - if type(params_dict["Det"]) == str : - D = det_efficiency_from_curve(x,params_dict["Det"]) - else : - D = det_efficiency(x,params_dict["Det"]) - - return B * A * D - -def G_bremsstrahlung(x,E0,params_dict,*,elements_dict = {}): + if len(params_dict) == 0: + return 0 * x + + B = lifshin_bremsstrahlung(x, b0=b0, b1=b1, E0=E0) + + A = absorption_correction(x, **params_dict["Abs"], elements_dict=elements_dict) + + if type(params_dict["Det"]) == str: + D = det_efficiency_from_curve(x, params_dict["Det"]) + else: + D = det_efficiency(x, params_dict["Det"]) + + return B * A * D + + +def G_bremsstrahlung(x, E0, params_dict, *, elements_dict={}): r""" Computes the two-parts continuum X-rays for the G matrix. The two parts of the bremsstrahlung are constructed separately so that its parameters can fitted to data. - Absorption and detection are multiplied to each part. + Absorption and detection are multiplied to each part. Parameters ---------- - x : + x : :np.array 1D: Energy scale. - params_dict : + params_dict : :dict: Dictionnary containing the absorption and detection parameters. - elements_dict : + elements_dict : :dict: Composition of the studied sample. It is required for absorption calculation. - + Returns ------- - continuum_xrays : + continuum_xrays : :np.array 2D: Two parts continuum X-rays model with shape (energy scale size, 2). """ - A = absorption_correction(x,**params_dict["Abs"],elements_dict= elements_dict) - - if type(params_dict["Det"]) == str : - D = det_efficiency_from_curve(x,params_dict["Det"]) - else : - D = det_efficiency(x,params_dict["Det"]) - - B0 = A*D*lifshin_bremsstrahlung_b0( - x, - b0 = 1, - E0 = E0 - ) - - B1 = A*D*lifshin_bremsstrahlung_b1( - x, - b1=1, - E0 = E0 - ) + A = absorption_correction(x, **params_dict["Abs"], elements_dict=elements_dict) + + if type(params_dict["Det"]) == str: + D = det_efficiency_from_curve(x, params_dict["Det"]) + else: + D = det_efficiency(x, params_dict["Det"]) + + B0 = A * D * lifshin_bremsstrahlung_b0(x, b0=1, E0=E0) + + B1 = A * D * lifshin_bremsstrahlung_b1(x, b1=1, E0=E0) + + B = np.vstack((B0, B1)).T - B = np.vstack((B0,B1)).T - return B -# @number_to_symbol_list -# def elts_dict_from_W (part_W,*,elements = []) : + +# @number_to_symbol_list +# def elts_dict_from_W (part_W,*,elements = []) : # r""" # Create a dictionnary of the elemental concentration from a fitted W. It useful to recompute absorption during the our custom NMF calculations. # Parameters # ---------- -# part_W : +# part_W : # :np.array 2D: W matrix output from the NMF calculation. It only makes sense when the NMF decomposition if performed with G. -# elements : +# elements : # :list: List of elements as atomic number or symbol. # Returns # ------- -# elements_dictionnary : +# elements_dictionnary : # :dict: Dictionnary containing the concentration associated to each chemical element of the problem. - + # Examples # -------- # >>> import numpy as np @@ -319,25 +324,26 @@ def G_bremsstrahlung(x,E0,params_dict,*,elements_dict = {}): # mask_zeros = np.sum(part_W,axis=0) != 0 # norm_P = np.mean(part_W[:,mask_zeros] / part_W[:,mask_zeros].sum(axis = 0),axis=1) # elements_dict = {} -# #with open(SYMBOLS_PERIODIC_TABLE,"r") as f : +# #with open(SYMBOLS_PERIODIC_TABLE,"r") as f : # # SPT = json.load(f)["table"] # for i,elt in enumerate(elements) : # elements_dict[elt] = norm_P[i] # * SPT[elt]["atomic_mass"] # factor = sum(elements_dict.values()) # return {key:elements_dict[key]/factor for key in elements_dict} -def elts_dict_from_dict_list (dict_list) : + +def elts_dict_from_dict_list(dict_list): r""" Create a single dictionnary of the elemental concentration from a list of dictionnary containing chemical compositions. The new dictionnary corresponds to the average chemical composition of the list. Parameters ---------- - dict_list : + dict_list : :list [dict]: list of chemical composition dictionnary Returns ------- - elements_dictionnary : + elements_dictionnary : :dict: Dictionnary containing the concentration associated to each chemical elements contained in the input. Examples @@ -352,13 +358,14 @@ def elts_dict_from_dict_list (dict_list) : ----- This function should maybe move to another part of espm. """ - unique_elts_dict = sum((Counter(x) for x in dict_list),Counter()) + unique_elts_dict = sum((Counter(x) for x in dict_list), Counter()) sum_elts = sum(unique_elts_dict.values()) - for e in unique_elts_dict : + for e in unique_elts_dict: unique_elts_dict[e] /= sum_elts return unique_elts_dict -def elts_list_from_dict_list (dict_list) : + +def elts_list_from_dict_list(dict_list): r""" Create a list of elements from a list of dictionnary containing chemical compositions. The list contains all the elements contained in the input. @@ -376,12 +383,9 @@ def elts_list_from_dict_list (dict_list) : -------- >>> import numpy as np >>> from espm.models.EDXS_function import elts_list_from_dict_list - >>> dicts = [{"Si" : 1.0, "C" : 0.5, "O" : 0.5},{"Ca" : 1.0, "K" : 2.0},{"O" : 1.0, "Si" : 0.5},{"C" : 1.0}] + >>> dicts = [{"Si" : 1.0, "C" : 0.5, "O" : 0.5},{"Ca" : 1.0, "K" : 2.0},{"O" : 1.0, "Si" : 0.5},{"C" : 1.0}] >>> elts_list_from_dict_list(dicts) ["Si","Ca","O","C","K"] - """ - unique_elts_dict = sum((Counter(x) for x in dict_list),Counter()) + """ + unique_elts_dict = sum((Counter(x) for x in dict_list), Counter()) return list(unique_elts_dict.keys()) - - - diff --git a/espm/models/__init__.py b/espm/models/__init__.py index 66b0188..8f8a0eb 100644 --- a/espm/models/__init__.py +++ b/espm/models/__init__.py @@ -4,10 +4,10 @@ .. note:: - For now espm supports only the modelling of EDS data but we aim + For now espm supports only the modelling of EDS data but we aim at supporting the modelling of EELS data too. """ from espm.models.base import PhysicalModel, ToyModel -from espm.models.edxs import EDXS \ No newline at end of file +from espm.models.edxs import EDXS diff --git a/espm/models/absorption_edxs.py b/espm/models/absorption_edxs.py index 03954f9..74bbf18 100644 --- a/espm/models/absorption_edxs.py +++ b/espm/models/absorption_edxs.py @@ -6,15 +6,17 @@ """ -from scipy.interpolate import interp1d -from espm.utils import number_to_symbol_dict, atomic_to_weight_dict, approx_density -from espm.conf import HSPY_MAC -import numpy as np from pathlib import Path -from espm.conf import DB_PATH + +import numpy as np +from scipy.interpolate import interp1d + +from espm.conf import DB_PATH, HSPY_MAC +from espm.utils import approx_density, atomic_to_weight_dict, number_to_symbol_dict + @number_to_symbol_dict -def absorption_coefficient (x,atomic_fraction = False,*,elements_dict = {"Si" : 1.0}) : +def absorption_coefficient(x, atomic_fraction=False, *, elements_dict={"Si": 1.0}): r""" Calculate the mass-absorption coefficent of a given composition at a certain energy or over a certain energy range. @@ -24,7 +26,7 @@ def absorption_coefficient (x,atomic_fraction = False,*,elements_dict = {"Si" : :np.array 1D: Energy value or energy scale over which the mass-absorption coefficient is calculated. atomic_fraction : :boolean: If True, the atomic concentrations of elements_dict are converted in atomic weight fractions. - elements_dict : + elements_dict : :dict: Dictionnary with chemical elements as keys and atomic weight fractions (or atomic concentrations) as values. Returns @@ -37,46 +39,56 @@ def absorption_coefficient (x,atomic_fraction = False,*,elements_dict = {"Si" : The mass-absorption coefficients are calculated using the database of hyperspy :cite:p:`francisco_de_la_pena_2022_7263263`. """ mu = np.zeros_like(x) - if atomic_fraction : - elements_dict = atomic_to_weight_dict(elements_dict = elements_dict) + if atomic_fraction: + elements_dict = atomic_to_weight_dict(elements_dict=elements_dict) sum_elts = sum(elements_dict.values()) - - for key in elements_dict.keys() : + + for key in elements_dict.keys(): x_db = HSPY_MAC[key]["energies (keV)"] y_db = HSPY_MAC[key]["mass_absorption_coefficient (cm2/g)"] - interp_func = interp1d(x_db,y_db,kind="cubic") - mu += elements_dict[key]*interp_func(x)/sum_elts + interp_func = interp1d(x_db, y_db, kind="cubic") + mu += elements_dict[key] * interp_func(x) / sum_elts - if len(elements_dict.keys()) == 0 : - mu = 1 / np.power(x,3) + if len(elements_dict.keys()) == 0: + mu = 1 / np.power(x, 3) return mu -def absorption_correction (x,thickness = 100e-7,toa = 90,density = None,atomic_fraction = False,*,elements_dict = {"Si" : 1.0},**kwargs) : + +def absorption_correction( + x, + thickness=100e-7, + toa=90, + density=None, + atomic_fraction=False, + *, + elements_dict={"Si": 1.0}, + **kwargs, +): r""" Calculate the contribution of the absorption on the EDX spectrum for a thin slab of material with a given composition at a certain energy or over a certain energy range. The absorption correction is calculated using the following formula : - + .. math:: - - \frac{1 - \exp(-\chi)}{\chi} - + + \frac{1 - \exp(-\chi)}{\chi} + where :math:`\chi = \mu \rho t / \sin(\theta)`, :math:`\mu` is the mass-absorption coefficient, :math:`\rho` is the density of the material, :math:`t` is the thickness of the sample and :math:`\theta` is the average take-off angle of the x-rays travelling from the sample to the x-ray detectors. Parameters ---------- x : :np.array 1D: Energy value or energy scale over which the mass-absorption coefficient is calculated. - thickness : + thickness : :float: Thickness of the material slab in meter. If the thickness is set to 0.0, the function will return 1.0. toa : :float: Take-off angle in degrees of the x-rays travelling from the sample to the x-ray detectors. - density : + density : :float: Density of the material in g/m3 (to be checked). If None, an approximation of the density based on the atomic numbers of the elements of the materials is calculated. atomic_fraction : :boolean: If True, the atomic concentrations of elements_dict are converted in atomic weight fractions. - elements_dict : + elements_dict : :dict: Dictionnary with chemical elements as keys and atomic weight fractions (or atomic concentrations) as values. Returns @@ -84,18 +96,21 @@ def absorption_correction (x,thickness = 100e-7,toa = 90,density = None,atomic_f absorption correction :np.array 1D: Calculated value of the absorption correction on all the energy range. """ - mu = absorption_coefficient(x,atomic_fraction,elements_dict = elements_dict) + mu = absorption_coefficient(x, atomic_fraction, elements_dict=elements_dict) rad_toa = np.deg2rad(toa) - if density is None : - density = approx_density(atomic_fraction,elements_dict = elements_dict) - - if thickness == 0 : + if density is None: + density = approx_density(atomic_fraction, elements_dict=elements_dict) + + if thickness == 0: return 1.0 - else : - chi = mu*density*thickness/np.sin(rad_toa) - return (1 - np.exp(-chi))/chi + else: + chi = mu * density * thickness / np.sin(rad_toa) + return (1 - np.exp(-chi)) / chi -def absorption_mass_thickness(x,mass_thickness, toa = 90, atomic_fraction = True, *, elements_dict = {"Si" : 1.0}) : + +def absorption_mass_thickness( + x, mass_thickness, toa=90, atomic_fraction=True, *, elements_dict={"Si": 1.0} +): r""" Calculate the contribution of the absorption of a mass-thickness map. @@ -103,13 +118,13 @@ def absorption_mass_thickness(x,mass_thickness, toa = 90, atomic_fraction = True ---------- x : :np.array 1D: Energy value or energy scale over which the mass-absorption coefficient is calculated. - mass_thickness: + mass_thickness: :np.array 2D: Mass-thickness map. See the notes for details. toa : :float: Take-off angle in degrees of the x-rays travelling from the sample to the x-ray detectors. atomic_fraction : :boolean: If True, the atomic concentrations of elements_dict are converted in atomic weight fractions. - elements_dict : + elements_dict : :dict: Dictionnary with chemical elements as keys and atomic weight fractions (or atomic concentrations) as values. Returns @@ -121,15 +136,20 @@ def absorption_mass_thickness(x,mass_thickness, toa = 90, atomic_fraction = True ----- The mass-thickness map can be extracted from an EDX spectrum image by calculating the concentration of an element for both K and L lines. Since there should be only one concentration for both lines, it is possible to determine the contribution of the absorption for each pixel and thus obtain the mass-thickness map. """ - mu = absorption_coefficient(x,atomic_fraction,elements_dict = elements_dict) + mu = absorption_coefficient(x, atomic_fraction, elements_dict=elements_dict) rad_toa = np.deg2rad(toa) - if type(mu) == float : - chi = mu*mass_thickness/np.sin(rad_toa) - else : - chi = mu[:,np.newaxis]@((mass_thickness.reshape(-1)[:,np.newaxis]).T)/np.sin(rad_toa) - return (1-np.exp(-chi))/chi - -def det_efficiency_from_curve (x,filename,kind = "cubic") : + if type(mu) == float: + chi = mu * mass_thickness / np.sin(rad_toa) + else: + chi = ( + mu[:, np.newaxis] + @ ((mass_thickness.reshape(-1)[:, np.newaxis]).T) + / np.sin(rad_toa) + ) + return (1 - np.exp(-chi)) / chi + + +def det_efficiency_from_curve(x, filename, kind="cubic"): r""" Interpolate a detection efficiency curve that is stored in ~/espm/tables. @@ -137,7 +157,7 @@ def det_efficiency_from_curve (x,filename,kind = "cubic") : ---------- x : :np.array 1D: Energy value or energy scale over which the mass-absorption coefficient is calculated. - filename: + filename: :string: Name of the file of the detection efficiency. kind : :string: Polynomial order of the interpolation. @@ -148,11 +168,19 @@ def det_efficiency_from_curve (x,filename,kind = "cubic") : :np.array 1D: Interpolated detection efficiency. """ array = np.loadtxt(DB_PATH / Path(filename)) - x_curve,y_curve = array[:,0], array[:,1] - interp_func = interp1d(x_curve,y_curve,kind = kind) + x_curve, y_curve = array[:, 0], array[:, 1] + interp_func = interp1d(x_curve, y_curve, kind=kind) return interp_func(x) -def det_efficiency_layer (x, thickness = 100e-7, density = None, atomic_fraction = False, *, elements_dict = {"Si" : 1.0}) : + +def det_efficiency_layer( + x, + thickness=100e-7, + density=None, + atomic_fraction=False, + *, + elements_dict={"Si": 1.0}, +): r""" Calculate the proportion of absorbed X-rays for one virtual layer of the detector. @@ -160,13 +188,13 @@ def det_efficiency_layer (x, thickness = 100e-7, density = None, atomic_fraction ---------- x : :np.array 1D: Energy value or energy scale over which the mass-absorption coefficient is calculated. - thickness : + thickness : :float: Thickness of the material slab in meter. If the thickness is set to 0.0, the function will return 1.0. - density : + density : :float: Density of the material in g/m3 (to be checked). If None, an approximation of the density based on the atomic numbers of the elements of the materials is calculated. atomic_fraction : :boolean: If True, the atomic concentrations of elements_dict are converted in atomic weight fractions. - elements_dict : + elements_dict : :dict: Dictionnary with chemical elements as keys and atomic weight fractions (or atomic concentrations) as values. Returns @@ -178,12 +206,13 @@ def det_efficiency_layer (x, thickness = 100e-7, density = None, atomic_fraction ----- We assume that the X-rays arrive perpendicular to the detector. """ - mu = absorption_coefficient(x,atomic_fraction,elements_dict=elements_dict) - if density is None : - density = approx_density(atomic_fraction,elements_dict = elements_dict) - return mu*thickness*density + mu = absorption_coefficient(x, atomic_fraction, elements_dict=elements_dict) + if density is None: + density = approx_density(atomic_fraction, elements_dict=elements_dict) + return mu * thickness * density + -def det_efficiency (x,det_dict) : +def det_efficiency(x, det_dict): r""" Simulate the detection efficiency of an EDX detector. The absorption wihtin one layer of the detector is calculated according to the following equation, corresponding to the Beer-Lambert law: @@ -221,9 +250,9 @@ def det_efficiency (x,det_dict) : """ efficiency = np.ones_like(x) - for layer in det_dict : - if layer == "detection" : - efficiency *= 1 - np.exp(-det_efficiency_layer(x,**det_dict[layer])) - else : - efficiency *= np.exp(-det_efficiency_layer(x,**det_dict[layer])) + for layer in det_dict: + if layer == "detection": + efficiency *= 1 - np.exp(-det_efficiency_layer(x, **det_dict[layer])) + else: + efficiency *= np.exp(-det_efficiency_layer(x, **det_dict[layer])) return efficiency diff --git a/espm/models/base.py b/espm/models/base.py index 457144e..811d4c5 100644 --- a/espm/models/base.py +++ b/espm/models/base.py @@ -2,40 +2,42 @@ Models base classes ------------------- -The PhysicalModel is the abstract class from which the EDX model inherits. +The PhysicalModel is the abstract class from which the EDX model inherits. The ToyModel can be used for testing data analysis algorithms. """ +import json from abc import ABC, abstractmethod from pathlib import Path -import json -import espm.conf as conf + import numpy as np -from typing import Optional + +from espm import conf + class Model(ABC): r"""Abstract class for models.""" + def __init__(self): super().__init__() @abstractmethod - def generate_g_matr (self, g_parameters) : + def generate_g_matr(self, g_parameters): pass @abstractmethod - def generate_phases (self, phase_parameters) : + def generate_phases(self, phase_parameters): pass - class ToyModel(Model): r"""Toy model. - + Simple model with a random G matrix and phases. The G matrix is generated with a random number of gaussians per column. The phases are generated using a vector drawn from a Laplacian distribution and multiplied by the G matrix. - + Parameters ---------- L : int @@ -66,7 +68,10 @@ class ToyModel(Model): """ - def __init__(self, L: int=200, C: int=15, K: int=3, seed: Optional[int]=None) -> None: + + def __init__( + self, L: int = 200, C: int = 15, K: int = 3, seed: int | None = None + ) -> None: super().__init__() self.L = L self.C = C @@ -74,8 +79,8 @@ def __init__(self, L: int=200, C: int=15, K: int=3, seed: Optional[int]=None) -> self.seed = seed self.G = None self.phases = None - - def generate_g_matr (self, *args, **kwargs) -> None: + + def generate_g_matr(self, *args, **kwargs) -> None: r"""Generate G matrix. Parameters @@ -89,30 +94,30 @@ def generate_g_matr (self, *args, **kwargs) -> None: """ - if self.G is not None : + if self.G is not None: return - + np.random.seed(seed=self.seed) n_el = 45 - n_gauss = np.random.randint(2, 5,[self.C]) - l = np.arange(0, 1, 1/self.L) + n_gauss = np.random.randint(2, 5, [self.C]) + l = np.arange(0, 1, 1 / self.L) mu_gauss = np.random.rand(n_el) - sigma_gauss = 1/n_el + np.abs(np.random.randn(n_el))/n_el/5 + sigma_gauss = 1 / n_el + np.abs(np.random.randn(n_el)) / n_el / 5 - G = np.zeros([self.L,self.C]) + G = np.zeros([self.L, self.C]) def gauss(x, mu, sigma): # return np.exp(-(x-mu)**2/(2*sigma**2)) / (sigma * np.sqrt(2*np.pi)) - return np.exp(-(x-mu)**2/(2*sigma**2)) + return np.exp(-((x - mu) ** 2) / (2 * sigma**2)) for i, c in enumerate(n_gauss): - inds = np.random.choice(n_el, size=[c] , replace=False) + inds = np.random.choice(n_el, size=[c], replace=False) for ind in inds: - w = 0.1+0.9*np.random.rand() - G[:,i] += w * gauss(l, mu_gauss[ind], sigma_gauss[ind]) + w = 0.1 + 0.9 * np.random.rand() + G[:, i] += w * gauss(l, mu_gauss[ind], sigma_gauss[ind]) self.G = G - def generate_phases (self, *args, **kwargs) -> None: + def generate_phases(self, *args, **kwargs) -> None: r"""Generate phases. Parameters @@ -126,27 +131,36 @@ def generate_phases (self, *args, **kwargs) -> None: """ - if self.phases is not None : + if self.phases is not None: return np.random.seed(seed=self.seed) Wdot = np.abs(np.random.laplace(size=[self.C, self.K])) - self.Wdot = Wdot / np.mean(Wdot)/self.L + self.Wdot = Wdot / np.mean(Wdot) / self.L self.generate_g_matr() self.phases = self.G @ self.Wdot - - -class PhysicalModel(Model) : + +class PhysicalModel(Model): r"""Abstract class of the models""" - def __init__(self, e_offset, e_size, e_scale, params_dict,db_name="default_xrays.json", E0 = 200, **kwargs) : + + def __init__( + self, + e_offset, + e_size, + e_scale, + params_dict, + db_name="default_xrays.json", + E0=200, + **kwargs, + ): super().__init__() self.x = self.build_energy_scale(e_offset, e_size, e_scale) self.params_dict = params_dict - if db_name is None : + if db_name is None: self.db_dict = {} self.db_mdata = {} - else : + else: self.db_dict = self.extract_DB(db_name) self.db_mdata = self.extract_DB_mdata(db_name) self.bkgd_in_G = False @@ -155,14 +169,13 @@ def __init__(self, e_offset, e_size, e_scale, params_dict,db_name="default_xrays self.phases = None self.E0 = E0 - - def extract_DB (self,db_name) : + def extract_DB(self, db_name): r""" Read the cross-sections from the database Parameters ---------- - db_name : + db_name : :string: Name of the json database file Returns @@ -171,17 +184,17 @@ def extract_DB (self,db_name) : :dict: A dictionnary containing the cross-sections in the database """ db_path = conf.DB_PATH / Path(db_name) - with open(db_path,"r") as f : + with open(db_path, "r") as f: json_dict = json.load(f) return json_dict["table"] - def extract_DB_mdata (self,db_name) : + def extract_DB_mdata(self, db_name): r""" Read the metadata of the database Parameters ---------- - db_name : + db_name : :string: Name of the json database file Returns @@ -190,11 +203,11 @@ def extract_DB_mdata (self,db_name) : :dict: A dictionnary containing the metadata related to the database """ db_path = conf.DB_PATH / Path(db_name) - with open(db_path,"r") as f : + with open(db_path, "r") as f: json_dict = json.load(f) return json_dict["metadata"] - def build_energy_scale(self,e_offset, e_size, e_scale) : + def build_energy_scale(self, e_offset, e_size, e_scale): r""" Build an array corresponding to a linear energy scale. @@ -202,9 +215,9 @@ def build_energy_scale(self,e_offset, e_size, e_scale) : ---------- e_offset :float: Offset of the energy scale - e_size + e_size :int: Number of channels of the detector - e_scale : + e_scale : :float: Scale in keV/channel Returns @@ -212,17 +225,17 @@ def build_energy_scale(self,e_offset, e_size, e_scale) : energy scale :np.array 1D: Linear energy scale based on the given parameters of shape (e_size,) """ - return np.linspace(e_offset,e_offset+e_size*e_scale,num=e_size) - + return np.linspace(e_offset, e_offset + e_size * e_scale, num=e_size) + @abstractmethod - def NMF_initialize_W (self, D) : + def NMF_initialize_W(self, D): """ Function to be called when initializing the NMF optimization. It returns the matrices W. The basic implementation is to return the pseudo-inverse of the matrix G. Parameters ---------- - D : + D : :np.array 2D: scikit-learn initialization matrix. It should have the shape (n_features, n_components). Returns @@ -230,36 +243,35 @@ def NMF_initialize_W (self, D) : W : :np.array 2D: The initialized matrix W of shape (n_G, n_components) """ - W = np.abs(np.linalg.lstsq(self.G, D,rcond=None)[0]) + W = np.abs(np.linalg.lstsq(self.G, D, rcond=None)[0]) return W @abstractmethod - def NMF_update (self, W = None) : + def NMF_update(self, W=None): """ Function to be called when during the NMF optimization. It returns the matrix G, updated if necessary. It should be run in between each W iteration. You do not need to implement it, if you do not need to update the matrix G during the optimization. Parameters ---------- - W : + W : :np.array 2D: The part of the matrix W that is required to update the matrix G. Returns ------- G : - :np.array 2D: The updated matrix G + :np.array 2D: The updated matrix G """ return self.G @abstractmethod - def NMF_simplex (self) : + def NMF_simplex(self): """ Function to be called when using the simplex constraint in the ESpM-NMF. Its purpose is to return the indices of the matrix W that will be considered to apply the simplex constraint. - Returns : + Returns : -------- indices : :np.array 1D: Indices to be considered to apply the simplex constraint """ return np.arange(self.G.shape[1]) - diff --git a/espm/models/edxs.py b/espm/models/edxs.py index 1db5245..4d09287 100644 --- a/espm/models/edxs.py +++ b/espm/models/edxs.py @@ -8,24 +8,37 @@ """ -import numpy as np import re + +import numpy as np + from espm.models import PhysicalModel -from espm.models.EDXS_function import G_bremsstrahlung, continuum_xrays, gaussian, read_lines_db, read_compact_db, elts_dict_from_dict_list -from espm.conf import DEFAULT_EDXS_PARAMS -from espm.utils import arg_helper, symbol_to_number_dict, symbol_to_number_list -from espm.models.absorption_edxs import absorption_correction, det_efficiency, det_efficiency_from_curve, absorption_mass_thickness +from espm.models.absorption_edxs import ( + absorption_correction, + det_efficiency, + det_efficiency_from_curve, +) +from espm.models.EDXS_function import ( + G_bremsstrahlung, + continuum_xrays, + elts_dict_from_dict_list, + gaussian, + read_compact_db, + read_lines_db, +) +from espm.utils import symbol_to_number_dict, symbol_to_number_list + # Class to model the EDXS spectra. This is a temporary version since there are some design issues. class EDXS(PhysicalModel): def __init__( - self, - *args, + self, + *args, width_slope=0.01, width_intercept=0.065, - custom_init = False, - **kwargs + custom_init=False, + **kwargs, ): r""" :database_path: file path to a database of x-ray data (energy and intensity ratios of the peaks) @@ -37,10 +50,10 @@ def __init__( :width_slope: The FWHM of the detector increases with energy which is modeled with an affine function. This is the slope of this affine function (float). :width_intercept: The FWHM of the detector increases with energy which is modeled with an affine function. This is the intercept of this affine function (float). """ - super().__init__(*args,**kwargs) + super().__init__(*args, **kwargs) self.width_slope = width_slope self.width_intercept = width_intercept - + self.lines = self.db_mdata["lines"] self.norm = 1.0 self.model_elts = [] @@ -48,128 +61,150 @@ def __init__( # Tranfer the ranges from eds_espm to the physical model self.ranges = None - def __add_elts_G(self, reference_elt = {}, *, elements=[]): + def __add_elts_G(self, reference_elt={}, *, elements=[]): for elt in elements: - if self.lines : - energies, cs = read_lines_db(elt,self.db_dict) - else : - energies, cs = read_compact_db(elt,self.db_dict) - if elt in reference_elt : + if self.lines: + energies, cs = read_lines_db(elt, self.db_dict) + else: + energies, cs = read_compact_db(elt, self.db_dict) + if elt in reference_elt: peaks_low = np.zeros((self.x.shape[0], 1)) peaks_high = np.zeros((self.x.shape[0], 1)) for i, energy in enumerate(energies): if (energy > np.min(self.x)) and (energy < np.max(self.x)): - - if type(self.params_dict["Det"]) == str : - D = det_efficiency_from_curve(energy,self.params_dict["Det"]) - else : - D = det_efficiency(energy,self.params_dict["Det"]) - - A = absorption_correction(energy,**self.params_dict["Abs"],elements_dict = {elt : 1.0}) - + if type(self.params_dict["Det"]) == str: + D = det_efficiency_from_curve( + energy, self.params_dict["Det"] + ) + else: + D = det_efficiency(energy, self.params_dict["Det"]) + + A = absorption_correction( + energy, **self.params_dict["Abs"], elements_dict={elt: 1.0} + ) + width = self.width_slope * energy + self.width_intercept - if energy < reference_elt[elt] : + if energy < reference_elt[elt]: peaks_low += ( - cs[i] - * gaussian(self.x, energy, width / 2.3548)[ - np.newaxis - ].T - )*D*A - else : + ( + cs[i] + * gaussian(self.x, energy, width / 2.3548)[ + np.newaxis + ].T + ) + * D + * A + ) + else: peaks_high += ( - cs[i] - * gaussian(self.x, energy, width / 2.3548)[ - np.newaxis - ].T - )*D*A - peaks = np.hstack((peaks_low,peaks_high)) - else : + ( + cs[i] + * gaussian(self.x, energy, width / 2.3548)[ + np.newaxis + ].T + ) + * D + * A + ) + peaks = np.hstack((peaks_low, peaks_high)) + else: peaks = np.zeros((self.x.shape[0], 1)) for i, energy in enumerate(energies): - # The actual detected width is calculated at each energy if (energy > np.min(self.x)) and (energy < np.max(self.x)): - - if type(self.params_dict["Det"]) == str : - D = det_efficiency_from_curve(energy,self.params_dict["Det"]) - else : - D = det_efficiency(energy,self.params_dict["Det"]) - - A = absorption_correction(energy,**self.params_dict["Abs"],elements_dict = {elt : 1.0}) - + if type(self.params_dict["Det"]) == str: + D = det_efficiency_from_curve( + energy, self.params_dict["Det"] + ) + else: + D = det_efficiency(energy, self.params_dict["Det"]) + + A = absorption_correction( + energy, **self.params_dict["Abs"], elements_dict={elt: 1.0} + ) + width = self.width_slope * energy + self.width_intercept - peaks += ( - cs[i] - * gaussian(self.x, energy, width / 2.3548)[ - np.newaxis - ].T - )*D*A - + ( + cs[i] + * gaussian(self.x, energy, width / 2.3548)[np.newaxis].T + ) + * D + * A + ) + # print(np.max(peaks, axis = 0)) # print(str(elt)) - if np.all((np.max(peaks, axis = 0)) > 0.0): + if np.all((np.max(peaks, axis=0)) > 0.0): self.G = np.concatenate((self.G, peaks), axis=1) - if elt in reference_elt : - self.model_elts.append(str(elt)+'_lo') - self.model_elts.append(str(elt)+'_hi') - else : + if elt in reference_elt: + self.model_elts.append(str(elt) + "_lo") + self.model_elts.append(str(elt) + "_hi") + else: self.model_elts.append(str(elt)) - else : - print("The energy split of the element : {} leads to empty G columns. Please remove split or change its energy.".format(elt)) - raise ValueError("Empty G column") + else: + print( + f"The energy split of the element : {elt} leads to empty G columns. Please remove split or change its energy." + ) + raise ValueError("Empty G column") - def _add_ignored_elts(self, elements = []) : + def _add_ignored_elts(self, elements=[]): for elt in elements: - if self.lines : - energies, cs = read_lines_db(elt,self.db_dict) - else : - energies, cs = read_compact_db(elt,self.db_dict) - + if self.lines: + energies, cs = read_lines_db(elt, self.db_dict) + else: + energies, cs = read_compact_db(elt, self.db_dict) + peaks_list = [] for i, energy in enumerate(energies): - # The actual detected width is calculated at each energy if (energy > np.min(self.x)) and (energy < np.max(self.x)): - - if type(self.params_dict["Det"]) == str : - D = det_efficiency_from_curve(energy,self.params_dict["Det"]) - else : - D = det_efficiency(energy,self.params_dict["Det"]) - - A = absorption_correction(energy,**self.params_dict["Abs"],elements_dict = {elt : 1.0}) - + if type(self.params_dict["Det"]) == str: + D = det_efficiency_from_curve(energy, self.params_dict["Det"]) + else: + D = det_efficiency(energy, self.params_dict["Det"]) + + A = absorption_correction( + energy, **self.params_dict["Abs"], elements_dict={elt: 1.0} + ) + width = self.width_slope * energy + self.width_intercept - peaks_list.append( - ( - cs[i] - * gaussian(self.x, energy, width / 2.3548) - )*D*A + (cs[i] * gaussian(self.x, energy, width / 2.3548)) * D * A ) peaks = np.array(peaks_list).T - if np.all((np.max(peaks, axis = 0)) > 0.0): + if np.all((np.max(peaks, axis=0)) > 0.0): self.G = np.concatenate((self.G, peaks), axis=1) - for i in range(peaks.shape[1]) : - self.model_elts.append(str(elt) + '_ign' + str(i)) - else : - print("The energy split of the element : {} leads to empty G columns. Please remove split or change its energy.".format(elt)) + for i in range(peaks.shape[1]): + self.model_elts.append(str(elt) + "_ign" + str(i)) + else: + print( + f"The energy split of the element : {elt} leads to empty G columns. Please remove split or change its energy." + ) raise ValueError("Empty G column") @symbol_to_number_list @symbol_to_number_dict - def generate_g_matr(self, g_type="bremsstrahlung", ignored_elements = ['Cu'],*,elements=[],elements_dict = {},**kwargs): + def generate_g_matr( + self, + g_type="bremsstrahlung", + ignored_elements=["Cu"], + *, + elements=[], + elements_dict={}, + **kwargs, + ): r""" - Generate the G matrix. With a complete model the matrix is (e_size,n+2). The first n columns correspond to the sum of X-ray characteristic peaks associated to each shell of the elements. The last 2 columns correspond to a bremsstrahlung model. - + Generate the G matrix. With a complete model the matrix is (e_size,n+2). The first n columns correspond to the sum of X-ray characteristic peaks associated to each shell of the elements. The last 2 columns correspond to a bremsstrahlung model. + The first n columns of G (noted :math:`\kappa`), corresponding to the characteristic X-rays, can be calculated using the following formula: - + .. math:: \kappa_{k,Z} = \sum_{ij} x_{ij}(Z) \frac{1}{\Delta(\varepsilon_k) \sqrt{2\pi} } e{^{-\frac{1}{2} {\left( \frac{\varepsilon_k - \varepsilon^{ij}(Z)}{\Delta(\varepsilon_k)} \right)}^2}} - + where :math:`\varepsilon_k` is the energy of the kth energy channel, :math:`\varepsilon^{ij}(Z)` is the energy of the ijth line of the Zth element, :math:`\Delta(\varepsilon_k)` is the FWHM of the detector at the energy :math:`\varepsilon_k` and :math:`x_{ij}(Z)` is the intensity ratio of the ijth line of the Zth element. The last term of the equation is a gaussian function centered at :math:`\varepsilon^{ij}(Z)` and with a FWHM of :math:`\Delta(\varepsilon_k)`. The last two columns of G (noted :math:`\beta`), corresponding to the bremsstrahlung model, can be calculated using the following formula: @@ -181,88 +216,90 @@ def generate_g_matr(self, g_type="bremsstrahlung", ignored_elements = ['Cu'],*,e \beta_{k,n+2} = \frac{(\varepsilon_0 - \varepsilon_k)^2}{\varepsilon^2_0 \varepsilon_k} Note that there is no parameter for the bremsstrahlung since it is supposed to be learned with the W matrix (see espm.estimators). - + Parameters ---------- - g_type : + g_type : :string: Options of the edxs model to include in the G matrix. The three possibilities are "identity", "no_brstlg" and "bremsstrahlung". G is going to be the identity matrix, only characteristic X-ray peaks or characteristic X-rays plus bremsstrahlung model, respectively. - elements_dict : + elements_dict : :dict: The keys are chemical elements (atomic number) and the values are cut-off energies. This argument is used to split some of the columns of G into 2 columns. The first column corresponds to characteristic X-rays before the cut-off and second one corresponds to characteristic X-rays before the cut-off. This feature is implemented to enable more accurate absorption correction. - elements : + elements : :list: List of modeled chemical elements. The list can be populated either with atomic numbers or chemical symbols, e.g. "Fe" or 26. Returns ------- g matrix : - :np.array 2D: matrix of the edx model. + :np.array 2D: matrix of the edx model. Notes ----- See our paper about the espm package :cite:p:`teurtrie2023espm` for more information about the equations of this model. """ - + # Reset the internally stored elements list self.model_elts = [] @symbol_to_number_list - def convert_elts(elements = ignored_elements) : + def convert_elts(elements=ignored_elements): return elements - + conv_ignored_elts = convert_elts(elements=ignored_elements) valid_elts = self.__check_elts_in_G(elements) valid_ignored = self.__check_elts_in_G(conv_ignored_elts) - - if g_type == "bremsstrahlung" : + + if g_type == "bremsstrahlung": self.bkgd_in_G = True - else : + else: self.bkgd_in_G = False # None is a default value for the G matrix and thus G will be considered to be the identity matrix in most of espm functions. - if len(valid_elts) == 0: - self.G = None - - elif g_type == "identity" : + if len(valid_elts) == 0 or g_type == "identity": self.G = None # model based on elements_list elif (g_type == "bremsstrahlung") or (g_type == "no_brstlg"): - # The number of shells depend on the element, it is then not straightforward to pre-determine the size of g_matr self.G = np.zeros((self.x.shape[0], 0)) # For each element we unpack all shells and then unpack all lines of each shell. - self.__add_elts_G( reference_elt = elements_dict, elements = valid_elts) + self.__add_elts_G(reference_elt=elements_dict, elements=valid_elts) self._add_ignored_elts(elements=valid_ignored) - + # Appends a pure continuum spectrum is needed if self.bkgd_in_G: - approx_elts = {key : 1.0/len(valid_elts) for key in valid_elts} - brstlg_spectrum = G_bremsstrahlung(self.x,self.E0,self.params_dict,elements_dict=approx_elts) - if np.max(brstlg_spectrum) > 0.0 : + approx_elts = {key: 1.0 / len(valid_elts) for key in valid_elts} + brstlg_spectrum = G_bremsstrahlung( + self.x, self.E0, self.params_dict, elements_dict=approx_elts + ) + if np.max(brstlg_spectrum) > 0.0: self.G = np.concatenate((self.G, brstlg_spectrum), axis=1) - else : - print("Bremsstrahlung parameters were not provided, bkgd not added in G") + else: + print( + "Bremsstrahlung parameters were not provided, bkgd not added in G" + ) self.bkgd_in_G = False norms = np.sum(self.G, axis=0, keepdims=True) - if g_type == "bremsstrahlung" : + if g_type == "bremsstrahlung": norms[0][:-2] = np.mean(norms[0][:-2]) - else : + else: norms[0] = np.mean(norms[0]) self.norm = norms self.G /= self.norm - else : - print("g_type has to be one of those : \"bremsstrahlung\", \"no_brstlg\" or \"identity\". G will be None, corresponding to \"identity\". ") - + else: + print( + 'g_type has to be one of those : "bremsstrahlung", "no_brstlg" or "identity". G will be None, corresponding to "identity". ' + ) + def __check_elts_in_G(self, elements): """ Check if the elements of the metadata are in the range of the energy axis. """ valid_elts = [] for elt in elements: - if self.lines : - energies, cs = read_lines_db(elt,self.db_dict) - else : - energies, cs = read_compact_db(elt,self.db_dict) + if self.lines: + energies, cs = read_lines_db(elt, self.db_dict) + else: + energies, cs = read_compact_db(elt, self.db_dict) energy_range = [np.min(self.x), np.max(self.x)] found = 0 @@ -272,21 +309,21 @@ def __check_elts_in_G(self, elements): found = 1 break if not found: - print("No peak is present in the energy range for element : {}".format(elt)) + print(f"No peak is present in the energy range for element : {elt}") return valid_elts - - def generate_phases(self, phases_parameters) : + + def generate_phases(self, phases_parameters): r""" - Generate a series of spectra from list of phase parameters. + Generate a series of spectra from list of phase parameters. Parameters ---------- - phases_parameters : + phases_parameters : :list: List of dicts containing the phase parameters. - + Returns ------- - phase_array : + phase_array : :np.array 2D: Array which lines correspond to the modelled phases in the input list. Notes @@ -294,33 +331,37 @@ def generate_phases(self, phases_parameters) : The absorption correction is done using the average composition of the phases. The same correction is used for each phase. """ self.phases = [] - unique_elts = dict(elts_dict_from_dict_list([x["elements_dict"] for x in phases_parameters])) + unique_elts = dict( + elts_dict_from_dict_list([x["elements_dict"] for x in phases_parameters]) + ) for p in phases_parameters: # p.update({"elements_dict" : unique_elts}) - self.phases.append(self.generate_spectrum(**p,abs_elts_dict = unique_elts)) + self.phases.append(self.generate_spectrum(**p, abs_elts_dict=unique_elts)) self.phases = np.array(self.phases) - self.phases /= self.phases.sum(axis = 1)[:,np.newaxis] + self.phases /= self.phases.sum(axis=1)[:, np.newaxis] @symbol_to_number_dict - def generate_spectrum(self, b0=0, b1 = 0, scale = 1.0,abs_elts_dict = {},*,elements_dict = {}): + def generate_spectrum( + self, b0=0, b1=0, scale=1.0, abs_elts_dict={}, *, elements_dict={} + ): r""" Generate a spectrum from bremsstrahlung parameters and a chemical composition. The modelling is done using the same formula as for the generate_g_matr function. Parameters ---------- - b0 : - :float: First bremsstrahlung parameter. - b1 : + b0 : + :float: First bremsstrahlung parameter. + b1 : :float: Second bremsstrahlung parameter. - scale : + scale : :float: Scale factor to apply to the bremsstrahlung. Feature to be removed, scaling should be done with b0 and b1. - abs_elts_dict : + abs_elts_dict : :dict: Dictionnary of elements and associated concentrations. This dictionnary is used to calculate the contribution of the absorption in the spectrum. - elements_dict : + elements_dict : :dict: Dictionnary of elements and associated concentrations describing the chemical composition of the modeled sample. Returns ------- - spectrum : + spectrum : :np.array 1D: Output edx spectrum corresponding to the input chemical composition. Examples @@ -344,80 +385,110 @@ def generate_spectrum(self, b0=0, b1 = 0, scale = 1.0,abs_elts_dict = {},*,eleme """ temp = np.zeros_like(self.x) for elt in elements_dict.keys(): - if self.lines : - energies, cs = read_lines_db(elt,self.db_dict) - else : - energies, cs = read_compact_db(elt,self.db_dict) + if self.lines: + energies, cs = read_lines_db(elt, self.db_dict) + else: + energies, cs = read_compact_db(elt, self.db_dict) for i, energy in enumerate(energies): if (energy > np.min(self.x)) and (energy < np.max(self.x)): - if type(self.params_dict["Det"]) == str : - D = det_efficiency_from_curve(energy,self.params_dict["Det"]) - else : - D = det_efficiency(energy,self.params_dict["Det"]) - A = absorption_correction(energy,**self.params_dict["Abs"],elements_dict = {elt : 1.0}) - + if type(self.params_dict["Det"]) == str: + D = det_efficiency_from_curve(energy, self.params_dict["Det"]) + else: + D = det_efficiency(energy, self.params_dict["Det"]) + A = absorption_correction( + energy, **self.params_dict["Abs"], elements_dict={elt: 1.0} + ) + width = self.width_slope * energy + self.width_intercept temp += ( - elements_dict[elt] - * cs[i] - * gaussian(self.x, energy, width / 2.3548) - )*A*D + ( + elements_dict[elt] + * cs[i] + * gaussian(self.x, energy, width / 2.3548) + ) + * A + * D + ) temp /= temp.sum() - if abs_elts_dict == {} : - temp += continuum_xrays(self.x,self.params_dict,b0,b1,self.E0,elements_dict=elements_dict) * scale - else : - temp += continuum_xrays(self.x,self.params_dict,b0,b1,self.E0,elements_dict=abs_elts_dict) * scale - + if abs_elts_dict == {}: + temp += ( + continuum_xrays( + self.x, + self.params_dict, + b0, + b1, + self.E0, + elements_dict=elements_dict, + ) + * scale + ) + else: + temp += ( + continuum_xrays( + self.x, + self.params_dict, + b0, + b1, + self.E0, + elements_dict=abs_elts_dict, + ) + * scale + ) + return temp - + @symbol_to_number_dict - def bremsstrahlung_only_tools(self,*,mass_thickness = 1.0, ranges = None,elements_dict = {}): - self.params_dict['Abs']['density'] = mass_thickness - self.params_dict['Abs']['thickness'] = 1.0 - brstlg_lines = G_bremsstrahlung(self.x,self.E0,self.params_dict,elements_dict=elements_dict) + def bremsstrahlung_only_tools( + self, *, mass_thickness=1.0, ranges=None, elements_dict={} + ): + self.params_dict["Abs"]["density"] = mass_thickness + self.params_dict["Abs"]["thickness"] = 1.0 + brstlg_lines = G_bremsstrahlung( + self.x, self.E0, self.params_dict, elements_dict=elements_dict + ) norms = np.sum(brstlg_lines, axis=0, keepdims=True) - normed_brstlg = brstlg_lines/norms - mask = self.carac_X_span(ranges) + normed_brstlg = brstlg_lines / norms + mask = self.carac_X_span(ranges) - filtered_brstlg = normed_brstlg[mask,:] + filtered_brstlg = normed_brstlg[mask, :] return filtered_brstlg, mask - def get_elements(self, include_ignored = True) : + def get_elements(self, include_ignored=True): ign = include_ignored # If include_ignored is True, it means that this generator will pass once to capture ignored elements # To automatically calculate bremsstrahlung windows it is important to include ignore elements # To estimate mass-thickness (see eds_spim.py), it is important to remove ignored elements for elt in self.model_elts: - if re.match(r'[0-9]*(_lo)',elt) : + if re.match(r"[0-9]*(_lo)", elt): pass - elif re.match(r'[0-9]*(_ign)[0-9]*', elt) and ign : - m = re.match(r'([0-9]*)(_ign)([0-9]*)', elt) + elif re.match(r"[0-9]*(_ign)[0-9]*", elt) and ign: + m = re.match(r"([0-9]*)(_ign)([0-9]*)", elt) yield m.group(1) ign = False - elif re.match(r'[0-9]*(_ign)[0-9]*', elt) and not(ign) : + elif re.match(r"[0-9]*(_ign)[0-9]*", elt) and not (ign): pass - elif re.match(r'[0-9]*(_hi)',elt) : - m = re.match(r'([0-9]*)(_hi)',elt) + elif re.match(r"[0-9]*(_hi)", elt): + m = re.match(r"([0-9]*)(_hi)", elt) yield m.group(1) - else : + else: yield elt - - def carac_X_span(self, ranges = None, include_ignored = True) : + + def carac_X_span(self, ranges=None, include_ignored=True): all_indices = [] - if ranges is not None : - for span in ranges : + if ranges is not None: + for span in ranges: indices = np.where((self.x > span[0]) & (self.x < span[1]))[0] all_indices.append(indices) - else : + else: for elt in self.get_elements(include_ignored=include_ignored): - if self.lines : - energies, cs = read_lines_db(elt,self.db_dict) - else : - energies, cs = read_compact_db(elt,self.db_dict) + if self.lines: + energies, cs = read_lines_db(elt, self.db_dict) + else: + energies, cs = read_compact_db(elt, self.db_dict) for energy in energies: width = self.width_slope * energy + self.width_intercept - span = [energy - 2*width, energy + 2*width] + span = [energy - 2 * width, energy + 2 * width] indices = np.where((self.x > span[0]) & (self.x < span[1]))[0] all_indices.append(indices) @@ -426,31 +497,37 @@ def carac_X_span(self, ranges = None, include_ignored = True) : # If we use manually selected windows, the mask is True in selected windows (i.e. bremsstrahlung) # If we automatically set windows, the mask is False in spectral areas where there are caracteristic X-rays - if self.ranges is None : + if self.ranges is None: mask = np.ones(self.G.shape[0], bool) mask[idx] = 0 - else : + else: mask = np.zeros(self.G.shape[0], bool) mask[idx] = 1 return mask - - def NMF_initialize_W(self, D) : - if self.G is None : - raise ValueError('The G matrix is identity, the W matrix cannot be initialized. Please use a np.array for G in the ESpM-NMF instead of the model object') + + def NMF_initialize_W(self, D): + if self.G is None: + raise ValueError( + "The G matrix is identity, the W matrix cannot be initialized. Please use a np.array for G in the ESpM-NMF instead of the model object" + ) if self.bkgd_in_G and self.custom_init: mask = self.carac_X_span(self.ranges) anti_mask = np.logical_not(mask) - - Wbrem = (np.linalg.lstsq(self.G[mask,-2:],D[mask,:],rcond = None)[0]).clip(min = 0) - Wcarac = (np.linalg.lstsq(self.G[anti_mask,:-2],D[anti_mask,:] ,rcond = None)[0]).clip(min = 0) + + Wbrem = ( + np.linalg.lstsq(self.G[mask, -2:], D[mask, :], rcond=None)[0] + ).clip(min=0) + Wcarac = ( + np.linalg.lstsq(self.G[anti_mask, :-2], D[anti_mask, :], rcond=None)[0] + ).clip(min=0) # filter = np.where(np.mean(self.G[:,:-2],axis=1)<(np.max(np.mean(self.G[:,:-2],axis=1))*0.001))[0] - W = np.vstack((Wcarac,Wbrem)) - else : - W = (np.linalg.lstsq(self.G,D,rcond = None)[0]).clip(min = 0) + W = np.vstack((Wcarac, Wbrem)) + else: + W = (np.linalg.lstsq(self.G, D, rcond=None)[0]).clip(min=0) return W - + def NMF_simplex(self): """ Produce the indices of the rows of W on which to perform the simplex constraint. @@ -459,42 +536,44 @@ def NMF_simplex(self): ind_list = [] # We skip the low energy lines for i, elt in enumerate(self.model_elts): - if re.match(r'[0-9]*(_lo)',elt) : - pass - elif re.match(r'[0-9]*(_ign)[0-9]*', elt) : + if re.match(r"[0-9]*(_lo)", elt) or re.match(r"[0-9]*(_ign)[0-9]*", elt): pass - else : + else: ind_list.append(i) # We skip the bremsstrahlung return ind_list - + def NMF_update(self, W=None): """ Update the G matrix with the new absorption correction. """ # We don't need to check whether G was correctyl initialized it should work anyway. - + if W is None: return self.G - if not(self.bkgd_in_G) : + if not (self.bkgd_in_G): return self.G - else : + else: new_brstlg = self.update_bremsstrahlung(W) new_G = self.G.copy() - new_G[:,-2:] = new_brstlg/self.norm[0][-2:] + new_G[:, -2:] = new_brstlg / self.norm[0][-2:] self.G = new_G return self.G - def update_bremsstrahlung(self, W) : + def update_bremsstrahlung(self, W): """ Update the bremsstrahlung part of the G matrix. This function is used for the NMF decomposition so that the absorption correction is updated in between each step. """ - if not(self.bkgd_in_G) : + if not (self.bkgd_in_G): raise AttributeError("The bremsstrahlung is not comprised in the model.") - + indices = self.NMF_simplex() - mean_compo = np.mean(W[indices,:],axis=1) - normed_compo = mean_compo/np.sum(mean_compo) - elements_dict = {key : normed_compo[i] for i,key in enumerate(self.get_elements(False))} - bremsstrahlung = G_bremsstrahlung(self.x,self.E0,self.params_dict,elements_dict=elements_dict) + mean_compo = np.mean(W[indices, :], axis=1) + normed_compo = mean_compo / np.sum(mean_compo) + elements_dict = { + key: normed_compo[i] for i, key in enumerate(self.get_elements(False)) + } + bremsstrahlung = G_bremsstrahlung( + self.x, self.E0, self.params_dict, elements_dict=elements_dict + ) return bremsstrahlung diff --git a/espm/models/generate_EDXS_phases.py b/espm/models/generate_EDXS_phases.py index 96ac208..334bc44 100644 --- a/espm/models/generate_EDXS_phases.py +++ b/espm/models/generate_EDXS_phases.py @@ -1,7 +1,9 @@ -import numpy as np -from espm.models import EDXS from copy import deepcopy + +import numpy as np + from espm.conf import DEFAULT_EDXS_PARAMS +from espm.models import EDXS # DEFAULT_ELTS = [{"b0" : 4.3298e-04 , "b1" : 6.7732e-02, "elements_dict" : {"8": 1.0, "12": 0.51, "14": 0.61, "13": 0.07, "20": 0.04, "62": 0.02, # "26": 0.028, "60": 0.002, "71": 0.003, "72": 0.003, "29": 0.02}}, @@ -10,7 +12,8 @@ # {"b0" : 5.3298e-04 , "b1" : 3.7732e-02, "elements_dict" : {"8": 1.0, "14": 0.12, "13": 0.18, "20": 0.47, # "62": 0.04, "26": 0.004, "60": 0.008, "72": 0.004, "29": 0.01}}] -def generate_brem_params (seed) : + +def generate_brem_params(seed): r""" Generate random parameters for the Bremsstrahlung model with a scaling factor that somewhat resembles the real data. @@ -26,11 +29,12 @@ def generate_brem_params (seed) : """ np.random.seed(seed) - b0 = float(np.random.rand(1)*1e-2) - b1 = float(np.random.rand(1)*1e-1) - return {"b0" : b0,"b1" : b1} + b0 = float(np.random.rand(1) * 1e-2) + b1 = float(np.random.rand(1) * 1e-1) + return {"b0": b0, "b1": b1} -def generate_elts_dict (seed, nb_elements = 3) : + +def generate_elts_dict(seed, nb_elements=3): r""" Generate a random dictionary of elements and their relative abundance. The elements are limited to the range 6-82. @@ -47,13 +51,14 @@ def generate_elts_dict (seed, nb_elements = 3) : Dictionary containing the elements and their relative abundance. """ np.random.seed(seed) - elts = np.random.choice(np.arange(6,82,dtype = int),nb_elements,replace=False) + elts = np.random.choice(np.arange(6, 82, dtype=int), nb_elements, replace=False) elts = [str(elt) for elt in elts] frac = np.random.rand(nb_elements) - elt_dict = dict(zip(elts,frac)) + elt_dict = dict(zip(elts, frac)) return elt_dict -def unique_elts (dict_list) : + +def unique_elts(dict_list): r""" Generate a list of unique chemical elements from a list of chemical elements dictionnaries. @@ -71,15 +76,16 @@ def unique_elts (dict_list) : -------- >>> unique_elts([{"elements_dict" : {"18" : 0.2, "8" : 0.5, "12" : 0.3}}, {"elements_dict" : {"8" : 0.5, "26" : 0.3, "14" : 0.2}}]) ['18', '8', '12', '26', '14'] - + """ full_elts_list = [] - for dict in dict_list : - for elt in dict["elements_dict"].keys() : + for dict in dict_list: + for elt in dict["elements_dict"].keys(): full_elts_list.append(elt) return list(set(full_elts_list)) -def generate_random_phases(n_phases = 3, seed = 0): + +def generate_random_phases(n_phases=3, seed=0): r""" Generate a list of phases of the EDXS model with random elemental compositions and Bremsstrahlung parameters. The model parameters are set to the default values. @@ -99,22 +105,25 @@ def generate_random_phases(n_phases = 3, seed = 0): def_pars = deepcopy(DEFAULT_EDXS_PARAMS) model = EDXS(**def_pars) np.random.seed(seed) - seed_list = np.random.choice(10000,size = n_phases,replace = False) - for s in seed_list : + seed_list = np.random.choice(10000, size=n_phases, replace=False) + for s in seed_list: temp = generate_brem_params(s) # temp["seed"] = s elt_dict = generate_elts_dict(s) temp["elements_dict"] = elt_dict temp["scale"] = 1.0 - dict_list.append(temp.copy()) + dict_list.append(temp.copy()) model.generate_phases(dict_list) - + return model.phases -def generate_modular_phases (elts_dicts = 3, brstlg_pars = None, scales = None, model_params = None, seed = 0) : + +def generate_modular_phases( + elts_dicts=3, brstlg_pars=None, scales=None, model_params=None, seed=0 +): r""" - Generate an array of phases of the EDXS model with set model parameters, elemental compositions and bremsstrahlung parameters. + Generate an array of phases of the EDXS model with set model parameters, elemental compositions and bremsstrahlung parameters. Parameters ---------- @@ -128,18 +137,18 @@ def generate_modular_phases (elts_dicts = 3, brstlg_pars = None, scales = None, Dictionary containing the model parameters. If the parameters are not set, they are set to the default values. See the config file for the default values. seed : int, optional Seed for the random number generator. The default is 0. - + Returns ------- np.array Array of phases of the EDXS model. - + Examples -------- .. plot:: :context: close-figs - + >>> from espm.models.generate_EDXS_phases import generate_modular_phases >>> import matplotlib.pyplot as plt >>> phases = generate_modular_phases(elts_dicts = [{"8": 0.5, "14": 0.2, "26": 0.3}, {"8" : 0.5, "23" : 0.3, "7" : 0.2}], @@ -149,37 +158,38 @@ def generate_modular_phases (elts_dicts = 3, brstlg_pars = None, scales = None, """ dict_list = [] - if type(elts_dicts) == int : + if type(elts_dicts) == int: n_phases = elts_dicts - else : + else: n_phases = len(elts_dicts) - if model_params is None : + if model_params is None: model_params = deepcopy(DEFAULT_EDXS_PARAMS) model = EDXS(**model_params) np.random.seed(seed) - seed_list = np.random.choice(10000,size = n_phases,replace = False) - for i,s in enumerate(seed_list) : - if brstlg_pars is None : + seed_list = np.random.choice(10000, size=n_phases, replace=False) + for i, s in enumerate(seed_list): + if brstlg_pars is None: temp = generate_brem_params(s) - else : + else: temp = brstlg_pars[i] - if type(elts_dicts) == int : + if type(elts_dicts) == int: elt_dict = generate_elts_dict(s) temp["elements_dict"] = elt_dict - else : + else: temp["elements_dict"] = elts_dicts[i] - if scales is None : + if scales is None: temp["scale"] = 1.0 - else : + else: temp["scale"] = scales[i] - dict_list.append(temp.copy()) + dict_list.append(temp.copy()) model.generate_phases(dict_list) return model.phases -# def G_from_phases (dict_list) : + +# def G_from_phases (dict_list) : # def_pars = deepcopy(DEFAULT_SYNTHETIC_DATA_DICT["model_parameters"]) # model = EDXS(**def_pars) # elts_list = unique_elts(dict_list) diff --git a/espm/tests/__init__.py b/espm/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/espm/tests/test_EDXS.py b/espm/tests/test_EDXS.py deleted file mode 100644 index f2cc72d..0000000 --- a/espm/tests/test_EDXS.py +++ /dev/null @@ -1,223 +0,0 @@ -from espm.models.absorption_edxs import det_efficiency, absorption_correction -from espm.models.generate_EDXS_phases import generate_elts_dict -import numpy as np -import espm.models.EDXS_function as ef -from espm.models import EDXS -from espm.models.EDXS_function import lifshin_bremsstrahlung, lifshin_bremsstrahlung_b0, lifshin_bremsstrahlung_b1 -import hyperspy as hs - -x = np.linspace(0.2,19,num = 2000) -det_dict = { - "detection" : { - "thickness" : 450e-3, - "density" : 2.5 - }, - "layer1" : { - "thickness" : 1e-6, - "density" : 2.3, - "atomic_fraction" : True, - "elements_dict" : {14 : 1, 8 : 2} - }, - "layer2" : { - "thickness" : 3e-6, - "density" : 2.7, - "elements_dict" : {"Al" : 1.0} - - } -} - -model_parameters = { - "e_offset" : 0.2, - "e_size" : 1900, - "e_scale" : 0.01, - "width_slope" : 0.01, - "width_intercept" : 0.065, - "db_name" : "default_xrays.json", - "E0" : 200, - "params_dict" : { - "Abs" : { - "thickness" : 200.0e-7, - "toa" : 35, - "density" : 3.12 - }, - "Det" : det_dict - } - } - -def create_data() : - a = np.random.rand(50,70,100) - s = hs.signals.Signal1D(a) - s.axes_manager[-1].offset = 0.2 - s.axes_manager[-1].scale = 0.1 - s.set_signal_type("EDS_espm") - s.set_microscope_parameters( beam_energy = 200, tilt_stage = 0.0, azimuth_angle = 0.0, elevation_angle = 35.0) - s.metadata.Acquisition_instrument.TEM.Stage.tilt_beta = 0.0 - s.set_analysis_parameters( thickness = 1e-6, density = 2.7) - s.set_elements(elements = ["Si","O","Fe","Ca"]) - return s - -def test_lifshin_bremsstrahlung () : - E0 = 200 - x = np.linspace(0.2,E0-0.1, num = 2000 ) - for i in range(10) : - b0 = np.random.rand() - b1 = np.random.rand() - y0 = lifshin_bremsstrahlung_b0(x,b0,E0) - y1 = lifshin_bremsstrahlung_b1(x,b1,E0) - y = lifshin_bremsstrahlung (x,b0,b1,E0) - - np.testing.assert_array_less(1e-30, y0) - np.testing.assert_array_less(1e-30, y1) - np.testing.assert_array_less(1e-30, y) - np.testing.assert_allclose(y, y1 + y0) - -def test_efficiency () : - det = det_efficiency(x,det_dict) - np.testing.assert_array_less(1e-30,det) - np.testing.assert_array_less(det,1.0) - -def test_absorption () : - elts = generate_elts_dict(42) - abs_corr = absorption_correction(x,thickness = 1e-4, toa = 35, density = 5, elements_dict=elts) - - np.testing.assert_array_less(1e-30,abs_corr) - np.testing.assert_array_less(abs_corr,1.0) - -def test_elts_dict_from_dict_list () : - dict_list = [{"chou" : 1, "carottes" : 2, "navet" : 3}, {"chou" : 2, "oignons" : 3, "navet" : 3}, {"orange" : 6, "citron" : 4, "poireau" : 8}] - unique_dict = ef.elts_dict_from_dict_list(dict_list) - sum_elts = 32 - assert unique_dict == {"chou" : 3/sum_elts, "carottes" : 2/sum_elts, "navet" : 6/sum_elts, "orange" : 6/sum_elts, "citron" : 4/sum_elts, "poireau" : 8/sum_elts, "oignons" : 3/sum_elts} - -def test_continuum_xrays () : - E0 = 200 - b0 = np.random.uniform(0.0,50.0) - b1 = np.random.uniform(0.0, 50.0) - cont_x = ef.continuum_xrays(x,model_parameters["params_dict"],b0= b0, b1 = b1, E0 = E0,elements_dict = {"Si" : 1.0, "C" : 0.2, "Bi" : 3.2}) - - np.testing.assert_array_less(1e-30,cont_x) - -def test_get_elements () : - s = create_data() - s.build_G(elements_dict ={'Fe' : 3.0}) - nelts = [] - for i in s.model.get_elements(False) : - nelts.append(i) - assert nelts == ['20', '26', '8', '14'] - nelts = [] - for i in s.model.get_elements(True) : - nelts.append(i) - print(nelts) - assert nelts == ['20', '26', '8', '14', '29'] - -def test_carac_x_span () : - # TODO : Find a way to implement the test of this function - pass - -def test_bremsstrahlung_only_tools () : - # TODO : Find a way to implement the test of this function - pass - -def test_NMF_initialize_W () : - model = EDXS(**model_parameters) - model.generate_g_matr(g_type = "bremsstrahlung", elements = ["Na", "Sr", "Ge", "Nb"], elements_dict={}, ignored_elements=[]) - D = np.random.rand(1900,2) - W = model.NMF_initialize_W(D) - assert W.shape == (6,2) - - model = EDXS(**model_parameters) - model.generate_g_matr(g_type = "no_brstlg", elements = ["Na", "Sr", "Ge", "Nb"], elements_dict={'Nb' : 3.0}, ignored_elements=[]) - D = np.random.rand(1900,4) - W = model.NMF_initialize_W(D) - assert W.shape == (5,4) - - model = EDXS(**model_parameters) - model.generate_g_matr(g_type = "identity", elements = ["Na", "Sr", "Ge", "Nb"], elements_dict={}, ignored_elements=[]) - with np.testing.assert_raises(ValueError) : - D = np.random.rand(1900,4) - W = model.NMF_initialize_W(D) - - model = EDXS(**model_parameters) - model.generate_g_matr(g_type = "bremsstrahlung", elements = ["Na", "Ge", "Nb"], elements_dict={'Nb' : 3.0}, ignored_elements=["Sr"]) - D = np.random.rand(1900,2) - W = model.NMF_initialize_W(D) - assert W.shape == (23,2) - -def test_NMF_simplex () : - model = EDXS(**model_parameters) - model.generate_g_matr(g_type = "bremsstrahlung", elements = ["Na", "Sr", "Ge", "Nb"], elements_dict={}) - inds = model.NMF_simplex() - assert inds == [0,1,2,3] - - model = EDXS(**model_parameters) - model.generate_g_matr(g_type = "no_brstlg", elements = ["Na", "Sr", "Ge", "Nb"], elements_dict={'Nb' : 3.0}) - inds = model.NMF_simplex() - assert inds == [0,1,2,4] - - model = EDXS(**model_parameters) - model.generate_g_matr(g_type = "bremsstrahlung", elements = ["Sr", "Ge", "Nb"], elements_dict={'Nb' : 3.0}, ignored_elements=["Na"]) - inds = model.NMF_simplex() - assert inds == [0,1,3] - -def test_NMF_update () : - model = EDXS(**model_parameters) - model.generate_g_matr(g_type = "no_brstlg", elements = ["Na", "Sr", "Ge", "Nb"], elements_dict={}) - temp_G = model.G.copy() - np.testing.assert_array_equal(model.NMF_update(), temp_G) - np.testing.assert_array_equal(model.NMF_update(np.random.rand(10,10)), temp_G) - - model = EDXS(**model_parameters) - model.generate_g_matr(g_type = "bremsstrahlung", elements = ["Na", "Sr", "Ge", "Nb"], elements_dict={}) - temp_G = model.G.copy() - np.testing.assert_array_equal(model.NMF_update(), temp_G) - assert model.NMF_update(np.random.rand(6,10)).shape == model.G.shape - with np.testing.assert_raises(AssertionError): - np.testing.assert_array_equal(model.NMF_update(np.random.rand(6,10))[:,-2:], temp_G[:,-2:]) - -def test_generate_g_matr () : - model1 = EDXS(**model_parameters) - model2 = EDXS(**model_parameters) - model3 = EDXS(**model_parameters) - model4 = EDXS(**model_parameters) - model5 = EDXS(**model_parameters) - size = model_parameters["e_size"] - elts_list = ["Na", "Sr", "Ge", "Nb"] - model1.generate_g_matr(g_type = "bremsstrahlung", elements = elts_list, elements_dict={}, ignored_elements=[]) - G_brem = model1.G - model2.generate_g_matr(g_type = "identity", elements = elts_list, elements_dict={}, ignored_elements=[]) - G_id = model2.G - model3.generate_g_matr(g_type = "no_brstlg", elements = elts_list, elements_dict={},ignored_elements=[]) - G_no_brstlg = model3.G - model4.generate_g_matr(g_type = "bremsstrahlung", elements = elts_list, elements_dict={'Ge' : 3.0}, ignored_elements=[]) - G_brem_ge = model4.G - model5.generate_g_matr(g_type = "bremsstrahlung", elements = ['Na', 'Ge', 'Nb'], elements_dict={'Ge' : 3.0}, ignored_elements=['Sr']) - G_brem_ge_ign = model5.G - - assert G_brem.shape == (size, 6) - assert G_no_brstlg.shape == (size, 4) - assert G_brem_ge.shape == (size, 7) - assert G_brem_ge_ign.shape == (size, 23) - - assert G_id is None - np.testing.assert_allclose(G_brem[:,:-2], G_no_brstlg) - np.testing.assert_allclose((G_brem_ge[:,3] + G_brem_ge[:,2])*model4.norm[0,0], G_brem[:,2]*model1.norm[0,0]) - - # np.testing.assert_array_less(-1e-30, G_id) - np.testing.assert_array_less(-1e-30, G_brem) - np.testing.assert_array_less(-1e-30, G_no_brstlg) - assert(model1.model_elts == ['11', '38', '32', '41']) - assert(model2.model_elts == []) - assert(model3.model_elts == ['11', '38', '32', '41']) - assert(model4.model_elts == ['11', '38', '32_lo', '32_hi', '41']) - assert(model5.model_elts == ['11', '32_lo', '32_hi', '41', '38_ign0', '38_ign1', '38_ign2', '38_ign3', '38_ign4', '38_ign5', '38_ign6', '38_ign7', '38_ign8', '38_ign9', '38_ign10', '38_ign11', '38_ign12', '38_ign13', '38_ign14', '38_ign15', '38_ign16']) - -def test_G_bremsstrahlung() : - model = EDXS(**model_parameters) - size = model_parameters["e_size"] - elts_dict = {"V" : 0.3, "Ti" : 0.1, "Hf" : 0.05, "Pb" : 0.55 } - B = ef.G_bremsstrahlung(model.x, model.E0, model.params_dict, elements_dict=elts_dict) - - assert B.shape == (size,2) - np.testing.assert_array_less(-1e-30,B) - - diff --git a/espm/tests/test_datasets.py b/espm/tests/test_datasets.py deleted file mode 100644 index dea56a8..0000000 --- a/espm/tests/test_datasets.py +++ /dev/null @@ -1,440 +0,0 @@ -from espm.datasets.eds_spim import get_metadata -import numpy as np -from espm.datasets.base import generate_dataset, generate_spim_sample, sample_to_EDSespm, generate_spim -from espm.models import EDXS -from espm.models.generate_EDXS_phases import generate_brem_params, generate_random_phases, unique_elts -import os -import hyperspy.api as hs -import shutil -from espm.conf import DATASETS_PATH -from pathlib import Path -from exspy.utils.eds import take_off_angle -import espm.weights.generate_weights as wts -from espm.weights.generate_weights import generate_weights -from espm.models.EDXS_function import elts_list_from_dict_list -from espm.models.generate_EDXS_phases import generate_modular_phases -from espm.estimators import SmoothNMF - -elts_dicts = [{"Fe" : 0.54860348, - "Pt" : 0.38286879, - "Mo" : 0.03166235, - "O" : 0.03686538}, - {"Ca" : 0.54860348, - "Si" : 0.38286879, - "O" : 0.15166235}, - {"Cu" : 0.34, - "Mo" : 0.12, - "Au" : 0.54}] - -brstlg_pars = [{"b0" : 5e-3, - "b1" : 3e-2}, - {"b0" : 7e-3, - "b1" : 5e-2}, - {"b0" : 3e-3, - "b1" : 5e-2}] - -scales = [0.05, 0.05, 0.05] - -model_params = {"e_offset" : 0.2, - "e_size" : 1900, - "e_scale" : 0.01, - "width_slope" : 0.01, - "width_intercept" : 0.065, - "db_name" : "200keV_xrays.json", - "E0" : 200, - "params_dict" : { - "Abs" : { - "thickness" : 100.0e-7, - "toa" : 35, - "density" : 5 - }, - "Det" : "SDD_efficiency.txt" - }} - -misc_params = {"N" : 400, - "densities" : [1.3,1.6,1.9], - "data_folder" : "test_gen_data", - "seed" : 42, - "shape_2d" : (100,120), - "model" : "EDXS"} - -elements = elts_list_from_dict_list(elts_dicts) - -def test_generate(): - - # Generate the phases - model = EDXS(**model_params) - model.generate_g_matr(g_type = "bremsstrahlung", elements=elements, elements_dict = {}) - phases1 = generate_modular_phases(elts_dicts = elts_dicts, brstlg_pars = brstlg_pars, scales = scales, model_params = model_params) - G = model.G - n_phases = len(elts_dicts) - maps = generate_weights(weight_type='sphere', shape_2d= misc_params["shape_2d"], n_phases=n_phases, seed=misc_params["seed"], radius = 15) - densities = np.array(misc_params["densities"]) - spim_sample = generate_spim_sample(phases1,maps,model_params=model_params, misc_params=misc_params) - spim = spim_sample["X"] - cont_spim = spim_sample["Xdot"] - Xdot = misc_params["N"]* maps @ np.diag(densities)@ phases1 - W = np.abs(np.linalg.lstsq(G,spim.sum(axis = (0,1)),rcond = None)[0]) - - assert phases1.shape == (3, 1900) - assert maps.shape == (100,120,3) - assert spim.shape == (100,120,1900) - np.testing.assert_allclose(np.sum(phases1, axis=1), np.ones([3])) - np.testing.assert_allclose( Xdot, cont_spim) - np.testing.assert_allclose( Xdot.sum(axis=(0,1)), G@W, rtol = 0.1 ) - - if os.path.exists("test.hspy"): - os.remove("test.hspy") - filename = "test.hspy" - hspy_spim = sample_to_EDSespm(spim_sample,elements=elements) - hspy_spim.save(filename) - si = hs.load(filename) - assert si.metadata.Signal.signal_type == "EDS_espm_Simulated" - si.build_G(problem_type = "bremsstrahlung") - G = si.G - phases, maps = si.phases, si.maps_2d - # weights = weights.reshape((100,120,n_phases)) - X = si.data - W = np.linalg.lstsq(G,X.sum(axis = (0,1)),rcond = None)[0] - - assert phases.shape == (1900, 3) - assert maps.shape == (100,120,3) - assert si.data.shape == (100,120,1900) - np.testing.assert_allclose( Xdot, maps @ phases.T) - np.testing.assert_allclose( Xdot.sum(axis=(0,1)), G@W, rtol = 0.2 ) - - os.remove(filename) - - if os.path.exists(str(DATASETS_PATH / Path(misc_params["data_folder"]))): - shutil.rmtree(str(DATASETS_PATH / Path(misc_params["data_folder"]))) - - generate_dataset(base_seed=misc_params['seed'], - sample_number=2, - model_params = model_params, - misc_params = misc_params, - phases = phases1, - weights = maps, - elements = elements) - gen_folder = DATASETS_PATH / Path(misc_params["data_folder"]) - gen_si = hs.load(gen_folder / Path("sample_0.hspy")) - - np.testing.assert_allclose(X,gen_si.data) - - shutil.rmtree(str(gen_folder)) - -def test_generate_spim(): - - k = 3 - ell = 7 - shape_2d = [4,5] - pd = 0.3 - seed = 0 - - phases = np.random.rand(k, ell) - weights = np.random.rand(*shape_2d, k) - densities = 1 - pd + 2*pd*np.random.rand(k) - - # Ns = np.linspace(10, 200, 30) - # vals = [] - # vals2 = [] - - # for N in Ns: - # Xdot = generate_spim(phases, weights, densities, N, seed=seed,continuous = True)/N - # X = generate_spim(phases, weights, densities, N, seed=seed,continuous = False)/N - # X2 = np.random.poisson(N * Xdot) / N - - # vals.append(np.mean(np.abs(X - Xdot))) - # vals2.append(np.mean(np.abs(X2 - Xdot))) - - # plt.plot(Ns, vals) - # plt.plot(Ns, vals2) - N = 10000 - X3 = np.zeros([*shape_2d, ell]) - for k, w in enumerate(densities): - # generating the spectroscopic events - for i in range(shape_2d[0]): - for j in range(shape_2d[1]): - # Draw a local_N based on the local density - local_N = np.random.poisson(N * w * weights[i, j, k]) - # draw local_N events from the ideal spectrum - counts = np.random.choice( - ell, local_N, p=phases[k]/np.sum(phases[k]) - ) - # Generate the spectrum based on the drawn events - hist = np.bincount(counts, minlength=ell) - X3[i, j] += hist - - - Xdot = generate_spim(phases, weights, densities, N, seed=seed,continuous = True)/N - X = generate_spim(phases, weights, densities, N, seed=seed,continuous = False)/N - # X2 = np.random.poisson(N * Xdot) / N - - X3 /= N - assert ( np.mean(np.abs(X - Xdot)) < 0.005) - assert ( np.mean(np.abs(X3 - Xdot)) < 0.005) - assert ( np.mean(np.abs(X - X3)) < 0.006) - - - -def test_generate_random_weights(): - shape_2d = [28, 36] - n_phases = 5 - - w = wts.random_weights(shape_2d=shape_2d, n_phases=n_phases) - - assert(w.shape == (*shape_2d, n_phases)) - np.testing.assert_array_less(-1e-30, w) - np.testing.assert_array_almost_equal(np.sum(w, axis=2), 1) - -def test_generate_laplacian_weights(): - shape_2d = [28, 36] - n_phases = 5 - - w = wts.laplacian_weights(shape_2d=shape_2d, n_phases=n_phases) - - assert(w.shape == (*shape_2d, n_phases)) - np.testing.assert_array_less(-1e-30, w) - np.testing.assert_array_almost_equal(np.sum(w, axis=2), 1) - -def test_generate_two_sphere(): - shape_2d = [80, 80] - n_phases = 3 - radius = 2 - - w = wts.spheres_weights(shape_2d=shape_2d, n_phases=n_phases, radius= radius) - - assert(w.shape == (80, 80, 3)) - np.testing.assert_array_less(-1e-30, w) - np.testing.assert_array_almost_equal(np.sum(w, axis=2), 1) - -def test_generate_gaussian_ripple() : - shape_2d = [100,40] - width = 10 - - w = wts.gaussian_ripple_weights(shape_2d, width = width) - - assert(w.shape == (100,40,2)) - np.testing.assert_array_less(-1e-30,w) - np.testing.assert_array_almost_equal(np.sum(w,axis = 2), 1) - -# def test_gen_EDXS () : - -# b_dict = generate_brem_params(42) -# assert b_dict["b0"] <= 1.0 -# assert b_dict["b1"] <= 1.0 - -# phases, dicts = generate_random_phases(n_phases=3,seed = 42) -# np.testing.assert_array_less(-1e-30, phases) -# model = EDXS(**DEFAULT_SYNTHETIC_DATA_DICT["model_parameters"]) -# model.generate_phases(dicts) -# np.testing.assert_almost_equal(model.phases,phases) - -# unique_list = unique_elts(dicts) -# assert len(unique_list) == len(set(unique_list)) - -def test_decomposition () : - if os.path.exists(str(DATASETS_PATH / Path(misc_params["data_folder"]))): - shutil.rmtree(str(DATASETS_PATH / Path(misc_params["data_folder"]))) - - phases1 = generate_modular_phases(elts_dicts = elts_dicts, brstlg_pars = brstlg_pars, scales = scales, model_params = model_params) - maps = generate_weights(weight_type='sphere', shape_2d= misc_params["shape_2d"], n_phases=len(elts_dicts), seed=misc_params["seed"], radius = 15) - generate_dataset(base_seed=misc_params['seed'], - sample_number=2, - model_params = model_params, - misc_params = misc_params, - phases = phases1, - weights = maps, - elements = elements) - - gen_folder = DATASETS_PATH / Path(misc_params["data_folder"]) - gen_si = hs.load(gen_folder / Path("sample_0.hspy")) - gen_si.change_dtype("float64") - gen_si.build_G() - est = SmoothNMF(n_components=3, G = gen_si.model, tol = 1e-7, hspy_comp = True) - gen_si.decomposition(algorithm = est) - assert gen_si.metadata.Signal.signal_type == "EDS_espm_Simulated" - np.testing.assert_allclose((est.G_@est.W_@est.H_).sum(axis = 1), gen_si.X.sum(axis = 1), rtol = 0.5) - indices = gen_si.model.NMF_simplex() - np.testing.assert_allclose(est.W_[indices,:].sum(axis = 0), np.ones(3), rtol = 0.1) - - shutil.rmtree(str(gen_folder)) - -def test_spim () : - - if os.path.exists(str(DATASETS_PATH / Path(misc_params["data_folder"]))): - shutil.rmtree(str(DATASETS_PATH / Path(misc_params["data_folder"]))) - phases1 = generate_modular_phases(elts_dicts = elts_dicts, brstlg_pars = brstlg_pars, scales = scales, model_params = model_params) - maps = generate_weights(weight_type='sphere', shape_2d= misc_params["shape_2d"], n_phases=len(elts_dicts), seed=misc_params["seed"], radius = 15) - generate_dataset(base_seed=misc_params['seed'], - sample_number=2, - model_params = model_params, - misc_params = misc_params, - phases = phases1, - weights = maps, - elements = elements) - gen_folder = DATASETS_PATH / Path(misc_params["data_folder"]) - gen_si = hs.load(gen_folder / Path("sample_0.hspy")) - - assert gen_si.metadata.Signal.signal_type == "EDS_espm_Simulated" - - mod_pars = get_metadata(gen_si) - # mod_pars["params_dict"]["Abs"]["atomic_fraction"] = False - - assert model_params == mod_pars - - shape = gen_si.shape_2d - gen_si = hs.load(gen_folder / Path("sample_0.hspy")) - gen_si.build_G(problem_type = "identity") - assert shape == (100,120) - assert gen_si.G is None - - gen_si = hs.load(gen_folder / Path("sample_0.hspy")) - gen_si.build_G(problem_type = "no_brstlg",elements_dict = {}, ignored_elements = []) - assert gen_si.G.shape == (1900, 8) - - gen_si = hs.load(gen_folder / Path("sample_0.hspy")) - gen_si.build_G(problem_type = "bremsstrahlung",elements_dict = {}, ignored_elements = []) - assert gen_si.G.shape == (1900, 10) - - Xflat = gen_si.X - assert Xflat.shape == (1900, 120*100) - - gen_si = hs.load(gen_folder / Path("sample_0.hspy")) - gen_si.build_G(problem_type = "bremsstrahlung",elements_dict = {"26" : 3.0}, ignored_elements = []) - assert gen_si.G.shape == (1900, 11) - - gen_si = hs.load(gen_folder / Path("sample_0.hspy")) - gen_si.build_G(problem_type = "no_brstlg",elements_dict = {"26" : 3.0}, ignored_elements = []) - assert gen_si.G.shape == (1900, 9) - - gen_si = hs.load(gen_folder / Path("sample_0.hspy")) - gen_si.build_G(problem_type = "bremsstrahlung",elements_dict = {"26" : 3.0}, ignored_elements = ['Cu']) - assert gen_si.G.shape == (1900, 22) - - detector_dict = { - "detection" : { - "thickness" : 45, - "elements_dict" : { - "Si" : 3, - "Se" : 4 - } - }, - "layer1" : { - "thickness" : 34, - "elements_dict" : { - "Ge" : 1, - "O" : 3 - } - } - } - - gen_si.set_microscope_parameters(beam_energy = 100, azimuth_angle = 2.0, elevation_angle = 3.0, tilt_stage = 4.0 ) - gen_si.set_elements(elements = ["Si"]) - gen_si.metadata.Acquisition_instrument.TEM.Stage.tilt_beta = 0.0 - - gen_si.set_analysis_parameters(thickness = 500e-7, density = 4, detector_type = detector_dict, width_slope = 0.3, width_intercept = 65.0, xray_db = "100keV_xrays.json") - - assert gen_si.metadata.Sample.thickness == 500e-7 - assert gen_si.metadata.Sample.density == 4 - assert gen_si.metadata.Sample.elements == ["Si"] - assert gen_si.metadata.xray_db == "100keV_xrays.json" - assert gen_si.metadata.Acquisition_instrument.TEM.Stage.tilt_alpha == 4.0 - assert gen_si.metadata.Acquisition_instrument.TEM.beam_energy == 100 - assert gen_si.metadata.Acquisition_instrument.TEM.Detector.EDS.take_off_angle == take_off_angle(4.0,2.0,3.0) - assert gen_si.metadata.Acquisition_instrument.TEM.Detector.EDS.type.as_dictionary() == detector_dict - assert gen_si.metadata.Acquisition_instrument.TEM.Detector.EDS.width_slope == 0.3 - assert gen_si.metadata.Acquisition_instrument.TEM.Detector.EDS.width_intercept == 65.0 - - shutil.rmtree(str(gen_folder)) - - -def test_carto_fixed_W() : - - def create_data() : - a = np.random.rand(50,70,100) - s = hs.signals.Signal1D(a) - s.axes_manager[-1].offset = 0.2 - s.axes_manager[-1].scale = 0.1 - s.set_signal_type("EDS_espm") - s.add_elements(elements = ["Si","O","Fe","Ca"]) - s.set_microscope_parameters(beam_energy = 100, azimuth_angle = 2.0, elevation_angle = 3.0, tilt_stage = 4.0 ) - s.metadata.Acquisition_instrument.TEM.Stage.tilt_beta = 0.0 - s.set_analysis_parameters(thickness = 500e-7, density = 4, detector_type = 'SDD_efficiency.txt', width_slope = 0.3, width_intercept = 65.0, xray_db = "100keV_xrays.json") - return s - - s = create_data() - s.build_G(problem_type = "bremsstrahlung",ignored_elements = []) - fw1 = s.carto_fixed_W(brstlg_comps = 2) - tw1_1 = np.diag(-1* np.ones(4)) - tw1_2 = np.zeros((2,4)) - tw1_prime = np.vstack((tw1_1,tw1_2)) - tw1_3 = np.zeros((4,2)) - tw1_4 = -1*np.ones((2,2)) - tw1_second = np.vstack((tw1_3,tw1_4)) - tw1 = np.hstack((tw1_prime,tw1_second)) - - assert(fw1.shape == (6,6)) - np.testing.assert_array_equal(fw1,tw1) - - s = create_data() - s.build_G(problem_type = "no_brstlg", ignored_elements = []) - fw2 = s.carto_fixed_W() - tw2 = np.diag(-1* np.ones(4)) - - assert(fw2.shape == (4,4)) - np.testing.assert_array_equal(fw2,tw2) - - s = create_data() - s.build_G(problem_type = "no_brstlg", elements_dict = {"26" : 3.0}, ignored_elements = []) - fw3 = s.carto_fixed_W() - tw3 = np.diag(-1* np.ones(5)) - - assert(fw3.shape == (5,5)) - np.testing.assert_array_equal(fw3,tw3) - -def test_set_fixed_W() : - def create_data() : - a = np.random.rand(50,70,100) - s = hs.signals.Signal1D(a) - s.axes_manager[-1].offset = 0.2 - s.axes_manager[-1].scale = 0.1 - s.set_signal_type("EDS_espm") - s.add_elements(elements = ["Si","O","Fe","Ca"]) - s.set_microscope_parameters(beam_energy = 100, azimuth_angle = 2.0, elevation_angle = 3.0, tilt_stage = 4.0 ) - s.metadata.Acquisition_instrument.TEM.Stage.tilt_beta = 0.0 - s.set_analysis_parameters(thickness = 500e-7, density = 4, detector_type = 'SDD_efficiency.txt', width_slope = 0.3, width_intercept = 65.0, xray_db = "100keV_xrays.json") - return s - - s = create_data() - s.build_G(problem_type = "bremsstrahlung", ignored_elements = []) - fw1 = s.set_fixed_W({'p0' : {"Si" : 0.0, 'O' : 0.5, 'b1' : 10.0},'p1' : {}, 'p2' : {'O' : 0.4, 'Fe' : 0.6, 'b0' : 3.0} }) - tw1 = (np.array([[-1.0, -1.0, 0.5, 0.0, -1.0, 10.0], - [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0], - [-1.0, 0.6, 0.4, -1.0, 3.0, -1.0]])).T - assert(fw1.shape == (6,3)) - np.testing.assert_array_equal(fw1,tw1) - - s = create_data() - s.build_G(problem_type = "no_brstlg", elements_dict = {"Fe" : 3.0}, ignored_elements = []) - fw2 = s.set_fixed_W({'p0' : {"Si" : 0.0, 'O' : 0.5},'p1' : {}, 'p2' : {'O' : 0.4, 'Fe' : 0.6} }) - tw2 = (np.array([[-1.0, -1.0, -1.0, 0.5,0.0], - [-1.0, -1.0, -1.0, -1.0, -1.0], - [-1.0, -1.0, 0.6, 0.4, -1.0]])).T - - assert(fw2.shape == (5,3)) - np.testing.assert_array_equal(fw2,tw2) - - s = create_data() - s.build_G(problem_type = "bremsstrahlung", ignored_elements = ['Cu']) - fw3 = s.set_fixed_W({'p0' : {"Si" : 0.0, 'O' : 0.5, 'b1' : 10.0},'p1' : {}, 'p2' : {'O' : 0.4, 'Fe' : 0.6, 'b0' : 3.0} }) - tw3 = (np.array([[-1.0,-1.0, 0.5, 0.0, -1.0, -1.0, -1.0, -1.0,-1.0, -1.0,-1.0, -1.0,-1.0, -1.0,-1.0, -1.0, 10.0], - [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0,-1.0, -1.0,-1.0, -1.0,-1.0, -1.0,-1.0, -1.0,-1.0, -1.0, -1.0], - [-1.0, 0.6, 0.4, -1.0, -1.0, -1.0, -1.0, -1.0,-1.0, -1.0,-1.0, -1.0,-1.0, -1.0,-1.0, 3.0, -1.0]])).T - assert(fw1.shape == (6,3)) - np.testing.assert_array_equal(fw3,tw3) - -def test_estimate_best_binning () : - pass - # TODO : Implement this test - - diff --git a/espm/tests/test_estimators.py b/espm/tests/test_estimators.py deleted file mode 100644 index 4ae0190..0000000 --- a/espm/tests/test_estimators.py +++ /dev/null @@ -1,302 +0,0 @@ -from sklearn.utils.estimator_checks import check_estimator -from espm.estimators.surrogates import diff_surrogate, smooth_l2_surrogate, smooth_dgkl_surrogate -from espm.estimators import SmoothNMF -from espm.estimators.base import normalization_factor -import numpy as np -from espm.models import EDXS -from espm.weights import generate_weights -from espm.datasets.base import generate_spim -from espm.measures import trace_xtLx -from espm.utils import create_laplacian_matrix -from espm.models.generate_EDXS_phases import generate_modular_phases -from espm.datasets.base import generate_spim_sample - - -phases_dict = { - 'elts_dicts' : [ - {"Fe" : 0.54860348, - "Pt" : 0.38286879, - "Mo" : 0.03166235}, - {"Ca" : 0.54860348, - "Si" : 0.38286879, - "O" : 0.03166235} - ], - 'brstlg_pars' : [ - {'b0': 5e-5, - 'b1': 3e-4}, - {'b0': 7e-5, - 'b1': 5e-4} - ], - 'scales' : [0.05,0.05], - 'model_params': {'e_offset': 0.2, - 'e_size': 2000, - 'e_scale': 0.01, - 'width_slope': 0.01, - 'width_intercept': 0.065, - 'db_name': '200keV_xrays.json', - 'E0': 200, - 'params_dict': {'Abs': {'thickness': 1e-5, - 'toa': 35, - 'density': 5, - 'atomic_fraction': False}, - 'Det': 'SDD_efficiency.txt'}} - } - -misc_dict = { - 'N': 100, - 'seed' : 42, - 'data_folder': 'built_in_particules', - 'shape_2d': [10, 20], - 'model': 'EDXS', - 'densities': [1.3, 1.6] -} - -def generate_one_sample(): - # Generate the phases - model = EDXS(**phases_dict["model_params"]) - phases = generate_modular_phases(**phases_dict) - model.generate_g_matr(g_type="bremsstrahlung", elements=["Fe", "Mo", "Ca", "Si", "O", "Pt"] ,elements_dict={}, ignored_elements=[]) - G = model.G - - weights = generate_weights.generate_weights("laplacian", misc_dict["shape_2d"], n_phases=len(phases_dict["elts_dicts"]), seed=misc_dict["seed"], size_x = 10, size_y = 10) - - sample = generate_spim_sample(phases, weights, phases_dict["model_params"],misc_dict, seed = 42,g_params = {}) - - X = sample["X"].reshape(-1, sample["X"].shape[-1]).T - X_cont = sample['Xdot'].reshape(-1, sample['Xdot'].shape[-1]).T - - D = sample['GW'].T - H = sample["H_flat"].T - - W = np.abs(np.linalg.lstsq(G, D, rcond=None)[0]) - for i in range(10) : - G = model.NMF_update(W) - W = np.abs(np.linalg.lstsq(G, D, rcond=None)[0]) - - - w = np.array(misc_dict["densities"]) - N = misc_dict["N"] - - return G, W, H, D, w, X, X_cont, N - -def gen_fixed_mat () : - fixed_W = -1*np.ones((8,2)) - fixed_W[0,0] = 0.0 - fixed_W[1,0] = 0.0 - - fixed_H = -1*np.ones((2,200)) - fixed_H[0,0:20] = 1.0 - fixed_H[1,0:20] = 0.0 - - return fixed_W, fixed_H - - -def test_generate_one_sample(): - G, W, H, D, w, X, Xdot, N = generate_one_sample() - np.testing.assert_allclose(G @ W , D, atol=1e-3) - np.testing.assert_allclose( D @ H , Xdot) - np.testing.assert_allclose(G @ W @ H , Xdot, atol=1e-1) - -def test_NMF_scikit () : - estimator = SmoothNMF(n_components= 5,max_iter=200, simplex_W=False, simplex_H = True,mu = 1.0, epsilon_reg = 1.0,hspy_comp = False) - check_estimator(estimator) - estimator = SmoothNMF( n_components= 5,lambda_L=2,max_iter=200, simplex_W=False, simplex_H = True,mu = 1.0, epsilon_reg = 1.0,hspy_comp = False) - check_estimator(estimator) - -def test_general(): - G, W, H, D, w, X, Xdot, N = generate_one_sample() - - # Check if we can recover D from H and Xdot - estimator = SmoothNMF(G=G,n_components= 2,max_iter=200, simplex_W=False, simplex_H=True, mu = 0, epsilon_reg = 1, hspy_comp = False) - D2 = estimator.fit_transform(H=H, X=Xdot) - np.testing.assert_allclose(D, D2, atol=6e-1) - - # Same without G - estimator = SmoothNMF(n_components= 2,max_iter=200, simplex_W=False, simplex_H=True, mu = 0, epsilon_reg = 1, hspy_comp = False) - D2 = estimator.fit_transform(H=H, X=Xdot) - np.testing.assert_allclose(D, D2, atol=6e-2) - - # Check if we can recover D, H from W and Xdot - estimator = SmoothNMF(G=G,n_components= 2,max_iter=200,simplex_W=False, simplex_H = True, mu = 0, epsilon_reg = 1, hspy_comp = False) - D2 = estimator.fit_transform( W=W, X=Xdot) - H2 = estimator.H_ - np.testing.assert_allclose(D, D2, atol=3e-2) - np.testing.assert_allclose(H, H2, atol=3e-2) - - estimator = SmoothNMF(G =G, n_components= 2,max_iter=200, simplex_W=False, simplex_H=True, mu = 0, epsilon_reg = 1, hspy_comp = False) - D2 = estimator.fit_transform(W=W, X=Xdot) - np.testing.assert_allclose(D, D2, atol=3e-1) - - estimator = SmoothNMF(G=G, n_components= 2,max_iter=200, simplex_W=False, simplex_H = True,mu = 0, epsilon_reg = 1, hspy_comp = False, tol = 1.0e-6) - estimator.fit_transform(X=Xdot) - P2, A2 = estimator.W_, estimator.H_ - np.testing.assert_allclose(G @ P2 @ A2, Xdot, atol=1) - - estimator = SmoothNMF(G=G, n_components= 2,max_iter=200, simplex_W=False, simplex_H = True,mu = 0, epsilon_reg = 1, hspy_comp = False, tol = 1.0e-6) - estimator.fit_transform(X=X) - P2, A2 = estimator.W_, estimator.H_ - np.testing.assert_allclose(G @ P2 @ A2, Xdot, atol=1.1) - - estimator = SmoothNMF(G=G, shape_2d=[10,20], lambda_L=0, n_components= 2,max_iter=200, simplex_W=False, simplex_H = True,mu = 0, epsilon_reg = 1, hspy_comp = False, tol = 1.0e-6) - estimator.fit_transform(X=X) - P2, A2 = estimator.W_, estimator.H_ - np.testing.assert_allclose(G @ P2 @ A2, Xdot, atol=1.1) - - estimator = SmoothNMF(G=G, lambda_L=100, n_components= 2,max_iter=200, simplex_W=False, simplex_H = True,mu = 0, epsilon_reg = 1, shape_2d=[10,20], hspy_comp = False, tol = 1.0e-6) - estimator.fit_transform(X=X) - P3, A3 = estimator.W_, estimator.H_ - np.testing.assert_allclose(G @ P3 @ A3, Xdot, atol=1.1) - L = create_laplacian_matrix(10, 20) - - assert(trace_xtLx(L, A3.T) < trace_xtLx(L, A2.T)) - # assert(trace_xtLx(L, A.T) < trace_xtLx(L, A2.T) ) - assert(trace_xtLx(L, A3.T) < trace_xtLx(L, H.T)*1.01 ) - -def test_fixed_mat () : - G, W, H, D, w, X, Xdot, N = generate_one_sample() - fW, fH = gen_fixed_mat() - estimator = SmoothNMF(G=G, n_components= 2,max_iter=200, simplex_W=False, simplex_H=True, fixed_W = fW, hspy_comp = False) - estimator.fit_transform(X=X) - P2, A2 = estimator.W_, estimator.H_ - np.testing.assert_allclose(P2[fW >= 0],fW[fW>=0]) - - estimator = SmoothNMF(G=G, n_components= 2,max_iter=200, simplex_W=False, simplex_H=True, fixed_H = fH, hspy_comp = False) - estimator.fit_transform(X=X) - P2, A2 = estimator.W_, estimator.H_ - np.testing.assert_allclose(A2[fH >= 0],fH[fH>=0]) - -def test_surrogate_smooth_nmf(): - L = create_laplacian_matrix(4, 3) - for i in range(10): - A1 = np.random.randn(3, 12) - v1 = smooth_l2_surrogate(A1, L) - v2 = smooth_l2_surrogate(A1, L, A1) - v3 = trace_xtLx(L, A1.T) / 2 - np.testing.assert_almost_equal(v1, v2) - np.testing.assert_almost_equal(v1, v3) - - for j in range(10): - A2 = np.random.randn(3, 12) - v4 = smooth_l2_surrogate(A1, L, A2) - v5 = trace_xtLx(L, A2.T) / 2 - assert v4 >= v5 - d = diff_surrogate(A1, A2, L=L, algo="l2_surrogate") - np.testing.assert_almost_equal(v4 - v5 , d) - -def test_surrogate_smooth_dgkl_nmf(): - - L = create_laplacian_matrix(4, 3) - for i in range(10): - A1 = np.random.rand(3, 12) - v1 = smooth_dgkl_surrogate(A1, L) - v2 = smooth_dgkl_surrogate(A1, L, A1) - v3 = trace_xtLx(L, A1.T) / 2 - np.testing.assert_almost_equal(v1, v2) - np.testing.assert_almost_equal(v1, v3) - - for j in range(10): - A2 = np.random.rand(3, 12) - v4 = smooth_dgkl_surrogate(A1, L, A2) - v5 = trace_xtLx(L, A2.T) / 2 - assert v4 >= v5 - d = diff_surrogate(A1, A2, L=L, algo="log_surrogate") - np.testing.assert_almost_equal(v4 - v5 , d) - - - -def test_X_normalize() : - np.random.seed(0) - X = np.random.rand(10,32)*100 - fac = np.random.rand()*50+0.1 - print(fac) - X_plus = np.concatenate([X/fac,X/fac],axis = 0) - - nc = 5 - - estim = SmoothNMF(n_components = nc,lambda_L=1.0, max_iter = 10, init = "nndsvd", normalize=True, shape_2d = [8, 4], random_state = 0, simplex_W=False, simplex_H=True) - GP = estim.fit_transform(X) - H = estim.H_ - GP_plus = estim.fit_transform(X_plus) - H_plus = estim.H_ - - def close_enough(a,b) : - num = np.sum(np.abs(a-b)) - den = np.sum(np.abs(a)) - ratio = num/den - print(num,den,ratio) - return ratio < 1e-4 - - # plt.figure(figsize=(10,5)) - # plt.subplot(121) - # plt.imshow(H) - # plt.colorbar() - # plt.subplot(122) - # plt.imshow(H_plus) - # plt.colorbar() - - # plt.figure(figsize=(10,5)) - # plt.subplot(121) - # plt.imshow(np.concatenate([GP, GP], axis=0)) - # plt.colorbar() - # plt.subplot(122) - # plt.imshow(GP_plus*fac) - # plt.colorbar() - # np.testing.assert_allclose(GP_plus*fac, np.concatenate([GP, GP], axis=0), atol=1e-10) - # np.testing.assert_allclose(H_plus, H, atol=1e-10) - assert close_enough(GP_plus*fac, np.concatenate([GP, GP], axis=0)) - assert close_enough(H_plus, H) - -def test_normalization_factor () : - X_high = np.random.rand(10,32) - fac = np.random.rand()*50 - X_low = X_high / fac - - nc = 5 - - X_high_norm = normalization_factor(X_high,nc) * X_high - X_low_norm = normalization_factor(X_low,nc) * X_low - - np.testing.assert_allclose(X_high_norm, X_low_norm) - - -# def test_losses(): -# G, P, A, D, w, X, Xdot, N = generate_one_sample() -# true_spectra = (G @ P @ np.diag(w)) -# true_maps = A - -# estimator = SmoothNMF(G = G, shape_2d = (10,20), n_components = 2, max_iter = 10, true_A = true_maps, true_D = true_spectra, lambda_L = 1, hspy_comp = False) - -# estimator.fit_transform(X = X) - -# loss = estimator.get_losses() -# # k = 2 -# # default_params = { -# # "n_components" : k, -# # "tol" : 1e-6, -# # "max_iter" : 10, -# # "init" : "random", -# # "random_state" : 1, -# # "verbose" : 0 -# # } - -# # params_snmf = { -# # "simplex_H" : True, -# # "mu": np.random.rand(k) -# # } - -# # params_evalution = { -# # "u" : True, -# # } - -# # # All parameters are contained here -# # exp = {"name": "snmfem smooth 30", "method": "SmoothNMF", "params": {**default_params, **params_snmf, "lambda_L" : 100.0}} - -# # spim = hs.signals.Signal1D(X) -# # spim.set_signal_type("EDXSsnmfem") - -# # estimator = SmoothNMF(**exp["params"]) - -# # m, (G, P, A), loss = run_experiment(spim,estimator,exp) - -# values = np.array([list(e) for e in loss]) -# np.testing.assert_allclose(KL(X, G@P @ A, average=True), values[-1,1]) \ No newline at end of file diff --git a/espm/tests/test_updates.py b/espm/tests/test_updates.py deleted file mode 100644 index d208289..0000000 --- a/espm/tests/test_updates.py +++ /dev/null @@ -1,776 +0,0 @@ -import numpy as np - -from espm.estimators.updates import dichotomy_simplex, multiplicative_step_w, multiplicative_step_h, update_q, dichotomy_simplex_acc, multiplicative_step_hq -from espm.estimators.updates import estimate_Lipschitz_bound_h, estimate_Lipschitz_bound_w, gradW, gradH, proj_grad_step_h, proj_grad_step_w -from espm.measures import KLdiv_loss, log_reg, Frobenius_loss, trace_xtLx -from espm.conf import log_shift, dicotomy_tol -from espm.utils import create_laplacian_matrix -import pytest - -def make_step_a(x_matr, g_matr, p_matr , a_matr, mu_sparse=0, eps=log_shift, eps_sparse=1, mask=None): - """ - Multiplicative step in A. - The main terms are calculated first. - With mu_sparse = 0, the steps in A are calculated once. For mu_sparse != 0, the steps in A are calculated first with particle regularization. Then only the entries allowed by the mask are calculaed, without particle regularization. - To calculate the regularized step, we make a linear approximation of the log. - """ - # Update of the d_matr (maybe it should be done in a dedicated function for clarity) - d_matr = g_matr @ p_matr - - # Multiplicative update numerator U and denominator V - d_a = d_matr @ a_matr - U = d_matr.T @ (x_matr.clip(min=1e-150) / d_a) - V = d_matr.sum(axis=0) - # Reset of the Lagrangian multiplier (Maybe non-necessary .. ?) - # lambda_s = np.zeros((x_shape[1] * x_shape[0],)) - - if mu_sparse != 0: - # Regularized part of the algorithm - if mask is None: - # In the linear approximation, the slope is constant. We modifiy this slope to approximate the log every 10 iterations. - # The number of iterations in between two slope changes is arbitrary. - # if num_iterations % 10 == 0: - fixed_a = a_matr.copy() - # Vectorized version of the regularization - vec_sparse = np.array([0] + (a_matr.shape[0] - 1) * [mu_sparse]) - denum = V[:, np.newaxis] + vec_sparse[:, np.newaxis] / ( - fixed_a + eps_sparse - ) - # Lagragian multiplier - lambda_s = dichotomy_simplex(a_matr * U, denum, 0, tol=dicotomy_tol) - # A update (regularized) - a_matr = ( - a_matr - / ( - V[:, np.newaxis] - + vec_sparse[:, np.newaxis] / (fixed_a + eps_sparse) - + lambda_s - ) - * U - ) - else: - # Lagragian multiplier - lambda_s = dichotomy_simplex( - a_matr * U, V[:, np.newaxis], 0, tol=dicotomy_tol - ) - # Update the entry that did not meet the sparsity requirements - n_mask = np.invert(mask) - # A update (masked) - a_matr[n_mask] = ( - a_matr[n_mask] - / (V[:, np.newaxis] + lambda_s)[n_mask] - * U[n_mask] - ) - - else: - # Lagragian multiplier - lambda_s = dichotomy_simplex( - a_matr * U, V[:, np.newaxis], 0, tol=dicotomy_tol - ) - # A update (not regularized) - a_matr = a_matr / (V[:, np.newaxis] + lambda_s) * U - - return a_matr - -def make_step_p(x_matr, g_matr, p_matr , a_matr, eps = log_shift): - """ - Multiplicative step in P. - """ - d_matr = g_matr @ p_matr - - d_a = d_matr @ a_matr - term1 = ( - g_matr.T @ (x_matr.clip(min=1e-150) / (d_a + eps)) @ a_matr.T - ) # The data are clipped to avoid issues during computation - term2 = ( - g_matr.sum(axis=0)[:, np.newaxis] - @ a_matr.sum(axis=1)[:, np.newaxis].T - ) - # P update - p_matr = p_matr / term2 * term1 - return p_matr - -def test_dichotomy_simplex(): - num = np.random.rand(1,1) + 1 - denum = np.random.rand(1,1) - sol = num - denum - tol = 1e-8 - sol2 = dichotomy_simplex(num, denum, 0, tol=tol ) - assert(np.abs(sol -sol2 )< 2*tol) - - n = 10 - num = np.random.rand(1,n) - denum = np.random.rand(1, n) - sol = np.squeeze(num - denum) - sol2 = dichotomy_simplex(num, denum, 0, tol=tol ) - np.testing.assert_allclose(sol2, sol, atol=tol) - - num = np.random.rand(n, 1) - denum = np.random.rand(n, 1) - tol = 1e-6 - sol = dichotomy_simplex(num, denum, 0, tol=tol) - np.sum(num/(denum + sol)) - np.testing.assert_allclose(np.sum(num/(denum + sol)), 1, atol=tol) - - num = np.random.rand(n, 6) - denum = np.random.rand(n, 6) - tol = 1e-6 - sol = dichotomy_simplex(num, denum, 0, tol=tol ) - v = np.sum(num/(denum + sol), axis=0) - np.testing.assert_allclose(v, np.ones([6]), atol=tol) - - num = np.random.rand(n, 1) - denum = np.random.rand(n, 1) - num = num/(np.sum(num/denum)) - np.testing.assert_allclose(np.sum(num/(denum)), 1) - tol = 1e-6 - sol = dichotomy_simplex(num, denum, 0, tol=tol ) - np.testing.assert_allclose(sol, 0, atol=tol) - - tol = 1e-6 - num = np.array([[1,1,0,0,0,2]]).T - denum = np.array([[1,1,3,5,4,2]]).T - sol = dichotomy_simplex(num, denum, 0, tol=tol ) - np.testing.assert_allclose(np.sum(num/(denum + sol)), 1, atol=tol) - - tol = 1e-6 - num = np.array([[1,1,0,0,0,2]]).T - denum = np.array([[1,1,0,5,0,2]]).T - sol = dichotomy_simplex(num, denum, 0, tol=tol ) - np.sum(num/(denum + sol)) - np.testing.assert_allclose(np.sum(num/(denum + sol)), 1, atol=tol) - -def test_dichotomy_simplex_contraint(): - # Test the other constraint - tol = 1e-8 - num = np.array([[3,0.5]]).T - denum = np.array([[1,1]]).T - sol = np.array([[3]]).T - tol = 1e-8 - sol2 = dichotomy_simplex(num, denum, 1/4, tol=tol ) - assert(np.abs(sol -sol2 )< 2*tol) - - n = 10 - num = np.random.rand(1,n) - denum = np.random.rand(1, n) - sol = np.squeeze(num - denum) - sol2 = dichotomy_simplex(num, denum, 0.5, tol=tol ) - np.testing.assert_allclose(sol2, sol, atol=tol) - - with pytest.raises(ValueError) as _: - num = np.random.rand(1,n) - denum = np.random.rand(1, n) - dichotomy_simplex(num, denum, 1.1, tol=tol ) - with pytest.raises(ValueError) as _: - num = np.random.rand(3,n) - denum = np.random.rand(3, n) - dichotomy_simplex(num, denum, 0.5, tol=tol ) - - n = 10 - num = np.random.rand(1,n) - denum = np.random.rand(1, n) - sol = np.squeeze(num - denum) - sol2 = dichotomy_simplex(num, denum, 0.99, tol=tol ) - np.testing.assert_allclose(sol2, sol, atol=tol) - - n = 10 - num = np.random.rand(n, 6) - denum = np.random.rand(n, 6) - tol = 1e-6 - log_shift = 0.05 - sol = dichotomy_simplex(num, denum, log_shift, tol=tol ) - v = np.sum(np.maximum(num/(denum + sol), log_shift), axis=0) - np.testing.assert_allclose(v, np.ones([6]), atol=tol) - -def test_dicotomy2(): - k = 5 - p = 6400 - span = np.logspace(-6,6,num=17) - its = 25 - tol = 0 - maxit = 100 - np.random.seed(0) - for _ in range(its) : - scale_num = np.random.choice(span,size=(k,p)) - num = scale_num * np.random.rand(k,p) - scale_denum = np.random.choice(span,size=(k,p)) - denum = scale_denum * np.random.rand(k,p) - sol = dichotomy_simplex(num, denum, 0, tol=tol, maxit=maxit) - v = np.sum(num/(denum + sol), axis=0) - np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) - - log_shift = 0.1 / k - sol = dichotomy_simplex(num, denum, log_shift, tol=tol, maxit=maxit) - v = np.sum(np.maximum(num/(denum + sol), log_shift), axis=0) - np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) - - for _ in range(its) : - scale_num = np.random.choice(span,size=(k,p)) - num = scale_num * np.random.rand(k,p) - num[np.tile(np.arange(k),p//k),np.arange(p)] = 0 - scale_denum = np.random.choice(span,size=(k,p)) - denum = scale_denum * np.random.rand(k,p) - sol = dichotomy_simplex(num, denum, 0, tol=tol, maxit=maxit) - v = np.sum(num/(denum + sol), axis=0) - np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) - - log_shift = 0.1 / k - sol = dichotomy_simplex(num, denum, log_shift, tol=tol, maxit=maxit) - v = np.sum(np.maximum(num/(denum + sol), log_shift), axis=0) - np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) - - for _ in range(its) : - scale_num = np.random.choice(span,size=(k,p)) - num = scale_num * np.random.rand(k,p) - scale_denum = np.random.choice(span,size=(k,1)) - denum = scale_denum * np.random.rand(k,1) - sol = dichotomy_simplex(num, denum, 0, tol=tol, maxit=maxit) - v = np.sum(num/(denum + sol), axis=0) - np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) - - log_shift = 0.1 / k - sol = dichotomy_simplex(num, denum, log_shift, tol=tol, maxit=maxit) - v = np.sum(np.maximum(num/(denum + sol), log_shift), axis=0) - np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) - - for _ in range(its) : - scale_num = np.random.choice(span,size=(k,p)) - num = scale_num * np.random.rand(k,p) - num[np.tile(np.arange(k),p//k),np.arange(p)] = 0 - scale_denum = np.random.choice(span,size=(k,1)) - denum = scale_denum * np.random.rand(k,1) - sol = dichotomy_simplex(num, denum, 0, tol=tol, maxit=maxit) - v = np.sum(num/(denum + sol), axis=0) - np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) - - log_shift = 0.1 / k - sol = dichotomy_simplex(num, denum, log_shift, tol=tol, maxit=maxit) - v = np.sum(np.maximum(num/(denum + sol), log_shift), axis=0) - np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) - -def test_dicotomy_aq(): - def func_abc(x, a, b, minus_c): - n_p = len(b) - return n_p * x + 2*a + np.sum( - np.sqrt( (b + x)**2 + 4*a*minus_c) + b, axis=0) - - # def func_abc_onedim(x, a, b, minus_c): - # return x + 2*a + b - np.sqrt( (b + x)**2 + 4*a*minus_c) - - - a = np.random.rand() - b = np.random.rand(1,1) - minus_c = np.random.rand(1,1) - # x + 2*a + b - np.sqrt( (b + x)**2 + 4*a*minus_c) = 0 - # x + 2*a + b = np.sqrt( (b + x)**2 + 4*a*minus_c) - # (x + b)^2 + 4*a^2 + 4*a (x + b) = (b + x)**2 + 4*a*minus_c - # a + (x + b) = minus_c - # x = minus_c - a -b - sol = minus_c - a - b - tol = 1e-8 - sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol ) - assert(np.abs(sol -sol2 )< 10*tol) - - a = np.random.rand() - b = np.zeros([1,1]) - minus_c = np.random.rand(1,1) - sol = minus_c - a - b - tol = 1e-8 - sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol ) - assert(np.abs(sol -sol2 )< 10*tol) - - a = np.random.rand() - b = np.random.rand(1,1) - minus_c = np.zeros([1,1]) - sol = minus_c - a - b - tol = 1e-8 - sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol ) - assert(np.abs(sol -sol2 )< 10*tol) - - a = np.random.rand() - b = np.zeros([1,1]) - minus_c = np.zeros([1,1]) - sol = minus_c - a - b - tol = 1e-8 - sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol ) - assert(np.abs(sol -sol2 )< 10*tol) - - - n = 10 - a = np.random.rand() - b = np.random.rand(1,n) - minus_c = np.random.rand(1,n) - sol = np.squeeze(minus_c - a - b) - sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol ) - np.testing.assert_allclose(sol2, sol, atol=10*tol) - - a = np.random.rand() - b = np.random.rand(n,6) - minus_c = np.random.rand(n,6) - tol = 1e-6 - sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol) - np.testing.assert_allclose(func_abc(sol, a, b, minus_c), np.zeros([6]), atol=10*tol) - - -def test_dichotomy_aq_constraint(): - # Test the other constraint - tol = 1e-8 - a = 4 - log_shift = 1/8 - b = np.array([[1,1,1, 8/log_shift, 8/log_shift]]).T - minus_c = np.array([[1,1,1, 1, 1]]).T - - sol = 2 - tol = 1e-8 - sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=log_shift, tol=tol ) - assert(np.abs(sol -sol2 )< 10*tol) - - - n = 10 - a = np.random.rand() - b = np.random.rand(1,n) - minus_c = np.random.rand(1,n) - sol = np.squeeze(minus_c - a - b) - sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0.5, tol=tol ) - np.testing.assert_allclose(sol2, sol, atol=10*tol) - - - with pytest.raises(ValueError) as _: - a = np.random.rand(1,n) - b = np.random.rand(1, n) - minus_c = np.random.rand(1,n) - sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=1.1, tol=tol ) - - with pytest.raises(ValueError) as _: - a = np.random.rand(3,n) - b = np.random.rand(3, n) - minus_c = np.random.rand(3,n) - sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0.5, tol=tol ) - - n = 10 - a = np.random.rand() - b = np.random.rand(1,n) - minus_c = np.random.rand(1,n) - sol = np.squeeze(minus_c - a - b) - sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0.99, tol=tol ) - np.testing.assert_allclose(sol2, sol, atol=10*tol) - - def func_abc(x, a, b, minus_c, log_shift): - return np.sum( np.maximum(np.sqrt( (b + x)**2 + 4*a*minus_c) - b - x, log_shift*2*a), axis=0) - 2*a - - n = 10 - a = np.random.rand() - b = np.random.rand(6,n) - minus_c = np.random.rand(6,n) - log_shift = 0.1 - sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=log_shift, tol=tol ) - v = func_abc(sol, a, b, minus_c, log_shift) - np.testing.assert_allclose(v, np.zeros([n]), atol=tol) - - -def test_dicotom_aq2(): - k = 5 - p = 6400 - maxit = 100 - span = np.logspace(-6,5,num=17) - its = 25 - tol = 1e-6 - # def func_abc(x, a, b, minus_c): - # n_p = len(b) - # return n_p * x + 2*a + np.sum( - np.sqrt( (b + x)**2 + 4*a*minus_c) + b, axis=0) - def func_abc(x, a, b, minus_c, log_shift=0): - return np.sum( np.maximum(np.sqrt( (b + x)**2 + 4*a*minus_c) - b - x, log_shift*2*a), axis=0) - 2*a - - - np.random.seed(0) - for _ in range(its) : - a = np.random.rand() - scale_num = np.random.choice(span,size=(k,p)) - b = scale_num * np.random.rand(k,p) - scale_denum = np.random.choice(span,size=(k,1)) - minus_c = scale_denum * np.random.rand(k,1) - sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol, maxit=maxit ) - np.testing.assert_allclose(func_abc(sol, a, b, minus_c), np.zeros([p]), atol=tol) - - log_shift = 0.1 / k - sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=log_shift, tol=tol, maxit=maxit ) - np.testing.assert_allclose(func_abc(sol, a, b, minus_c, log_shift=log_shift), np.zeros([p]), atol=tol) - - for _ in range(its) : - a = np.random.rand() - scale_num = np.random.choice(span,size=(k,p)) - b = scale_num * np.random.rand(k,p) - scale_denum = np.random.choice(span,size=(k,p)) - minus_c = scale_denum * np.random.rand(k,p) - sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol, maxit=maxit ) - np.testing.assert_allclose(func_abc(sol, a, b, minus_c), np.zeros([p]), atol=tol) - - log_shift = 0.1 / k - sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=log_shift, tol=tol, maxit=maxit ) - np.testing.assert_allclose(func_abc(sol, a, b, minus_c, log_shift=log_shift), np.zeros([p]), atol=tol) - - for _ in range(its) : - a = np.random.rand() - scale_num = np.random.choice(span,size=(k,p)) - b = scale_num * np.random.rand(k,p) - b[np.tile(np.arange(k),p//k),np.arange(p)] = 0 - scale_denum = np.random.choice(span,size=(k,p)) - minus_c = scale_denum * np.random.rand(k,p) - sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol, maxit=maxit ) - np.testing.assert_allclose(func_abc(sol, a, b, minus_c), np.zeros([p]), atol=tol) - - log_shift = 0.1 / k - sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=log_shift, tol=tol, maxit=maxit ) - np.testing.assert_allclose(func_abc(sol, a, b, minus_c, log_shift=log_shift), np.zeros([p]), atol=tol) - - for _ in range(its) : - a = np.random.rand() - scale_num = np.random.choice(span,size=(k,p)) - b = scale_num * np.random.rand(k,p) - b[np.tile(np.arange(k),p//k),np.arange(p)] = 0 - scale_denum = np.random.choice(span,size=(k,1)) - minus_c = scale_denum * np.random.rand(k,1) - sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol, maxit=maxit ) - np.testing.assert_allclose(func_abc(sol, a, b, minus_c), np.zeros([p]), atol=tol) - - log_shift = 0.1 / k - sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=log_shift, tol=tol, maxit=maxit ) - np.testing.assert_allclose(func_abc(sol, a, b, minus_c, log_shift=log_shift), np.zeros([p]), atol=tol) - -def test_multiplicative_step_w(): - l = 26 - k = 5 - p = 100 - c = 17 - - A = np.random.rand(k,p) - A = A/np.sum(A, axis=1, keepdims=True) - - G = np.random.rand(l,c) - P = np.random.rand(c,k) - GP = G @ P - - X = GP @ A - - Pp = multiplicative_step_w(X, G, P, A, log_shift=0, simplex_W = False) - np.testing.assert_array_almost_equal(Pp, P) - - Pp = multiplicative_step_w(X, G, P, A, log_shift=0, l2=True, simplex_W = False) - np.testing.assert_array_almost_equal(Pp, P) - - for _ in range(10): - P = np.random.rand(c,k) - Pp = multiplicative_step_w(X, G, P, A, simplex_W = False) - Pp2 = make_step_p(X, G, P, A) - np.testing.assert_array_almost_equal(Pp, Pp2) - GP = G @ P - GPp = G @ Pp - val1 = KLdiv_loss(X, GP, A) - val2 = KLdiv_loss(X, GPp, A) - np.testing.assert_array_less(0, Pp) - assert(val1 > val2) - - - for _ in range(10): - P = np.random.rand(c,k) - Pp = multiplicative_step_w(X, G, P, A, l2=True, simplex_W = False) - GP = G @ P - GPp = G @ Pp - val1 = Frobenius_loss(X, GP, A) - val2 = Frobenius_loss(X, GPp, A) - np.testing.assert_array_less(0, Pp) - assert(val1 > val2) - - -def test_multiplicative_step_h(): - l = 26 - k = 5 - p = 100 - c = 17 - - A = np.random.rand(k,p) - A = A/np.sum(A, axis=0, keepdims=True) - - G = np.random.rand(l,c) - P = np.random.rand(c,k) - GP = G @ P - - X = GP @ A - np.testing.assert_allclose(np.sum(A, axis=0), np.ones([A.shape[1]]), atol=dicotomy_tol) - - Ap = multiplicative_step_h(X, G, P, A, simplex_H=False, mu=0, log_shift=0, epsilon_reg=1, safe=True) - np.testing.assert_array_almost_equal(A, Ap) - np.testing.assert_allclose(np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol) - - Ap = multiplicative_step_h(X, G, P, A, simplex_H=True, mu=0, log_shift=0, epsilon_reg=1, safe=True) - np.testing.assert_allclose(A, Ap, atol=dicotomy_tol) - - # Same test for l2 - Ap = multiplicative_step_h(X, G, P, A, simplex_H=False, mu=0, log_shift=0, epsilon_reg=1, safe=True, l2=True) - np.testing.assert_array_almost_equal(A, Ap) - np.testing.assert_allclose(np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol) - - Ap = multiplicative_step_h(X, G, P, A, simplex_H=True, mu=0, log_shift=0, epsilon_reg=1, safe=True, l2=True) - np.testing.assert_allclose(A, Ap, atol=dicotomy_tol) - - for _ in range(10): - A = np.random.rand(k,p) - A = A/np.sum(A, axis=1, keepdims=True) - Ap = multiplicative_step_h(X, G, P, A, simplex_H=False, mu=0, log_shift=0, epsilon_reg=1, safe=True) - val1 = KLdiv_loss(X, GP, A) - val2 = KLdiv_loss(X, GP, Ap) - np.testing.assert_array_less(0, Ap) - assert(val1 > val2) - - Ap = multiplicative_step_h(X, G, P, A, simplex_H=True, mu=0, log_shift=log_shift, epsilon_reg=1, safe=True) - Ap2 = make_step_a(X, G, P, A, mu_sparse=0, eps=log_shift, eps_sparse=1, mask=None) - np.testing.assert_array_almost_equal(Ap2, Ap) - np.testing.assert_allclose(np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol) - - val1 = KLdiv_loss(X, GP, A) - val2 = KLdiv_loss(X, GP, Ap) - np.testing.assert_array_less(0, Ap) - assert(val1 > val2) - - epsilon_reg = 1 - mu = np.ones(k) - mu[0] = 0 - Ap = multiplicative_step_h(X, G, P, A, simplex_H=True, mu=mu, log_shift=log_shift, epsilon_reg=epsilon_reg, safe=True) - Ap2 = make_step_a(X, G, P, A, mu_sparse=1, eps=log_shift, eps_sparse=epsilon_reg, mask=None) - np.testing.assert_array_almost_equal(Ap2, Ap) - np.testing.assert_allclose(np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol) - - - Ap = multiplicative_step_h(X, G, P, A, simplex_H=True, mu=3*mu, log_shift=log_shift, epsilon_reg=epsilon_reg, safe=True) - Ap2 = make_step_a(X, G, P, A, mu_sparse=3, eps=log_shift, eps_sparse=epsilon_reg, mask=None) - np.testing.assert_array_almost_equal(Ap2, Ap) - np.testing.assert_allclose(np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol) - - - val1 = KLdiv_loss(X, GP, A) + log_reg(A, 3*mu, epsilon_reg) - val2 = KLdiv_loss(X, GP, Ap) + log_reg(A, 3*mu, epsilon_reg) - np.testing.assert_array_less(0, Ap) - assert(val1 > val2) - - # Ap = multiplicative_step_h(X, G, P, A, simplex_H=True, mu=3, eps=log_shift, epsilon_reg=1, safe=True) - # Ap2 = make_step_a(X, G, P, A, mu_sparse=3, eps=log_shift, eps_sparse=1, mask=np.zeros([k])>0) - # np.testing.assert_array_almost_equal(Ap2, Ap) - # np.testing.assert_allclose(np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol) - - for _ in range(10): - A = np.random.rand(k,p) - A = A/np.sum(A, axis=1, keepdims=True) - Ap = multiplicative_step_h(X, G, P, A, simplex_H=False, mu=0, log_shift=0, epsilon_reg=1, safe=True, l2=True) - val1 = Frobenius_loss(X, GP, A) - val2 = Frobenius_loss(X, GP, Ap) - np.testing.assert_array_less(0, Ap) - assert(val1 > val2) - - Ap = multiplicative_step_h(X, G, P, A, simplex_H=True, mu=0, log_shift=log_shift, epsilon_reg=1, safe=True, l2=True) - np.testing.assert_allclose(np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol) - - val1 = Frobenius_loss(X, GP, A) - val2 = Frobenius_loss(X, GP, Ap) - np.testing.assert_array_less(0, Ap) - assert(val1 > val2) - -def test_multiplicative_step_hq(): - - l = 26 - k = 5 - p = 100 - c = 17 - - A = np.random.rand(k,p) - A = A/np.sum(A, axis=0, keepdims=True) - - G = np.random.rand(l,c) - P = np.random.rand(c,k) - GP = G @ P - - X = GP @ A - np.testing.assert_allclose(np.sum(A, axis=0), np.ones([A.shape[1]]), atol=dicotomy_tol) - - Ap = multiplicative_step_hq(X, G, P, A, simplex_H=False, log_shift=0,safe=True) - np.testing.assert_array_almost_equal(A, Ap) - np.testing.assert_allclose(np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol) - - Ap = multiplicative_step_hq(X, G, P, A, simplex_H=True, log_shift=0, safe=True) - np.testing.assert_allclose(A, Ap, atol=dicotomy_tol) - - for _ in range(10): - A = np.random.rand(k,p) - A = A/np.sum(A, axis=1, keepdims=True) - Ap = multiplicative_step_hq(X, G, P, A, simplex_H=False, log_shift=0, safe=True) - val1 = KLdiv_loss(X, GP, A) - val2 = KLdiv_loss(X, GP, Ap) - np.testing.assert_array_less(0, Ap) - assert(val1 > val2) - - Ap = multiplicative_step_hq(X, G, P, A, simplex_H=True, log_shift=log_shift, safe=True) - Ap2 = make_step_a(X, G, P, A, mu_sparse=0, eps=log_shift, eps_sparse=1, mask=None) - np.testing.assert_array_almost_equal(Ap2, Ap) - np.testing.assert_allclose(np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol) - - val1 = KLdiv_loss(X, GP, A) - val2 = KLdiv_loss(X, GP, Ap) - np.testing.assert_array_less(0, Ap) - assert(val1 > val2) - - - - -def test_Q_step(): - l = 4 - p = 5 - i = 3 - - D = np.random.randn(l,i) - A = np.random.randn(i,p) - - Q = np.zeros([l,p,i]) - for a in range(l): - for b in range(p): - for c in range(i): - Q[a,b,c] = A[c,b] * D[a,c] / np.sum(A[:,b] * D[a,:]) - Q2 = update_q(D, A) - np.testing.assert_array_almost_equal(Q, Q2) - -def test_Q_step2(): - l = 4 - p = 5 - i = 3 - - D = np.random.randn(l,i) - A = np.random.randn(i,p) - A[0,:] = 0 - A[:,0] = 0 - D[0,:] = 0 - D[:,0] = 0 - - Q = np.zeros([l,p,i]) - for a in range(l): - for b in range(p): - for c in range(i): - Q[a,b,c] = A[c,b] * D[a,c] / (np.sum(A[:,b] * D[a,:]) + log_shift) - Q2 = update_q(D, A) - np.testing.assert_array_almost_equal(Q, Q2) - assert((np.isnan(Q)==False).all()) - -def create_toy_problem(l = 25, k = 3, p = 100, c = 10, n_poisson=200, simplex_H=True): - - A = np.random.rand(k,p) - if simplex_H: - A = A/np.sum(A, axis=0, keepdims=True) - - G = np.random.rand(l,c) - P = np.random.rand(c,k) - GP = G @ P - - X = GP @ A - - Xdot = 1/n_poisson * np.random.poisson(n_poisson * X) - - return G, P, A, X, Xdot - -def test_proj_step_h(): - - - def full_loss(X, G, W, H, log_shift, mu, lambda_L, L): - return KLdiv_loss(X, G@W, H, log_shift=log_shift) + lambda_L * trace_xtLx(L, H.T) + log_reg(H, mu) - - shape_2d = [10, 15] - k = 5 - n_poisson = 200 - log_shift = 0.01 - epsilon_reg = 2 - L = create_laplacian_matrix(*shape_2d) - - for _ in range(10): - - G, W, H, Xtrue, X = create_toy_problem(p = shape_2d[0]*shape_2d[1], k=k, n_poisson=n_poisson) - - true_D = G @ W - true_H = H - - for simplex_H in [True, False]: - for lambda_L in [0, 1, 10]: - for mu in [0, 1 ,10]: - - W0 = np.random.rand(*W.shape) - H0 = np.random.rand(*H.shape) - W0 = np.maximum(W0, log_shift) - H0 = np.maximum(H0, log_shift) - X = np.maximum(X, log_shift) - - - gamma_h = estimate_Lipschitz_bound_h(log_shift, X, G, k, mu=mu, lambda_L=lambda_L, epsilon_reg=epsilon_reg) - - loss1 = full_loss(X, G, W0, H0, log_shift, mu, lambda_L, L) - - grad = gradH(X, G, W0, H0, log_shift=log_shift, l2=False, mu=mu, lambda_L=lambda_L, L=L, epsilon_reg=epsilon_reg) - H1 = H0 - 1/gamma_h * grad - loss2 = full_loss(X, G, W0, H1, log_shift, mu, lambda_L, L) - H2 =proj_grad_step_h(X, G, W0, H0, gamma_h, log_shift=log_shift, safe=True, l2=False, simplex_H=simplex_H, mu=mu, lambda_L=lambda_L, L=L, epsilon_reg=epsilon_reg) - loss3 = full_loss(X, G, W0, H2, log_shift, mu, lambda_L, L) - assert( loss1 >= loss2) - assert(loss1 >= loss3) - assert((H2 >= log_shift).all()) - - maxit = 10 - loss_old = loss3 - grad = gradH(X, G, W0, H2, log_shift=log_shift, l2=False) - n_grad_old = np.sum(grad**2) - for _ in range(maxit): - H2 = proj_grad_step_h(X, G, W0, H2, gamma_h, log_shift=log_shift, safe=True, l2=False, simplex_H=simplex_H, mu=mu, lambda_L=lambda_L, L=L, epsilon_reg=epsilon_reg) - grad = gradH(X, G, W0, H2, log_shift=log_shift, mu=mu, lambda_L=lambda_L, L=L, l2=False) - n_grad = np.sum(grad**2) - loss = full_loss(X, G, W0, H2, log_shift, mu, lambda_L, L) - assert loss<=loss_old+np.abs(loss_old)*0.00001 - # assert n_grad<=n_grad_old - loss_old = loss - n_grad_old = n_grad - -def test_proj_step_w(): - for _ in range(10): - shape_2d = [10, 15] - log_shift = 0.01 - - k = 5 - n_poisson = 200 - G, W, H, Xtrue, X = create_toy_problem(p = shape_2d[0]*shape_2d[1], k=k, n_poisson=n_poisson) - - true_D = G @ W - true_H = H - - W0 = np.random.rand(*W.shape) - H0 = np.random.rand(*H.shape) - - - gamma_w = estimate_Lipschitz_bound_w(log_shift, X, G, k) - - loss1 = KLdiv_loss(X, G@W0, H0, log_shift=log_shift) - grad = gradW(X, G, W0, H0, log_shift=log_shift, l2=False) - W1 = W0 - 1/gamma_w * grad - loss2 = KLdiv_loss(X, G@W1, H0, log_shift=log_shift) - W2 =proj_grad_step_w(X, G, W0, H0, gamma_w, log_shift=log_shift, safe=True, l2=False, simplex_W = False) - loss3 = KLdiv_loss(X, G@W2, H0, log_shift=log_shift) - assert( loss1 >= loss2) - assert(loss1 >= loss3) - assert((W2 >= log_shift).all()) - - maxit = 10 - loss_old = loss3 - grad = gradW(X, G, W2, H0, log_shift=log_shift, l2=False) - n_grad_old = np.sum(grad**2) - for _ in range(maxit): - W2 = proj_grad_step_w(X, G, W2, H0, gamma_w, log_shift=log_shift, safe=True, l2=False, simplex_W = False) - grad = gradW(X, G, W2, H0, log_shift=log_shift, l2=False) - - n_grad = np.sum(grad**2) - loss = KLdiv_loss(X, G@W2, H0, log_shift=log_shift) - assert loss<=loss_old - assert n_grad<=n_grad_old - loss_old = loss - n_grad_old = n_grad - - diff --git a/espm/tests/test_utils.py b/espm/tests/test_utils.py deleted file mode 100644 index 9be6da3..0000000 --- a/espm/tests/test_utils.py +++ /dev/null @@ -1,115 +0,0 @@ -from exspy.material import density_of_mixture -import numpy as np -import espm.utils as u - - -def test_rescale() : - np.random.seed(0) - for _ in range(20): - # for k small - W = np.random.rand(27,5) - H = np.random.rand(5,150) - W_r, H_r = u.rescaled_DH(W,H) - assert(np.abs(np.mean(H_r.sum(axis=0)) -1) < np.abs(np.mean(H.sum(axis=0)) -1)) - np.testing.assert_array_almost_equal(W@H,W_r@H_r) - - # for k large - W = np.random.rand(50,10) - H = np.random.rand(10,5) - W_r, H_r = u.rescaled_DH(W,H) - np.testing.assert_array_almost_equal(W@H,W_r@H_r) - np.testing.assert_allclose(H_r.sum(axis=0),1, atol=0.2) - -def test_number_symbols () : - mixed_dict = {"Fe" : 1.0, "Si" : 2.0, 27 : 3.0, "83" : 4.0, 8 : 5.0} - symbol_dict = {"Fe" : 1.0, "Si" : 2.0, "Co" : 3.0, "Bi" : 4.0, "O" : 5.0} - number_dict = {26 : 1.0, 14 : 2.0, 27 : 3.0, 83 : 4.0, 8 : 5.0} - - mixed_list = ["Fe","Si",27,"83",8] - number_list = [26,14,27,83,8] - symbol_list = ["Fe", "Si", "Co", "Bi", "O"] - - @u.symbol_to_number_dict - def s_to_n (elements_dict = {}) : - return elements_dict - - @u.number_to_symbol_dict - def n_to_s (elements_dict = {}) : - return elements_dict - - @u.symbol_to_number_list - def s_to_n_list (elements = []) : - return elements - - @u.number_to_symbol_list - def n_to_s_list (elements = []) : - return elements - - assert s_to_n(elements_dict = mixed_dict) == number_dict - assert n_to_s(elements_dict = mixed_dict) == symbol_dict - assert s_to_n_list(elements=mixed_list) == number_list - assert n_to_s_list(elements = mixed_list) == symbol_list - - -def test_is_symbol () : - symbols = ["Si", "45", 34, "si", "Zbrra"] - bools = [True, False, False, False, False] - - bools_test = [] - for i in symbols : - bools_test.append(u.is_symbol(i)) - - assert bools == bools_test - -def test_hspy_wrappers () : - atomic_fractions = {8 : 0.2, 14 : 0.3, 26 : 0.1, 83 : 0.4} - weight_fractions = {'O': 0.03174415159035731, 'Si': 0.08358598161408992, 'Fe': 0.055400582070687154, 'Bi': 0.8292692847248655} - - res_wt = u.atomic_to_weight_dict(elements_dict = atomic_fractions) - assert res_wt == weight_fractions - - res_dens = u.approx_density(atomic_fraction = True, elements_dict=atomic_fractions) - hspy_dens = density_of_mixture([0.03174415159035731, 0.08358598161408992, 0.055400582070687154, 0.8292692847248655],['O','Si','Fe','Bi']) - - assert res_dens == hspy_dens - -def test_arg_helper () : - default_dict = { - "chou" : "non", - "carottes" : 3, - "salades" : {"roquette" : 2, "batavia" : 5}, - "tubercules" : { - "oignons" : {"jaune" : [1,2,3], "blanc" : [3,2,1]}, - "echalottes" : 5 - } - } - - input_dict = { - "chou" : 4, - "navets" : 3, - "salades" : {"clous" : 3, "roquette" : 2}, - "tubercules" : { - "oignons" : {"jaune" : 9}, - "vis" : {"cruciformes" : 3, "hexagonales" : 6} - } - } - - result_dict = { - "chou" : 4, - "carottes" : 3, - "navets" : 3, - "salades" : {"clous" : 3, "roquette" : 2, "batavia" : 5}, - "tubercules" : { - "oignons" : {"jaune" : 9, "blanc" : [3,2,1]}, - "echalottes" : 5, - "vis" : {"cruciformes" : 3, "hexagonales" : 6} - } - } - - assert u.arg_helper(input_dict,default_dict) == result_dict - -def test_bin_spim () : - array = np.random.rand(100,20,30) - - assert u.bin_spim(array,50,10).shape == (50,10,30) - assert u.bin_spim(array,30,6).shape == (30,6,30) \ No newline at end of file diff --git a/espm/utils.py b/espm/utils.py index 2c15168..e8eea0b 100644 --- a/espm/utils.py +++ b/espm/utils.py @@ -1,28 +1,29 @@ r"""Utils for the ESPM package""" -import numpy as np -from scipy.sparse import lil_matrix, block_diag -from scipy.optimize import nnls -from espm.conf import SYMBOLS_PERIODIC_TABLE, NUMBER_PERIODIC_TABLE import json -from exspy.material import atomic_to_weight, density_of_mixture from functools import wraps -import espm -from IPython.utils import io import hyperspy.api as hs -import seaborn import matplotlib as mpl import matplotlib.pyplot as plt -from sklearn.linear_model import LinearRegression as LR +import numpy as np +import seaborn import skimage as ski +from exspy.material import atomic_to_weight, density_of_mixture +from IPython.utils import io +from scipy.optimize import nnls +from scipy.sparse import block_diag, lil_matrix +from sklearn.linear_model import LinearRegression as LR +import espm +from espm.conf import NUMBER_PERIODIC_TABLE, SYMBOLS_PERIODIC_TABLE _qtg_widgets = [] _plt_figures = [] + def process_losses(losses): - r""" Process the losses to be plotted + r"""Process the losses to be plotted Parameters ---------- @@ -45,10 +46,11 @@ def process_losses(losses): values = np.array(values) return values, names + def create_laplacian_matrix(nx, ny=None): r""" Helper method to create the laplacian matrix for the laplacian regularization - + Parameters ---------- :param nx: height of the original image @@ -64,208 +66,238 @@ def create_laplacian_matrix(nx, ny=None): """ if ny is None: ny = nx - assert(nx>1) - assert(ny>1) - #Blocks corresponding to the corner of the image (linking row elements) - top_block=lil_matrix((ny,ny),dtype=np.float32) - top_block.setdiag([2]+[3]*(ny-2)+[2]) - top_block.setdiag(-1,k=1) - top_block.setdiag(-1,k=-1) - #Blocks corresponding to the middle of the image (linking row elements) - mid_block=lil_matrix((ny,ny),dtype=np.float32) - mid_block.setdiag([3]+[4]*(ny-2)+[3]) - mid_block.setdiag(-1,k=1) - mid_block.setdiag(-1,k=-1) - #Construction of the diagonal of blocks - list_blocks=[top_block]+[mid_block]*(nx-2)+[top_block] - blocks=block_diag(list_blocks) - #Diagonals linking different rows - blocks.setdiag(-1,k=ny) - blocks.setdiag(-1,k=-ny) + assert nx > 1 + assert ny > 1 + # Blocks corresponding to the corner of the image (linking row elements) + top_block = lil_matrix((ny, ny), dtype=np.float32) + top_block.setdiag([2] + [3] * (ny - 2) + [2]) + top_block.setdiag(-1, k=1) + top_block.setdiag(-1, k=-1) + # Blocks corresponding to the middle of the image (linking row elements) + mid_block = lil_matrix((ny, ny), dtype=np.float32) + mid_block.setdiag([3] + [4] * (ny - 2) + [3]) + mid_block.setdiag(-1, k=1) + mid_block.setdiag(-1, k=-1) + # Construction of the diagonal of blocks + list_blocks = [top_block] + [mid_block] * (nx - 2) + [top_block] + blocks = block_diag(list_blocks) + # Diagonals linking different rows + blocks.setdiag(-1, k=ny) + blocks.setdiag(-1, k=-ny) return blocks -def rescaled_DH(D,H) : +def rescaled_DH(D, H): r"""Rescale the matrices D and H such that the columns of H sums approximately to one. :param np.array 2D D: n x k matrix :param np.array 2D H: k x m matrix :return: D_rescale, H_rescale - :rtype: np.array 2D, np.array 2D + :rtype: np.array 2D, np.array 2D """ _, p = H.shape o = np.ones((p,)) s = np.linalg.lstsq(H.T, o, rcond=None)[0] - if (s<=0).any(): + if (s <= 0).any(): s = np.maximum(nnls(H.T, o)[0], 1e-10) - D_rescale = D@np.diag(1/s) - H_rescale = np.diag(s)@H + D_rescale = D @ np.diag(1 / s) + H_rescale = np.diag(s) @ H return D_rescale, H_rescale -def bin_spim(data,n,m): - r""" - + +def bin_spim(data, n, m): + r""" + Take a 3D array of size (x,y,k) [px, py, e] Returns a 3D array of size (n,m,k) [new_px, new_py, e] """ # return a matrix of shape (n,m,k) - bs = data.shape[0]//n,data.shape[1]//m # blocksize averaged over + bs = data.shape[0] // n, data.shape[1] // m # blocksize averaged over k = data.shape[2] - return np.reshape(np.array([np.sum(data[k1*bs[0]:(k1+1)*bs[0],k2*bs[1]:(k2+1)*bs[1]],axis=(0,1)) for k1 in range(n) for k2 in range(m)]),(n,m,k)) - - -def number_to_symbol_dict (func) : + return np.reshape( + np.array( + [ + np.sum( + data[k1 * bs[0] : (k1 + 1) * bs[0], k2 * bs[1] : (k2 + 1) * bs[1]], + axis=(0, 1), + ) + for k1 in range(n) + for k2 in range(m) + ] + ), + (n, m, k), + ) + + +def number_to_symbol_dict(func): r""" Decorator Takes a dict of elements (a.k.a chemical composition) with atomic numbers as keys (e.g. 26 for Fe) returns a dict of elements with symbols as keys (e.g. Fe for iron) """ + @wraps(func) - def inner(*args,**kwargs) : + def inner(*args, **kwargs): elts_dict = kwargs["elements_dict"] new_dict = {} - with open(NUMBER_PERIODIC_TABLE,"r") as f : + with open(NUMBER_PERIODIC_TABLE, "r") as f: NPT = json.load(f)["table"] - - for key in elts_dict.keys() : - - if is_symbol(key) : + + for key in elts_dict.keys(): + if is_symbol(key): new_dict[key] = elts_dict[key] - - elif is_number(key) : + + elif is_number(key): new_dict[NPT[str(key)]["symbol"]] = elts_dict[key] - - else : - raise ValueError("Input has to be either atomic number, either chemical symbols") - + + else: + raise ValueError( + "Input has to be either atomic number, either chemical symbols" + ) + kwargs["elements_dict"] = new_dict - return func(*args,**kwargs) + return func(*args, **kwargs) return inner -def symbol_to_number_dict (func) : + +def symbol_to_number_dict(func): r""" Decorator Takes a dict of elements (a.k.a chemical composition) with symbols as keys (e.g. Fe for iron) returns a dict of elements with atomic numbers as keys (e.g. 26 for iron) """ + @wraps(func) - def inner(*args,**kwargs) : + def inner(*args, **kwargs): elts_dict = kwargs["elements_dict"] new_dict = {} - with open(SYMBOLS_PERIODIC_TABLE,"r") as f : + with open(SYMBOLS_PERIODIC_TABLE, "r") as f: SPT = json.load(f)["table"] - for key in elts_dict.keys() : - - if is_number(key) : + for key in elts_dict.keys(): + if is_number(key): new_dict[int(key)] = elts_dict[key] - - elif is_symbol(key) : + + elif is_symbol(key): new_dict[SPT[key]["number"]] = elts_dict[key] - - else : - raise ValueError("Input has to be either atomic number, either chemical symbols") - + + else: + raise ValueError( + "Input has to be either atomic number, either chemical symbols" + ) + kwargs["elements_dict"] = new_dict - return func(*args,**kwargs) + return func(*args, **kwargs) + return inner -def symbol_to_number_list (func) : + +def symbol_to_number_list(func): r""" Decorator Takes a dict of elements (a.k.a chemical composition) with symbols as keys (e.g. Fe for iron) returns a dict of elements with atomic numbers as keys (e.g. 26 for iron) """ + @wraps(func) - def inner(*args,**kwargs) : + def inner(*args, **kwargs): elts_list = kwargs["elements"] new_list = [] - with open(SYMBOLS_PERIODIC_TABLE,"r") as f : + with open(SYMBOLS_PERIODIC_TABLE, "r") as f: SPT = json.load(f)["table"] - for key in elts_list : - if is_number(key) : + for key in elts_list: + if is_number(key): new_list.append(int(key)) - elif is_symbol(key) : + elif is_symbol(key): new_list.append(SPT[key]["number"]) - else : - raise ValueError("Input has to be either atomic number, either chemical symbols") - + else: + raise ValueError( + "Input has to be either atomic number, either chemical symbols" + ) + kwargs["elements"] = new_list - return func(*args,**kwargs) - + return func(*args, **kwargs) + return inner -def number_to_symbol_list (func) : + +def number_to_symbol_list(func): r""" Decorator Takes a dict of elements (a.k.a chemical composition) with symbols as keys (e.g. Fe for iron) returns a dict of elements with atomic numbers as keys (e.g. 26 for iron) """ + @wraps(func) - def inner(*args,**kwargs) : + def inner(*args, **kwargs): elts_list = kwargs["elements"] new_list = [] - with open(NUMBER_PERIODIC_TABLE,"r") as f : + with open(NUMBER_PERIODIC_TABLE, "r") as f: NPT = json.load(f)["table"] - for key in elts_list : - if is_number(key) : + for key in elts_list: + if is_number(key): new_list.append(NPT[str(key)]["symbol"]) - elif is_symbol(key) : + elif is_symbol(key): new_list.append(key) - else : - raise ValueError("Input has to be either atomic number, either chemical symbols") - + else: + raise ValueError( + "Input has to be either atomic number, either chemical symbols" + ) + kwargs["elements"] = new_list - return func(*args,**kwargs) - + return func(*args, **kwargs) + return inner + @number_to_symbol_dict -def atomic_to_weight_dict (*,elements_dict = {}) : +def atomic_to_weight_dict(*, elements_dict={}): r""" Wrapper to the atomic_to_weight function of hyperspy. Takes a dict of chemical composition expressed in atomic fractions. Returns a dict of chemical composition expressed in atomic weight fratiom. - """ - if len(elements_dict.keys()) == 0 : + """ + if len(elements_dict.keys()) == 0: return elements_dict - else : + else: list_elts = [] list_at = [] - for elt in elements_dict.keys() : + for elt in elements_dict.keys(): list_elts.append(elt) list_at.append(elements_dict[elt]) - list_wt = atomic_to_weight(list_at,list_elts)/100 - + list_wt = atomic_to_weight(list_at, list_elts) / 100 + new_dict = {} - for i, elt in enumerate(list_elts) : + for i, elt in enumerate(list_elts): new_dict[elt] = list_wt[i] - + return new_dict + @number_to_symbol_dict -def approx_density(atomic_fraction = False,*,elements_dict = {}) : +def approx_density(atomic_fraction=False, *, elements_dict={}): r""" Wrapper to the density_of_mixture function of hyperspy. Takes a dict of chemical composition expressed in atomic weight fractions. Returns an approximated density. - """ - if len(elements_dict.keys()) == 0 : + """ + if len(elements_dict.keys()) == 0: return 1.0 - else : + else: list_elts = [] list_wt = [] - if atomic_fraction : - elements_dict = atomic_to_weight_dict(elements_dict = elements_dict) - - for elt in elements_dict.keys() : + if atomic_fraction: + elements_dict = atomic_to_weight_dict(elements_dict=elements_dict) + + for elt in elements_dict.keys(): list_elts.append(elt) list_wt.append(elements_dict[elt]) - - return density_of_mixture(list_wt,list_elts) -def arg_helper(params, d_params, replace = True): - r""" Check if all parameter of d_params are in params. If not, they are added to params with the default value. + return density_of_mixture(list_wt, list_elts) + + +def arg_helper(params, d_params, replace=True): + r"""Check if all parameter of d_params are in params. If not, they are added to params with the default value. Parameters ---------- @@ -273,22 +305,23 @@ def arg_helper(params, d_params, replace = True): Dictionary of parameters to be checked. d_params : dict Dictionary of default parameters. - + Returns ------- params : dict Dictionary of parameters with the default parameters added if not present. - + """ for key in d_params.keys(): params[key] = params.get(key, d_params[key]) if isdict(params[key]) and isdict(d_params[key]): params[key] = arg_helper(params[key], d_params[key], replace=replace) - check_keys(params, d_params, replace = replace) + check_keys(params, d_params, replace=replace) return params -def check_keys(params, d_params, upperkeys = '',toprint = True, replace = True): - r""" Check if all parameter of d_params are in params. If not, they are added to params with the default value. + +def check_keys(params, d_params, upperkeys="", toprint=True, replace=True): + r"""Check if all parameter of d_params are in params. If not, they are added to params with the default value. Parameters ---------- @@ -300,7 +333,7 @@ def check_keys(params, d_params, upperkeys = '',toprint = True, replace = True): String of the upper keys. toprint : bool If True, print the warning. - + Returns ------- params : dict @@ -313,35 +346,44 @@ def check_keys(params, d_params, upperkeys = '',toprint = True, replace = True): >>> check_keys(params,d_params) >>> params {'a': 1, 'b': 2, 'c': 3} - + """ keys = set(d_params.keys()) for key in params.keys(): if key not in keys: - if toprint : - print('Warning! Optional argument: {}[\'{}\'] specified by user but not used'.format(upperkeys,key)) + if toprint: + print( + f"Warning! Optional argument: {upperkeys}['{key}'] specified by user but not used" + ) else: if isdict(params[key]): -# if not(isdict(d_params[key])): -# print('Warning! Optional argument: {}{} is not supposed to be a dictionary'.format(upperkeys,key)) -# else: -# check_keys(params[key],d_params[key],upperkeys=upperkeys+'[\'{}\']'.format(key)) + # if not(isdict(d_params[key])): + # print('Warning! Optional argument: {}{} is not supposed to be a dictionary'.format(upperkeys,key)) + # else: + # check_keys(params[key],d_params[key],upperkeys=upperkeys+'[\'{}\']'.format(key)) if isdict(d_params[key]): - if toprint : - check_keys(params[key],d_params[key],upperkeys=upperkeys+'[\'{}\']'.format(key), toprint = toprint, replace = replace) + if toprint: + check_keys( + params[key], + d_params[key], + upperkeys=upperkeys + f"['{key}']", + toprint=toprint, + replace=replace, + ) else: - if replace : + if replace: # If we prefer to keep the values of the default parameters pass - else : + else: # If we prefer to let the values of the default parameters unchanged # useful in EDSespm to keep the original metadata params[key] = d_params[key] return True + def isdict(p): r"""Return True if the variable a dictionary. - + :param p: variable to check :type p: any :return: True if p is a dictionary @@ -350,9 +392,10 @@ def isdict(p): """ return type(p) is dict -def is_symbol (i) : - r""" Return True if i is a chemical symbol - + +def is_symbol(i): + r"""Return True if i is a chemical symbol + :param i: variable to check :type i: any :return: True if i is a chemical symbol @@ -360,13 +403,14 @@ def is_symbol (i) : """ symb_list = symbol_list() - if i in symb_list : + if i in symb_list: return True - else : + else: return False -def is_number (i) : - r""" Return True if i is a number + +def is_number(i): + r"""Return True if i is a number :param i: variable to check :type i: any @@ -375,23 +419,26 @@ def is_number (i) : """ - try : + try: int(i) return True - except ValueError : + except ValueError: return False - -def symbol_list () : + + +def symbol_list(): symbol_list = [] - with open(NUMBER_PERIODIC_TABLE,"r") as f : - NPT = json.load(f)["table"] - for num in NPT.keys() : + with open(NUMBER_PERIODIC_TABLE, "r") as f: + NPT = json.load(f)["table"] + for num in NPT.keys(): symbol_list.append(NPT[num]["symbol"]) return symbol_list + def close_all(): r"""Close all opened windows.""" import matplotlib.pyplot as plt + global _qtg_widgets for widget in _qtg_widgets: widget.close() @@ -402,8 +449,9 @@ def close_all(): plt.close(fig) _plt_figures = [] -def get_explained_intensity_W(G, W, H) : - r""" Compute the explained intensity of each element of W. + +def get_explained_intensity_W(G, W, H): + r"""Compute the explained intensity of each element of W. :param np.array 2D G: G matrix of the ESpM-NMF decomposition :param np.array 2D W: W matrix of the ESpM-NMF decomposition @@ -413,12 +461,12 @@ def get_explained_intensity_W(G, W, H) : """ # I couldn't find a linear algebra trick - #int_matrix = np.zeros(W.shape) - #for i in range(W.shape[0]) : - # for j in range(W.shape[1]) : + # int_matrix = np.zeros(W.shape) + # for i in range(W.shape[0]) : + # for j in range(W.shape[1]) : # int_matrix[i,j] = np.sum(G[:,i, np.newaxis]*W[i,j]*H[np.newaxis,j,:]) - int_matrix =G.sum(0)[:,np.newaxis]*W*H.sum(1)[np.newaxis,:] #This? + int_matrix = G.sum(0)[:, np.newaxis] * W * H.sum(1)[np.newaxis, :] # This? return int_matrix @@ -431,15 +479,15 @@ def num_to_symbol(num): ---------- num : str Number to be converted to atomic symbol. E. g. "1" return "H" - + Returns ------- element : str Corresponding atomic symbol. """ - d = {str(i+1):el for i,el in enumerate(symbol_list())} + d = {str(i + 1): el for i, el in enumerate(symbol_list())} try: - return d[num.split("_")[0]]+"_"+num.split("_")[1] + return d[num.split("_")[0]] + "_" + num.split("_")[1] except: try: return d[num] @@ -447,49 +495,56 @@ def num_to_symbol(num): return num -def quant_spectrum(s1, skip_elements = []): +def quant_spectrum(s1, skip_elements=[]): r""" Performs quantification in atomic % for a single spectrum. Elements from the metadata of another spectrum can be passed. The quantification is done using SmoothNMF with one component so as to only do the fitting. - + Parameters ---------- s1 : hs.signals.Signal1D Spectrum to be quantified. Elements to be quantified should be defined in the metadata. - + Returns ------- quantification : dict A dictionary containing the atomic % of each element. s1 : hs.signals.EDS_espm - The spectrum after doing espm quantification. The object contains all espm related information. + The spectrum after doing espm quantification. The object contains all espm related information. """ s = s1.deepcopy() s.set_signal_type("EDS_espm") - + selected_elements = s.metadata.Sample.elements - selected_elements = [element for element in selected_elements if element not in skip_elements] - + selected_elements = [ + element for element in selected_elements if element not in skip_elements + ] + s.build_G() - est = espm.estimators.SmoothNMF(n_components = 1,G = s.G(),verbose=0) + est = espm.estimators.SmoothNMF(n_components=1, G=s.G(), verbose=0) with io.capture_output() as captured: - est.fit_transform(X = s1.data[:,np.newaxis], H = np.array([1.0])[:,np.newaxis]) - s.learning_results.decomposition_algorithm=est + est.fit_transform(X=s1.data[:, np.newaxis], H=np.array([1.0])[:, np.newaxis]) + s.learning_results.decomposition_algorithm = est with io.capture_output() as captured: - s.print_concentration_report(selected_elts = selected_elements) - #print(captured) - return dict([[i.split(":")[0][:-1], float(i.split(":")[1]) ]for i in captured.stdout.splitlines()[2:]]),s + s.print_concentration_report(selected_elts=selected_elements) + # print(captured) + return dict( + [ + [i.split(":")[0][:-1], float(i.split(":")[1])] + for i in captured.stdout.splitlines()[2:] + ] + ), s -def cluster_analysis_concentration_report(s,cluster_source = None,print_std=False): +def cluster_analysis_concentration_report(s, cluster_source=None, print_std=False): r""" Performs quantification in atomic % each cluster. - s must have been quantified using .elemental_mapping() method. - + s must have been quantified using .elemental_mapping() method. + Parameters ---------- s : hs.signals.Signal1D @@ -502,7 +557,7 @@ def cluster_analysis_concentration_report(s,cluster_source = None,print_std=Fals print_std : Bool If True, the method also prints the standard deviation of quantifications for each element, for each cluster. - + Returns ------- quantification : np.ndarray @@ -514,11 +569,11 @@ def cluster_analysis_concentration_report(s,cluster_source = None,print_std=Fals if cluster_source is None: ls = s.get_cluster_labels().data - - elif isinstance(cluster_source,hs.signals.Signal1D): + + elif isinstance(cluster_source, hs.signals.Signal1D): ls = cluster_source.get_cluster_labels().data - elif isinstance(cluster_source,np.ndarray): + elif isinstance(cluster_source, np.ndarray): ls = cluster_source else: @@ -526,23 +581,23 @@ def cluster_analysis_concentration_report(s,cluster_source = None,print_std=Fals qs = np.vstack([s.quantification_signal_1d.data[l].mean(0) for l in ls]).T ds = np.vstack([s.quantification_signal_1d.data[l].std(0) for l in ls]).T - qs = np.round(qs,2) - ds = np.round(ds,2) + qs = np.round(qs, 2) + ds = np.round(ds, 2) els = s.quantification_signal.metadata.Sample.elements - - print("",end="\t") - print(*["c"+str(i+1) for i in range(qs.shape[1])],sep="\t" ) - for el,q,d in zip(els,qs,ds): - print(el,end="\t") + print(end="\t") + print(*["c" + str(i + 1) for i in range(qs.shape[1])], sep="\t") + for el, q, d in zip(els, qs, ds): + print(el, end="\t") if print_std: - q_d=[str(qi)+u"\u00B1"+str(di) for qi,di in zip(q,d)] - print(*q_d,sep="\t") + q_d = [str(qi) + "\u00b1" + str(di) for qi, di in zip(q, d)] + print(*q_d, sep="\t") else: - print(*q,sep="\t") + print(*q, sep="\t") + + return qs, ds - return qs,ds def fancy_cluster_plot(s): r""" @@ -554,26 +609,29 @@ def fancy_cluster_plot(s): s should be a signal for which you have performed cluster analysis. """ - if isinstance(s,hs.signals.BaseSignal): + if isinstance(s, hs.signals.BaseSignal): data = s.get_cluster_labels().data - elif isinstance(s,np.ndarray): + elif isinstance(s, np.ndarray): data = s - if isinstance(s,np.ndarray) and len(s.shape)==2: - n = data.max()+1 - cmap = mpl.colors.ListedColormap(seaborn.color_palette("bright",n)) + if isinstance(s, np.ndarray) and len(s.shape) == 2: + n = data.max() + 1 + cmap = mpl.colors.ListedColormap(seaborn.color_palette("bright", n)) hs.signals.Signal2D(data).plot(cmap=cmap) return - - n = data.shape[0] - d = (data*np.array(range(1,n+1))[:,np.newaxis,np.newaxis]).sum(0).astype("float") - d[d==0]=np.nan - cmap = mpl.colors.ListedColormap(seaborn.color_palette("bright",n)) + d = ( + (data * np.array(range(1, n + 1))[:, np.newaxis, np.newaxis]) + .sum(0) + .astype("float") + ) + d[d == 0] = np.nan + cmap = mpl.colors.ListedColormap(seaborn.color_palette("bright", n)) hs.signals.Signal2D(d).plot(cmap=cmap) -def quant_profile_linear_fit(pf,**linfit_kwargs): + +def quant_profile_linear_fit(pf, **linfit_kwargs): r""" Performs linear fit on profile data. @@ -582,7 +640,7 @@ def quant_profile_linear_fit(pf,**linfit_kwargs): pf : list of hs.signals.Signal1D Output of s.quantification_profile - linfit_kwargs: are passed to sklearn.linear_model.LinearRegression + linfit_kwargs: are passed to sklearn.linear_model.LinearRegression """ @@ -592,46 +650,46 @@ def quant_profile_linear_fit(pf,**linfit_kwargs): n = len(pf) rows = int(np.floor(np.sqrt(n))) cols = int(np.ceil(np.sqrt(n))) - fit_results ={} - for i,p in enumerate(pf): - - ax = plt.subplot(rows,cols,i+1) - xf = p.axes_manager[0].axis + fit_results = {} + for i, p in enumerate(pf): + ax = plt.subplot(rows, cols, i + 1) + xf = p.axes_manager[0].axis yf = p.data - keep = ~(np.isnan(xf)|np.isnan(yf)) - out = lr.fit(xf[keep].reshape(-1,1), - yf[keep].reshape(-1,1)) + keep = ~(np.isnan(xf) | np.isnan(yf)) + out = lr.fit(xf[keep].reshape(-1, 1), yf[keep].reshape(-1, 1)) x = p.axes_manager[0].axis el = p.metadata.General.name - plt.plot(x,p.data,label="Profile data "+el) - ax.set_xlabel(r"profile ({})".format(p.axes_manager[0].units)) + plt.plot(x, p.data, label="Profile data " + el) + ax.set_xlabel(rf"profile ({p.axes_manager[0].units})") ax.set_ylabel("Atomic %") - ax.set_title(el+" profile") - + ax.set_title(el + " profile") a = out.coef_[0] - b=out.intercept_ - yfit = x*a+b + b = out.intercept_ + yfit = x * a + b - a,b =np.round([a,b],2) - fit_el={"slope":a,"intercept":b} - plt.plot(x,yfit, - label= r"Fit : {}$\frac{{at \%}}{{{}}}$ x+{} at%".format(a[0], - p.axes_manager[0].units,b[0])) + a, b = np.round([a, b], 2) + fit_el = {"slope": a, "intercept": b} + plt.plot( + x, + yfit, + label=rf"Fit : {a[0]}$\frac{{at \%}}{{{p.axes_manager[0].units}}}$ x+{b[0]} at%", + ) plt.legend() - fit_results[el]=fit_el + fit_results[el] = fit_el return fit_results -def radial_profile(data,center = "middle"): + +def radial_profile(data, center="middle"): if center == "middle": - center = np.array(data.shape)/2 - - y, x = np.indices((data.shape)) - r = np.sqrt((x - center[0])**2 + (y - center[1])**2) + center = np.array(data.shape) / 2 + + y, x = np.indices(data.shape) + r = np.sqrt((x - center[0]) ** 2 + (y - center[1]) ** 2) r = r.astype("int") - + keep = ~np.isnan(data.ravel()) tbin = np.bincount(r.ravel()[keep], data.ravel()[keep]) nr = np.bincount(r.ravel()[keep]) @@ -639,7 +697,8 @@ def radial_profile(data,center = "middle"): radialprofile = tbin / nr return radialprofile -def erode_masks(masks,erosion_radius=1,footprint = np.ones((3,3)),extra_safe = False): + +def erode_masks(masks, erosion_radius=1, footprint=np.ones((3, 3)), extra_safe=False): r""" Erodes masks so as to prevent phase overlap regions @@ -657,25 +716,29 @@ def erode_masks(masks,erosion_radius=1,footprint = np.ones((3,3)),extra_safe = F extra_safe: bool Perfroms erosion before opening/closing. In practice it creats a harsher erosion.""" out = masks.copy() - if out.ndim>2: - for i,m in enumerate(out): + if out.ndim > 2: + for i, m in enumerate(out): if erosion_radius > 0 and extra_safe: - m = ski.morphology.binary_erosion(m,ski.morphology.disk(erosion_radius)) - m = ski.morphology.binary_opening(m,footprint=footprint) - m = ski.morphology.binary_closing(m,footprint=footprint) - #processed = ski.morphology.binary_dilation(processed,footprint=np.ones((3,3))) + m = ski.morphology.binary_erosion( + m, ski.morphology.disk(erosion_radius) + ) + m = ski.morphology.binary_opening(m, footprint=footprint) + m = ski.morphology.binary_closing(m, footprint=footprint) + # processed = ski.morphology.binary_dilation(processed,footprint=np.ones((3,3))) if erosion_radius > 0: - m = ski.morphology.binary_erosion(m,ski.morphology.disk(erosion_radius)) - out[i]=m + m = ski.morphology.binary_erosion( + m, ski.morphology.disk(erosion_radius) + ) + out[i] = m else: - m=out + m = out if erosion_radius > 0 and extra_safe: - m = ski.morphology.binary_erosion(m,ski.morphology.disk(erosion_radius)) - m = ski.morphology.binary_opening(m,footprint=footprint) - m = ski.morphology.binary_closing(m,footprint=footprint) - #processed = ski.morphology.binary_dilation(processed,footprint=np.ones((3,3))) + m = ski.morphology.binary_erosion(m, ski.morphology.disk(erosion_radius)) + m = ski.morphology.binary_opening(m, footprint=footprint) + m = ski.morphology.binary_closing(m, footprint=footprint) + # processed = ski.morphology.binary_dilation(processed,footprint=np.ones((3,3))) if erosion_radius > 0: - m = ski.morphology.binary_erosion(m,ski.morphology.disk(erosion_radius)) - out=m - - return out + m = ski.morphology.binary_erosion(m, ski.morphology.disk(erosion_radius)) + out = m + + return out diff --git a/espm/weights/__init__.py b/espm/weights/__init__.py index 2611f4e..501a408 100644 --- a/espm/weights/__init__.py +++ b/espm/weights/__init__.py @@ -5,4 +5,4 @@ This module contains functions to generate phase distribution maps. Generate simple maps using :mod:`espm.weights.generate_weights` or more complex maps using :mod:`espm.weights.abundance`. """ -from espm.weights import generate_weights \ No newline at end of file +from espm.weights import generate_weights diff --git a/espm/weights/abundance.py b/espm/weights/abundance.py index c309914..315d848 100644 --- a/espm/weights/abundance.py +++ b/espm/weights/abundance.py @@ -5,16 +5,16 @@ The :mod:`espm.weights.abundance` module implements the class :class:`Abundance` which is used to create the weights of the phases. The weights are stored in the attribute :attr:`weights` and are normalized to 1. """ -import numpy as np -from espm.models.EDXS_function import gaussian -import scipy.ndimage as ndimage -from skimage.filters import threshold_otsu import hyperspy.api as hs -from skimage.filters import median +import numpy as np +from scipy import ndimage from scipy.interpolate import RectBivariateSpline +from skimage.filters import median, threshold_otsu -class Abundance(object): - +from espm.models.EDXS_function import gaussian + + +class Abundance: def __init__(self, shape_2d, n_phases): self.shape_2d = shape_2d self.n_phases = n_phases @@ -25,11 +25,11 @@ def __init__(self, shape_2d, n_phases): ##################### @property - def weights (self) : - #if np.sum(self._weights[:,:,0]) == 0.0 : - self._weights[:,:,0] = 1 - np.sum(self._weights[:,:,1:], axis=2) + def weights(self): + # if np.sum(self._weights[:,:,0]) == 0.0 : + self._weights[:, :, 0] = 1 - np.sum(self._weights[:, :, 1:], axis=2) return self._weights - + def check_add_weights(self, val, phase_id): r""" Check if the sum of the weights is below 1. If it is, add the new abundance. If not, print a message. @@ -40,21 +40,23 @@ def check_add_weights(self, val, phase_id): Array of the new abundance. phase_id : int Index of the phase. It has to be between 1 and n_phases-1. - + Returns ------- None. """ - s = self._weights[:,:,1:].sum(axis=2) + s = self._weights[:, :, 1:].sum(axis=2) s += val test = s <= 1 - if np.all(test) : + if np.all(test): self._weights[:, :, phase_id] += val - self.signal = hs.signals.Signal2D(np.rollaxis(self.weights,-1,0)) - else : - print("The weights contain values above 1, adding the new abundance was aborted.") + self.signal = hs.signals.Signal2D(np.rollaxis(self.weights, -1, 0)) + else: + print( + "The weights contain values above 1, adding the new abundance was aborted." + ) - def scale_phase(self,values,conc_min,conc_max) : + def scale_phase(self, values, conc_min, conc_max): r""" Scale the values of a phase between conc_min and conc_max. If the values are all zero, the function returns the values without scaling. @@ -72,12 +74,14 @@ def scale_phase(self,values,conc_min,conc_max) : scaled_values : array Array of the scaled abundance. """ - if np.max(values) == 0.0 and np.min(values) == 0.0 : - print('The abundance is zero everywhere, scaling was aborted.') + if np.max(values) == 0.0 and np.min(values) == 0.0: + print("The abundance is zero everywhere, scaling was aborted.") return values - else : - return ((values - np.min(values))/(np.max(values) - np.min(values)))*(conc_max - conc_min)+conc_min - + else: + return ((values - np.min(values)) / (np.max(values) - np.min(values))) * ( + conc_max - conc_min + ) + conc_min + ################# # Add functions # ################# @@ -100,7 +104,7 @@ def add_wedge(self, ind_origin, length, width, conc_min, conc_max, phase_id): Maximum concentration of the wedge. It has to be between 0.0 and 1.0. phase_id : int Index of the phase. It has to be between 1 and n_phases-1. - + Returns ------- None. @@ -115,7 +119,9 @@ def add_wedge(self, ind_origin, length, width, conc_min, conc_max, phase_id): >>> wedge.add_wedge((0,0), 50, 50, 0.0, 1.0, 1) >>> plt.imshow(wedge.weights[:,:,1]) """ - assert phase_id != 0, "The phase_id cannot be 0, it has to be between 1 and n_phases-1." + assert phase_id != 0, ( + "The phase_id cannot be 0, it has to be between 1 and n_phases-1." + ) if (ind_origin[0] + length <= self.shape_2d[0]) or ( ind_origin[1] + width <= self.shape_2d[1] ): @@ -130,15 +136,17 @@ def add_wedge(self, ind_origin, length, width, conc_min, conc_max, phase_id): ind_origin[0] : ind_origin[0] + length, ind_origin[1] : ind_origin[1] + width, ] = wedge - else : - print('The wedge is at least partially outside the current weights, please choose other top left coordinates.') - - scaled_values = self.scale_phase(val,conc_min,conc_max) - self.check_add_weights(scaled_values, phase_id) + else: + print( + "The wedge is at least partially outside the current weights, please choose other top left coordinates." + ) - + scaled_values = self.scale_phase(val, conc_min, conc_max) + self.check_add_weights(scaled_values, phase_id) - def add_sphere(self, ind_origin, radius, conc_max, phase_id, asym_x=1.0, asym_y=1.0): + def add_sphere( + self, ind_origin, radius, conc_max, phase_id, asym_x=1.0, asym_y=1.0 + ): r""" Function to define a sphere abundance of a defined phase. The concentration is max at centre and 0.0 on the edges. @@ -146,9 +154,9 @@ def add_sphere(self, ind_origin, radius, conc_max, phase_id, asym_x=1.0, asym_y= ---------- ind_origin : tuple of integers Coordinates of the centre of the sphere. - radius : float + radius : float Radius of the sphere in pixels. - conc_max : float + conc_max : float Maximum concentration of the sphere. phase_id : int Index of the phase @@ -156,7 +164,7 @@ def add_sphere(self, ind_origin, radius, conc_max, phase_id, asym_x=1.0, asym_y= Asymmetry of the sphere in the x direction. The default is 1.0. asym_y : float, optional Asymmetry of the sphere in the y direction. The default is 1.0. - + Returns ------- None. @@ -172,17 +180,23 @@ def add_sphere(self, ind_origin, radius, conc_max, phase_id, asym_x=1.0, asym_y= >>> plt.imshow(sphere.weights[:,:,1]) """ - assert phase_id != 0, "The phase_id cannot be 0, it has to be between 1 and n_phases-1." + assert phase_id != 0, ( + "The phase_id cannot be 0, it has to be between 1 and n_phases-1." + ) xx, yy = np.mgrid[: self.shape_2d[0], : self.shape_2d[1]] - sq_sphere = (radius**2 - ((xx - ind_origin[0])/asym_x)**2 - ((yy - ind_origin[1])/asym_y)**2) + sq_sphere = ( + radius**2 + - ((xx - ind_origin[0]) / asym_x) ** 2 + - ((yy - ind_origin[1]) / asym_y) ** 2 + ) mask = sq_sphere > 0 sphere = np.zeros(self.shape_2d) - sphere[mask] = 2*np.sqrt(sq_sphere[mask]) - scaled_sphere = self.scale_phase(sphere,0,conc_max) + sphere[mask] = 2 * np.sqrt(sq_sphere[mask]) + scaled_sphere = self.scale_phase(sphere, 0, conc_max) self.check_add_weights(scaled_sphere, phase_id) - - def add_gaussian_ripple(self, center, width, conc_max, phase_id) : + + def add_gaussian_ripple(self, center, width, conc_max, phase_id): r""" Function to define a gaussian ripple spanning over the whole length of the weights abundance of a defined phase. The concentration is max at the center and 0.0 on the edges. @@ -196,7 +210,7 @@ def add_gaussian_ripple(self, center, width, conc_max, phase_id) : Maximum concentration of the gaussian ripple. It has to be between 0.0 and 1.0. phase_id : int Index of the phase. It has to be between 1 and n_phases-1. - + Returns ------- None. @@ -212,16 +226,18 @@ def add_gaussian_ripple(self, center, width, conc_max, phase_id) : >>> plt.imshow(gaussian_ripple.weights[:,:,1]) """ - assert phase_id != 0, "The phase_id cannot be 0, it has to be between 1 and n_phases-1." + assert phase_id != 0, ( + "The phase_id cannot be 0, it has to be between 1 and n_phases-1." + ) x = np.arange(self.shape_2d[1]) - gauss_line = gaussian(x,center,width/2.355) - gaussian_ripple = np.tile(gauss_line,(self.shape_2d[0],1)) - scaled_gaussian_ripple = self.scale_phase(gaussian_ripple,0,conc_max) + gauss_line = gaussian(x, center, width / 2.355) + gaussian_ripple = np.tile(gauss_line, (self.shape_2d[0], 1)) + scaled_gaussian_ripple = self.scale_phase(gaussian_ripple, 0, conc_max) self.check_add_weights(scaled_gaussian_ripple, phase_id) - def add_laplacian(self,seed, phase_id, conc_min, conc_max,size_x = 50, size_y = 50) : + def add_laplacian(self, seed, phase_id, conc_min, conc_max, size_x=50, size_y=50): r""" - Function to generate smooth noise. The characteristic length scale of the noise variations is defined by the scale_x and scale_y parameters. + Function to generate smooth noise. The characteristic length scale of the noise variations is defined by the scale_x and scale_y parameters. Parameters ---------- @@ -237,7 +253,7 @@ def add_laplacian(self,seed, phase_id, conc_min, conc_max,size_x = 50, size_y = characteristic length scale of the noise variations in the x direction. The default is 50. size_y : int, optional characteristic length scale of the noise variations in the y direction. The default is 50. - + Returns ------- None. @@ -253,19 +269,24 @@ def add_laplacian(self,seed, phase_id, conc_min, conc_max,size_x = 50, size_y = >>> plt.imshow(laplacian.weights[:,:,1]) """ - assert phase_id != 0, "The phase_id cannot be 0, it has to be between 1 and n_phases-1." + assert phase_id != 0, ( + "The phase_id cannot be 0, it has to be between 1 and n_phases-1." + ) np.random.seed(seed) - rnd = np.random.rand(size_x,size_y) + rnd = np.random.rand(size_x, size_y) lapl = median(median(rnd)) # f = interp2d(np.arange(size_x), np.arange(size_y), lapl, kind='cubic') f = RectBivariateSpline(np.arange(size_x), np.arange(size_y), lapl.T) # For some dumb reason, the interpolation function has to have the coordinates in the opposite order - res = f(np.linspace(0,size_y-1,num = self.shape_2d[1]),np.linspace(0,size_x-1,num = self.shape_2d[0])).T + res = f( + np.linspace(0, size_y - 1, num=self.shape_2d[1]), + np.linspace(0, size_x - 1, num=self.shape_2d[0]), + ).T - scaled_res = self.scale_phase(res,conc_min,conc_max) + scaled_res = self.scale_phase(res, conc_min, conc_max) self.check_add_weights(scaled_res, phase_id) - def add_random(self,seed, phase_id, conc_min, conc_max) : + def add_random(self, seed, phase_id, conc_min, conc_max): r""" Function to generate random noise. @@ -273,13 +294,13 @@ def add_random(self,seed, phase_id, conc_min, conc_max) : ---------- seed : int Seed for the random number generator. - phase_id : int + phase_id : int Index of the phase. It has to be between 1 and n_phases-1. conc_min : float Minimum concentration. It has to be between 0.0 and 1.0. conc_max : float Maximum concentration. It has to be between 0.0 and 1.0. - + Returns ------- None. @@ -295,15 +316,17 @@ def add_random(self,seed, phase_id, conc_min, conc_max) : >>> plt.imshow(random.weights[:,:,1]) """ - assert phase_id != 0, "The phase_id cannot be 0, it has to be between 1 and n_phases-1." + assert phase_id != 0, ( + "The phase_id cannot be 0, it has to be between 1 and n_phases-1." + ) np.random.seed(seed) - rnd = np.random.rand(self.shape_2d[0],self.shape_2d[1]) - scaled_rnd = self.scale_phase(rnd,conc_min,conc_max) + rnd = np.random.rand(self.shape_2d[0], self.shape_2d[1]) + scaled_rnd = self.scale_phase(rnd, conc_min, conc_max) self.check_add_weights(scaled_rnd, phase_id) - def add_image(self, image, phase_id, conc_min, conc_max) : + def add_image(self, image, phase_id, conc_min, conc_max): r""" - Function to add a 2D numpy array as a phase. + Function to add a 2D numpy array as a phase. Parameters ---------- @@ -332,11 +355,15 @@ def add_image(self, image, phase_id, conc_min, conc_max) : >>> image.add_image(data.data, 1, 0.0, 1.0) >>> plt.imshow(image.weights[:,:,1]) """ - assert phase_id != 0, "The phase_id cannot be 0, it has to be between 1 and n_phases-1." - scaled_image = self.scale_phase(image,conc_min,conc_max) + assert phase_id != 0, ( + "The phase_id cannot be 0, it has to be between 1 and n_phases-1." + ) + scaled_image = self.scale_phase(image, conc_min, conc_max) self.check_add_weights(scaled_image, phase_id) - def add_chemical_map(self, file, element_line, conc_min,conc_max, sigma, phase_id, **kwargs) : + def add_chemical_map( + self, file, element_line, conc_min, conc_max, sigma, phase_id, **kwargs + ): r""" Function to add a chemical map extracted from a EDS spectrum image as a phase. A threshold and blurring are automatically applied to the map to suppress noise. @@ -356,40 +383,43 @@ def add_chemical_map(self, file, element_line, conc_min,conc_max, sigma, phase_i Index of the phase. It has to be between 1 and n_phases-1. **kwargs : Keyword arguments for the get_lines_intensity method of the EDS spectrum image. - + Returns ------- None. """ - assert phase_id != 0, "The phase_id cannot be 0, it has to be between 1 and n_phases-1." + assert phase_id != 0, ( + "The phase_id cannot be 0, it has to be between 1 and n_phases-1." + ) spim = hs.load(str(file)) map = spim.get_lines_intensity([element_line], **kwargs) blur = ndimage.gaussian_filter(map[0].data, sigma=sigma, order=0) - + thresh = threshold_otsu(blur) mask = np.where(blur > thresh, blur, np.zeros_like(blur)) - mask[mask>0.0] = self.scale_phase(mask[mask>0.0],conc_min,conc_max) + mask[mask > 0.0] = self.scale_phase(mask[mask > 0.0], conc_min, conc_max) self.check_add_weights(mask, phase_id) + ################################ # Old version of chemical maps # ################################ -# def chemical_maps_weights(file, element_lines, conc_max, sigma = 4, **kwargs) : +# def chemical_maps_weights(file, element_lines, conc_max, sigma = 4, **kwargs) : # spim = hs.load(str(file)) # maps = spim.get_lines_intensity(element_lines, **kwargs) # blurs = [] -# for map in maps : +# for map in maps : # blurs.append(ndimage.gaussian_filter(map.data, sigma=sigma, order=0)) # masks = [] -# for blur in blurs : +# for blur in blurs : # thresh = threshold_otsu(blur) # masks.append(np.where(blur > thresh, blur, np.zeros_like(blur))) - -# for mask in masks : + +# for mask in masks : # ind_0 = np.where(mask > 0) # min_mask = mask[ind_0] - np.min(mask[ind_0]) # norm_mask = min_mask / ((1/conc_max)* np.max(min_mask)) @@ -431,4 +461,3 @@ def add_chemical_map(self, file, element_line, conc_min,conc_max, sigma, phase_i # self.weights[:, :, phase_id] += circle # else: # print("the phases concentrations add up to more than one") - diff --git a/espm/weights/generate_weights.py b/espm/weights/generate_weights.py index 9bdd3a8..07cf22d 100644 --- a/espm/weights/generate_weights.py +++ b/espm/weights/generate_weights.py @@ -5,17 +5,19 @@ The :mod: `espm.weights.generate_weights` module implements functions to generate predetermined weights to quickly and easily generate spatial phase distributions. For a more advanced weights creation, see the :mod: `espm.weights.abundance` module. """ -import numpy as np -import matplotlib.pyplot as plt from pathlib import Path + +import hyperspy.api as hs +import matplotlib.pyplot as plt +import numpy as np + from espm.conf import BASE_PATH from espm.weights.abundance import Abundance -import hyperspy.api as hs def toy_weights(**kwargs): r"""Load the toy problem weights. - + Returns ------- Hdot : np.ndarray @@ -41,14 +43,15 @@ def toy_weights(**kwargs): im1 = plt.imread(BASE_PATH / Path("datasets/toy-problem/phase1.png")).sum(axis=-1) im2 = plt.imread(BASE_PATH / Path("datasets/toy-problem/phase2.png")).sum(axis=-1) - a = Abundance(im1.shape,3) - a.add_image(im1,1,0.0,0.5) - a.add_image(im2,2,0.0,0.5) + a = Abundance(im1.shape, 3) + a.add_image(im1, 1, 0.0, 0.5) + a.add_image(im2, 2, 0.0, 0.5) return a.weights -def random_weights(shape_2d, n_phases=3, seed=0) : - r""" Generates a random weight matrix with a uniform distribution. + +def random_weights(shape_2d, n_phases=3, seed=0): + r"""Generates a random weight matrix with a uniform distribution. The random weights are then normalized to sum up to one. @@ -60,7 +63,7 @@ def random_weights(shape_2d, n_phases=3, seed=0) : Number of phases. The default is 3. seed : int, optional Seed for the random number generator. The default is 0. - + Returns ------- weights : numpy.ndarray @@ -81,17 +84,17 @@ def random_weights(shape_2d, n_phases=3, seed=0) : ... axs[i].imshow(weights[:,:,i], cmap=plt.cm.gray_r) ... axs[i].set_title(f"Map {i+1}") >>> fig.show() - + """ a = Abundance(shape_2d, n_phases) - for i in range(1,n_phases) : - a.add_random(seed+i,i,0.0,1/n_phases) + for i in range(1, n_phases): + a.add_random(seed + i, i, 0.0, 1 / n_phases) return a.weights - -def laplacian_weights(shape_2d, n_phases=3, seed=0, size_x = 10, size_y = 10,**kwargs) : - r""" Generates a random weight matrix with a laplacian distribution. + +def laplacian_weights(shape_2d, n_phases=3, seed=0, size_x=10, size_y=10, **kwargs): + r"""Generates a random weight matrix with a laplacian distribution. The random weight are then filtered with median filter 2 times to smooth the distribution. Eventually, the result is normalized to sum up to one. @@ -104,7 +107,7 @@ def laplacian_weights(shape_2d, n_phases=3, seed=0, size_x = 10, size_y = 10,**k Number of phases. The default is 3. seed : int, optional Seed for the random number generator. The default is 0. - + Returns ------- weights : numpy.ndarray @@ -128,13 +131,14 @@ def laplacian_weights(shape_2d, n_phases=3, seed=0, size_x = 10, size_y = 10,**k >>> fig.show() """ - + a = Abundance(shape_2d, n_phases) - for i in range(1,n_phases) : - a.add_laplacian(seed+i,i,0.0,1/n_phases, size_x = size_x, size_y=size_y) + for i in range(1, n_phases): + a.add_laplacian(seed + i, i, 0.0, 1 / n_phases, size_x=size_x, size_y=size_y) return a.weights -def gaussian_ripple_weights(shape_2d, width = 1, seed = 0, **kwargs) : + +def gaussian_ripple_weights(shape_2d, width=1, seed=0, **kwargs): r""" Generate a weight matrix with a gaussian ripple of the given width and randomly centered. @@ -146,7 +150,7 @@ def gaussian_ripple_weights(shape_2d, width = 1, seed = 0, **kwargs) : Width of the gaussian ripple. The default is 1. seed : int, optional Seed for the random number generator. The default is 0. If seed is 0, the gaussian ripple is centered at half the second dimension of the weight matrix. - + Returns ------- weights : numpy.ndarray @@ -170,16 +174,23 @@ def gaussian_ripple_weights(shape_2d, width = 1, seed = 0, **kwargs) : """ a = Abundance(shape_2d, 2) - if seed == 0 : - a.add_gaussian_ripple(center = shape_2d[1]//2, width = width, conc_max= 1, phase_id=1) - else : + if seed == 0: + a.add_gaussian_ripple( + center=shape_2d[1] // 2, width=width, conc_max=1, phase_id=1 + ) + else: np.random.seed(seed) - a.add_gaussian_ripple(center=np.random.randint(1,shape_2d[1]),width=width, conc_max=1, phase_id=1) + a.add_gaussian_ripple( + center=np.random.randint(1, shape_2d[1]), + width=width, + conc_max=1, + phase_id=1, + ) return a.weights - - -def spheres_weights(shape_2d=[80, 80], n_phases=3, seed=0, radius = 1, **kwargs): + + +def spheres_weights(shape_2d=[80, 80], n_phases=3, seed=0, radius=1, **kwargs): r""" Generate a weight matrix with randomly placed spheres of the given radius. @@ -193,12 +204,12 @@ def spheres_weights(shape_2d=[80, 80], n_phases=3, seed=0, radius = 1, **kwargs Seed for the random number generator. The default is 0. radius : int, optional Radius of the spheres in pixels. The default is 1. - + Returns ------- weights : numpy.ndarray Weight matrix with spheres distribution. - + Examples -------- @@ -218,10 +229,16 @@ def spheres_weights(shape_2d=[80, 80], n_phases=3, seed=0, radius = 1, **kwargs """ a = Abundance(shape_2d, n_phases) np.random.seed(seed) - for i in range(1,n_phases) : - a.add_sphere((np.random.randint(1,shape_2d[0]),np.random.randint(1,shape_2d[1])),radius,1/(n_phases-1),i) + for i in range(1, n_phases): + a.add_sphere( + (np.random.randint(1, shape_2d[0]), np.random.randint(1, shape_2d[1])), + radius, + 1 / (n_phases - 1), + i, + ) return a.weights + def wedge_weights(shape_2d=[80, 80]): r""" Generate a weight matrix with a wedge of phase 2 and complementary phase 1. @@ -230,7 +247,7 @@ def wedge_weights(shape_2d=[80, 80]): ---------- shape_2d : tuple, optional Shape of the weight matrix. The default is [80, 80]. - + Returns ------- weights : numpy.ndarray @@ -254,11 +271,12 @@ def wedge_weights(shape_2d=[80, 80]): """ a = Abundance(shape_2d, 2) - a.add_wedge((0,0),shape_2d[0],shape_2d[1],0.0,1/2,1) - + a.add_wedge((0, 0), shape_2d[0], shape_2d[1], 0.0, 1 / 2, 1) + return a.weights -def chemical_map_weights(file = None, line_list = [], conc_list = [] , **kwargs): + +def chemical_map_weights(file=None, line_list=[], conc_list=[], **kwargs): r""" Generate a weight matrix based on an experimental EDS spectrum image. For each selected line a map is generated with the corresponding concentration. @@ -272,25 +290,30 @@ def chemical_map_weights(file = None, line_list = [], conc_list = [] , **kwargs) List of lines to be used. The default is []. If line_list is empty, the function does nothing. The lines must be in the form of a string, e.g. "Fe_Ka". conc_list : list, optional List of concentrations for each line. The default is []. If conc_list is empty, the concentration of each line is set to 1/len(line_list). - + Returns ------- weights : numpy.ndarray Weight matrix with chemical map distribution. - + """ - if conc_list == [] : - conc_list = [1/len(line_list)]*len(line_list) - if file is None or line_list==[] : - print("Please provide a file name and a list of X-ray emission lines. Nothing was done.") - else : + if conc_list == []: + conc_list = [1 / len(line_list)] * len(line_list) + if file is None or line_list == []: + print( + "Please provide a file name and a list of X-ray emission lines. Nothing was done." + ) + else: shape_2d = hs.load(file).data.shape[:-1] - a = Abundance(shape_2d, len(line_list)+1) - for i in range(len(line_list)) : - a.add_chemical_map(file,line_list[i],0.0,conc_list[i],sigma = 3,phase_id=1+i) + a = Abundance(shape_2d, len(line_list) + 1) + for i in range(len(line_list)): + a.add_chemical_map( + file, line_list[i], 0.0, conc_list[i], sigma=3, phase_id=1 + i + ) return a.weights + def generate_weights(weight_type, shape_2d, n_phases=3, seed=0, **params): r""" Generate a weight matrix with the given type. Additional parameters can be passed to the function as keyword arguments. @@ -307,27 +330,28 @@ def generate_weights(weight_type, shape_2d, n_phases=3, seed=0, **params): Seed for the random number generator. The default is 0. **params : dict Additional parameters for the weight matrix generation. See the documentation of the corresponding function for more details. - + Returns ------- weights : numpy.ndarray Weight matrix with the given type. """ - if weight_type=="random": - return random_weights(shape_2d, n_phases, seed) - elif weight_type=="laplacian": - return laplacian_weights(shape_2d, n_phases, seed,**params) - elif weight_type=="sphere": - return spheres_weights(shape_2d, n_phases, seed, **params) - elif weight_type == "gaussian_ripple" : - return gaussian_ripple_weights(shape_2d = shape_2d, seed = seed,**params) - elif weight_type=="wedge": + if weight_type == "random": + return random_weights(shape_2d, n_phases, seed) + elif weight_type == "laplacian": + return laplacian_weights(shape_2d, n_phases, seed, **params) + elif weight_type == "sphere": + return spheres_weights(shape_2d, n_phases, seed, **params) + elif weight_type == "gaussian_ripple": + return gaussian_ripple_weights(shape_2d=shape_2d, seed=seed, **params) + elif weight_type == "wedge": return wedge_weights(shape_2d=shape_2d) - elif weight_type=="toy_problem": + elif weight_type == "toy_problem": return toy_weights() - elif weight_type=="chemical_map": + elif weight_type == "chemical_map": return chemical_map_weights(**params) else: - raise ValueError("Wrong weight_type: {}. Accepted types : random, laplacian, sphere, gaussian_ripple, wedge, toy_problem, chemical_map".format(weight_type)) - \ No newline at end of file + raise ValueError( + f"Wrong weight_type: {weight_type}. Accepted types : random, laplacian, sphere, gaussian_ripple, wedge, toy_problem, chemical_map" + ) diff --git a/notebooks/EDS_spectrum_fitting.ipynb b/notebooks/EDS_spectrum_fitting.ipynb index 7a4bde8..53e840f 100644 --- a/notebooks/EDS_spectrum_fitting.ipynb +++ b/notebooks/EDS_spectrum_fitting.ipynb @@ -8,22 +8,25 @@ "source": [ "%matplotlib qt\n", "import hyperspy.api as hs\n", - "from espm.estimators import SmoothNMF\n", "import numpy as np\n", "\n", - "def expand_spectrum(spectrum) :\n", + "from espm.estimators import SmoothNMF\n", + "\n", + "\n", + "def expand_spectrum(spectrum):\n", " r\"\"\"\n", " Expand the navigation dimension of a spectrum to enable hyperspy decomposition.\n", " \"\"\"\n", " a = np.tile(spectrum.data, (2, 1))\n", " temp_spectrum = hs.signals.Signal1D(a)\n", - " temp_spectrum.set_signal_type('EDS_espm')\n", + " temp_spectrum.set_signal_type(\"EDS_espm\")\n", " d = spectrum.metadata.as_dictionary()\n", " temp_spectrum.metadata.add_dictionary(d)\n", - " temp_spectrum.axes_manager.signal_axes[0].update_from(spectrum.axes_manager.signal_axes[0], [\"name\", \"units\", \"scale\", \"offset\"])\n", + " temp_spectrum.axes_manager.signal_axes[0].update_from(\n", + " spectrum.axes_manager.signal_axes[0], [\"name\", \"units\", \"scale\", \"offset\"]\n", + " )\n", " temp_spectrum.model_ = spectrum.model\n", - " return temp_spectrum\n", - " " + " return temp_spectrum" ] }, { @@ -48,11 +51,11 @@ "outputs": [], "source": [ "# We crop the signal dimension since our model can't deal with energy scales containing 0eV.\n", - "spectrum = hs.load().isig[0.1 : 20.0].sum()\n", + "spectrum = hs.load().isig[0.1:20.0].sum()\n", "# The data are changed from ints to floats. The algorithm can only work with floats.\n", - "spectrum.change_dtype('float64')\n", + "spectrum.change_dtype(\"float64\")\n", "# To apply the methods of this package we change to object to the package type\n", - "spectrum.set_signal_type('EDS_espm')" + "spectrum.set_signal_type(\"EDS_espm\")" ] }, { @@ -92,13 +95,14 @@ "metadata": {}, "outputs": [], "source": [ - "spectrum.set_microscope_parameters(elevation_angle=35.0,\n", - " azimuth_angle= 0.0,\n", - " tilt_stage= 0.0,\n", - " beam_energy = 200,\n", - " )\n", + "spectrum.set_microscope_parameters(\n", + " elevation_angle=35.0,\n", + " azimuth_angle=0.0,\n", + " tilt_stage=0.0,\n", + " beam_energy=200,\n", + ")\n", "spectrum.metadata.Acquisition_instrument.TEM.Stage.tilt_beta = 0.0\n", - "spectrum.set_elements([ 'O', 'Sc', 'Dy'])\n" + "spectrum.set_elements([\"O\", \"Sc\", \"Dy\"])" ] }, { @@ -121,13 +125,14 @@ "metadata": {}, "outputs": [], "source": [ - "spectrum.set_analysis_parameters(thickness= 1e-5,\n", - " density = 3.5,\n", - " xray_db = '200keV_xrays.json',\n", - " width_slope = 0.01,\n", - " width_intercept = 0.065,\n", - " detector_type = 'SDD_efficiency.txt'\n", - " )" + "spectrum.set_analysis_parameters(\n", + " thickness=1e-5,\n", + " density=3.5,\n", + " xray_db=\"200keV_xrays.json\",\n", + " width_slope=0.01,\n", + " width_intercept=0.065,\n", + " detector_type=\"SDD_efficiency.txt\",\n", + ")" ] }, { @@ -198,7 +203,7 @@ "metadata": {}, "outputs": [], "source": [ - "spectrum.build_G(ignored_elements = ['Cu'],elements_dict={ 'Dy' : 3.0})" + "spectrum.build_G(ignored_elements=[\"Cu\"], elements_dict={\"Dy\": 3.0})" ] }, { @@ -222,7 +227,9 @@ "outputs": [], "source": [ "tspectrum = expand_spectrum(spectrum)\n", - "nmf = SmoothNMF(n_components=1, max_iter=1000, tol=1e-6, G = tspectrum.model, hspy_comp = True)" + "nmf = SmoothNMF(\n", + " n_components=1, max_iter=1000, tol=1e-6, G=tspectrum.model, hspy_comp=True\n", + ")" ] }, { @@ -256,7 +263,7 @@ "metadata": {}, "outputs": [], "source": [ - "tspectrum.print_concentration_report(selected_elts=['Sc', 'Dy'])" + "tspectrum.print_concentration_report(selected_elts=[\"Sc\", \"Dy\"])" ] }, { @@ -274,7 +281,7 @@ "metadata": {}, "outputs": [], "source": [ - "tspectrum.plot_1D_results(elements=['O', 'Sc', 'Dy'])" + "tspectrum.plot_1D_results(elements=[\"O\", \"Sc\", \"Dy\"])" ] }, { @@ -301,10 +308,11 @@ "source": [ "spectrum.set_analysis_parameters(geom_eff=1.0)\n", "\n", - "spectrum.set_microscope_parameters(real_time=10.0,\n", - " beam_current = 1)\n", + "spectrum.set_microscope_parameters(real_time=10.0, beam_current=1)\n", "\n", - "spectrum.estimate_mass_thickness(ignored_elements = ['Cu'],elements_dict={ 'Dy' : 3.0}, tol = 1e-10)" + "spectrum.estimate_mass_thickness(\n", + " ignored_elements=[\"Cu\"], elements_dict={\"Dy\": 3.0}, tol=1e-10\n", + ")" ] }, { @@ -344,8 +352,8 @@ "outputs": [], "source": [ "spectrum = mean_spectrum.deepcopy()\n", - "spectrum.change_dtype('float64')\n", - "spectrum.set_signal_type('EDS_espm')" + "spectrum.change_dtype(\"float64\")\n", + "spectrum.set_signal_type(\"EDS_espm\")" ] }, { diff --git a/notebooks/api.ipynb b/notebooks/api.ipynb index 43643cf..da2409b 100644 --- a/notebooks/api.ipynb +++ b/notebooks/api.ipynb @@ -29,13 +29,11 @@ "%autoreload 2\n", "%matplotlib inline\n", "\n", - "# Generic imports \n", - "import hyperspy.api as hs\n", - "import numpy as np\n", + "# Generic imports\n", "\n", "# espm imports\n", - "from espm.estimators import SmoothNMF\n", - "import espm.datasets as ds" + "import espm.datasets as ds\n", + "from espm.estimators import SmoothNMF" ] }, { @@ -59,9 +57,9 @@ "ds.generate_built_in_datasets(seeds_range=1)\n", "\n", "# Load the data\n", - "spim = ds.load_particules(sample = 0)\n", + "spim = ds.load_particules(sample=0)\n", "# We need to change the data to floats for the decomposition to run\n", - "spim.change_dtype('float64')" + "spim.change_dtype(\"float64\")" ] }, { @@ -104,7 +102,7 @@ "metadata": {}, "outputs": [], "source": [ - "est = SmoothNMF( n_components = 3, tol = 1e-6, max_iter = 200, G = spim.model,hspy_comp = True)" + "est = SmoothNMF(n_components=3, tol=1e-6, max_iter=200, G=spim.model, hspy_comp=True)" ] }, { @@ -123,7 +121,7 @@ "metadata": {}, "outputs": [], "source": [ - "out = spim.decomposition(algorithm = est, return_info=True)" + "out = spim.decomposition(algorithm=est, return_info=True)" ] }, { diff --git a/notebooks/convergence-speed.ipynb b/notebooks/convergence-speed.ipynb index 7581b7e..04fe4d8 100644 --- a/notebooks/convergence-speed.ipynb +++ b/notebooks/convergence-speed.ipynb @@ -17,9 +17,8 @@ "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", "import matplotlib.pyplot as plt\n", - "\n" + "import numpy as np" ] }, { @@ -65,7 +64,8 @@ "outputs": [], "source": [ "from pathlib import Path\n", - "pathfig = Path('./figures/')\n", + "\n", + "pathfig = Path(\"./figures/\")\n", "pathfig.mkdir(exist_ok=True)" ] }, @@ -78,7 +78,7 @@ "force_simplex = True\n", "for laplacian in [False, True]:\n", " for noise in [False, True]:\n", - " # load data \n", + " # load data\n", " filename = f\"losses_{laplacian}_{noise}_{force_simplex}.npz\"\n", " data = np.load(filename, allow_pickle=True)\n", " losses = data[\"losses\"]\n", @@ -97,14 +97,13 @@ " k = H.shape[0]\n", " times = data[\"times\"]\n", " print(len(losses))\n", - " losses = np.mean(losses-l_infty.reshape(-1, 1, 1), axis=0)\n", - "\n", + " losses = np.mean(losses - l_infty.reshape(-1, 1, 1), axis=0)\n", "\n", " plt.figure(figsize=[8, 4])\n", " for loss, caption, color in zip(losses, captions, cs):\n", - " iterations = np.arange(len(loss))+1\n", - " if len(iterations)>10:\n", - " plt.plot(iterations, loss, \".-\", color=color, label=caption)\n", + " iterations = np.arange(len(loss)) + 1\n", + " if len(iterations) > 10:\n", + " plt.plot(iterations, loss, \".-\", color=color, label=caption)\n", " max_y = np.max(losses)\n", " min_y = np.min(losses)\n", " plt.ylim([min_y, max_y])\n", @@ -116,7 +115,7 @@ "\n", " fig = plt.gcf()\n", " filename = f\"convergence_{'' if laplacian else 'no_'}laplacian_{'' if noise else 'no_'}noise.pdf\"\n", - " fig.savefig(pathfig / Path(filename), bbox_inches='tight')" + " fig.savefig(pathfig / Path(filename), bbox_inches=\"tight\")" ] }, { @@ -133,12 +132,12 @@ " lower = []\n", " n_low = []\n", " for i in range(xc.shape[1]):\n", - " n_up.append(np.sum(xc[:,i]>=0))\n", - " upper.append(np.sum((xc[:,i][xc[:,i]>=0])**2))\n", - " n_low.append(np.sum(xc[:,i]<0))\n", - " lower.append(np.sum((xc[:,i][xc[:,i]<0])**2))\n", + " n_up.append(np.sum(xc[:, i] >= 0))\n", + " upper.append(np.sum((xc[:, i][xc[:, i] >= 0]) ** 2))\n", + " n_low.append(np.sum(xc[:, i] < 0))\n", + " lower.append(np.sum((xc[:, i][xc[:, i] < 0]) ** 2))\n", " return np.sqrt(np.array(lower)), np.sqrt(np.array(upper))\n", - " # return np.sqrt(np.array(lower))/np.sqrt(n_low), np.sqrt(np.array(upper))/np.sqrt(n_up)\n" + " # return np.sqrt(np.array(lower))/np.sqrt(n_low), np.sqrt(np.array(upper))/np.sqrt(n_up)" ] }, { @@ -163,7 +162,15 @@ " asymmetric_error = np.array(list(zip(lower_error, upper_error))).T\n", " X_axis = np.arange(len(captions))\n", " # plt.boxplot(data[\"times\"],positions=X_axis-0.3+shift, widths=0.2);\n", - " plt.bar(X_axis - 0.3 + shift, times, 0.2, yerr=asymmetric_error, ecolor='black', capsize=3, label = f\"{'Smooth' if laplacian else 'random'} nois{'y' if noise else 'less'} data\")\n", + " plt.bar(\n", + " X_axis - 0.3 + shift,\n", + " times,\n", + " 0.2,\n", + " yerr=asymmetric_error,\n", + " ecolor=\"black\",\n", + " capsize=3,\n", + " label=f\"{'Smooth' if laplacian else 'random'} nois{'y' if noise else 'less'} data\",\n", + " )\n", " # plt.bar(X_axis - 0.3 + shift, times, 0.2, ecolor='black', capsize=3, label = f\"{'Smooth' if laplacian else 'random'} nois{'y' if noise else 'less'} data\")\n", " shift += 0.2\n", "\n", @@ -171,12 +178,12 @@ " plt.xticks(rotation=45, ha=\"right\")\n", " plt.xlabel(\"Algorithm\")\n", " plt.ylabel(\"Seconds\")\n", - " plt.title(\"n = \"+str(int(l)))\n", + " plt.title(\"n = \" + str(int(l)))\n", " plt.legend()\n", "\n", " fig = plt.gcf()\n", " filename = f\"speed_comparison_{l}.pdf\"\n", - " fig.savefig(pathfig / Path(filename), bbox_inches='tight')" + " fig.savefig(pathfig / Path(filename), bbox_inches=\"tight\")" ] }, { @@ -210,8 +217,7 @@ "source": [ "laplacian = False\n", "noise = True\n", - "simplex_H = True\n", - "\n" + "simplex_H = True" ] }, { @@ -221,12 +227,29 @@ "outputs": [], "source": [ "from convergence import run_experiment_set\n", + "\n", "seed = 50\n", "max_iter = 1000\n", - "losses, final_losses, Ws, Hs, params, captions, gammas, l_infty, W, H, true_D, true_H, X, Xdot, times = run_experiment_set(laplacian, noise, simplex_H, seed=seed, max_iter=max_iter)\n", + "(\n", + " losses,\n", + " final_losses,\n", + " Ws,\n", + " Hs,\n", + " params,\n", + " captions,\n", + " gammas,\n", + " l_infty,\n", + " W,\n", + " H,\n", + " true_D,\n", + " true_H,\n", + " X,\n", + " Xdot,\n", + " times,\n", + ") = run_experiment_set(laplacian, noise, simplex_H, seed=seed, max_iter=max_iter)\n", "losses = np.array(losses) - l_infty.reshape(-1, 1)\n", "shape_2d = params[0][0][\"shape_2d\"]\n", - "k = H.shape[0]\n" + "k = H.shape[0]" ] }, { @@ -249,8 +272,8 @@ "# plt.figure(figsize=[15, 10])\n", "\n", "for loss, caption in zip(losses, captions):\n", - " iterations = np.arange(len(loss))+1\n", - " if len(iterations)>10:\n", + " iterations = np.arange(len(loss)) + 1\n", + " if len(iterations) > 10:\n", " plt.plot(iterations, loss, \".-\", label=caption)\n", "max_y = np.max(losses)\n", "min_y = np.min(losses)\n", @@ -286,20 +309,20 @@ "Hmat_true = true_H.reshape(k, shape_2d[0], shape_2d[1])\n", "scale = 4\n", "cmap = plt.cm.viridis\n", - "plt.figure(figsize=(scale*k,2*scale))\n", + "plt.figure(figsize=(scale * k, 2 * scale))\n", "for i in range(k):\n", - " plt.subplot(2,k,i+1)\n", + " plt.subplot(2, k, i + 1)\n", " plt.imshow(Hmat[i], cmap=cmap, vmin=0, vmax=1)\n", " plt.title(f\"Estimated H {i}\")\n", - " plt.axis('off')\n", + " plt.axis(\"off\")\n", " plt.colorbar()\n", - " plt.subplot(2,k,i+1+k)\n", + " plt.subplot(2, k, i + 1 + k)\n", " plt.imshow(Hmat_true[i], cmap=cmap, vmin=0, vmax=1)\n", " plt.title(f\"True H {i}\")\n", - " plt.axis('off')\n", + " plt.axis(\"off\")\n", " plt.colorbar()\n", "\n", - "plt.tight_layout()\n" + "plt.tight_layout()" ] }, { diff --git a/notebooks/convergence.py b/notebooks/convergence.py index d143410..3026e37 100644 --- a/notebooks/convergence.py +++ b/notebooks/convergence.py @@ -1,24 +1,25 @@ +from copy import deepcopy +from time import time + import numpy as np -from espm.conf import log_shift, dicotomy_tol, sigmaL -from espm.utils import create_laplacian_matrix +from tqdm import tqdm + +from espm.conf import dicotomy_tol, log_shift, sigmaL +from espm.datasets.base import generate_spim_sample from espm.estimators import SmoothNMF +from espm.estimators.updates import initialize_algorithms from espm.models import ToyModel -from copy import deepcopy +from espm.utils import create_laplacian_matrix from espm.weights import generate_weights as gw -from espm.datasets.base import generate_spim_sample -from espm.estimators.updates import initialize_algorithms -from tqdm import tqdm -from time import time, process_time - # global parameters global_param = dict() global_param["l2"] = False -global_param["verbose"]= 0 +global_param["verbose"] = 0 global_param["tol"] = 0 global_param["dicotomy_tol"] = dicotomy_tol global_param["debug"] = False -global_param["log_shift"] = log_shift +global_param["log_shift"] = log_shift global_param["eval_print"] = 10 global_param["hspy_comp"] = False global_param["no_stop_criterion"] = True @@ -31,33 +32,42 @@ n_poisson = 200 lambda_L = 1000 -def create_toy_problem(l = 25, k = 3, shape_2d = [10, 10], c = 10, n_poisson=200, seed=0, simplex_H=True): + +def create_toy_problem( + l=25, k=3, shape_2d=[10, 10], c=10, n_poisson=200, seed=0, simplex_H=True +): p = np.prod(shape_2d) assert len(shape_2d) == 2 np.random.seed(seed) - H = np.random.rand(k,p) + H = np.random.rand(k, p) if simplex_H: - H = H/np.sum(H, axis=0, keepdims=True) - + H = H / np.sum(H, axis=0, keepdims=True) + # G = np.random.rand(l,c) # W = np.random.rand(c,k) # D = G @ W # Let us ignore G - D = np.random.rand(l,k) + D = np.random.rand(l, k) G = None X = D @ H - Xdot = 1/n_poisson * np.random.poisson(n_poisson * X) + Xdot = 1 / n_poisson * np.random.poisson(n_poisson * X) return G, D, H, X, Xdot -def get_toy_sample(l = 25, k = 3, shape_2d = [10, 10], c = 10, n_poisson=200, seed=0): + +def get_toy_sample(l=25, k=3, shape_2d=[10, 10], c=10, n_poisson=200, seed=0): model_params = {"L": l, "C": c, "K": k, "seed": seed} # densities = np.random.uniform(0.1, 2.0, 3) densities = np.ones([k]) - misc_params = {"N": n_poisson, "seed": seed, 'densities' : densities, "model": "ToyModel"} + misc_params = { + "N": n_poisson, + "seed": seed, + "densities": densities, + "model": "ToyModel", + } toy_model = ToyModel(**model_params) toy_model.generate_phases() @@ -69,18 +79,25 @@ def get_toy_sample(l = 25, k = 3, shape_2d = [10, 10], c = 10, n_poisson=200, se # weights[:,:,0] = 1 - np.sum(weights[:,:,1:], axis=-1) # Option 2 - weights = gw.generate_weights("laplacian", shape_2d=shape_2d, n_phases=k+1, seed=seed) - weights = weights[:,:,1:] + weights = gw.generate_weights( + "laplacian", shape_2d=shape_2d, n_phases=k + 1, seed=seed + ) + weights = weights[:, :, 1:] weights = weights / np.sum(weights, axis=-1, keepdims=True) - sample = generate_spim_sample(phases, weights, model_params,misc_params, seed = seed) + sample = generate_spim_sample(phases, weights, model_params, misc_params, seed=seed) return sample -def create_laplacian_problem(l = 25, k = 3, shape_2d = [10, 10], c = 10, n_poisson=200, seed=0): - sample = get_toy_sample(l=l, k =k, shape_2d = shape_2d, c=c, n_poisson=n_poisson, seed=seed) + +def create_laplacian_problem(l=25, k=3, shape_2d=[10, 10], c=10, n_poisson=200, seed=0): + sample = get_toy_sample( + l=l, k=k, shape_2d=shape_2d, c=c, n_poisson=n_poisson, seed=seed + ) + def to_vec(X): n = X.shape[2] - return X.transpose(2,0,1).reshape(n, -1) + return X.transpose(2, 0, 1).reshape(n, -1) + D = sample["GW"].T G = sample["G"] H = to_vec(sample["H"]) @@ -96,7 +113,18 @@ def one_experiment(X, experiment_param, algo_param, global_param): simplex_H_init = True start_time = time() if simplex_H_init: - _, W0, H0 = initialize_algorithms(X, est.G, None, None, n_components=est.n_components, init=est.init, random_state=est.random_state, simplex_H=True, simplex_W=False, logshift=log_shift) + _, W0, H0 = initialize_algorithms( + X, + est.G, + None, + None, + n_components=est.n_components, + init=est.init, + random_state=est.random_state, + simplex_H=True, + simplex_W=False, + logshift=log_shift, + ) W = est.fit_transform(X, W=W0, H=H0) else: W = est.fit_transform(X) @@ -109,14 +137,16 @@ def one_experiment(X, experiment_param, algo_param, global_param): return loss, final_loss, W.copy(), H.copy(), gamma, end_time - start_time - - -def run_experiment_set(laplacian, noise, simplex_H, seed = 0, max_iter=1000, l = 25): +def run_experiment_set(laplacian, noise, simplex_H, seed=0, max_iter=1000, l=25): if laplacian: - G, D, H, X, Xdot = create_laplacian_problem(l=l, k =k, shape_2d = shape_2d, c=c, n_poisson=n_poisson, seed=seed) + G, D, H, X, Xdot = create_laplacian_problem( + l=l, k=k, shape_2d=shape_2d, c=c, n_poisson=n_poisson, seed=seed + ) else: - G, D, H, X, Xdot = create_toy_problem(l=l, k =k, shape_2d = shape_2d, c=c, n_poisson=n_poisson, seed=seed) + G, D, H, X, Xdot = create_toy_problem( + l=l, k=k, shape_2d=shape_2d, c=c, n_poisson=n_poisson, seed=seed + ) if noise: Y = X @@ -131,7 +161,7 @@ def run_experiment_set(laplacian, noise, simplex_H, seed = 0, max_iter=1000, l = experiment_param = dict() experiment_param["simplex_H"] = simplex_H experiment_param["simplex_W"] = False - experiment_param["lambda_L"] = lambda_L + experiment_param["lambda_L"] = lambda_L experiment_param["mu"] = 0 experiment_param["epsilon_reg"] = 1 experiment_param["normalize"] = False @@ -157,10 +187,10 @@ def run_experiment_set(laplacian, noise, simplex_H, seed = 0, max_iter=1000, l = else: algos = ["log_surrogate", "l2_surrogate", "bmd", "projected_gradient"] # algos = [ "bmd"] - + for algo in algos: for linesearch in [False, True]: - # for linesearch in [False]: + # for linesearch in [False]: # for sL in [sigmaL/4, sigmaL/2, sigmaL]: for sL in [sigmaL]: # algo parameters @@ -169,12 +199,14 @@ def run_experiment_set(laplacian, noise, simplex_H, seed = 0, max_iter=1000, l = algo_param["algo"] = algo # algo_param["gamma"] = sL if algo == "projected_gradient": - algo_param["gamma"] = [10000*sL, 10000*sL] + algo_param["gamma"] = [10000 * sL, 10000 * sL] else: algo_param["gamma"] = sL params.append(deepcopy([experiment_param, algo_param, global_param])) - loss, final_loss, W, H, gamma, delta_t = one_experiment(Y, experiment_param, algo_param, global_param) + loss, final_loss, W, H, gamma, delta_t = one_experiment( + Y, experiment_param, algo_param, global_param + ) times.append(delta_t) losses.append(loss) final_losses.append(final_loss) @@ -188,14 +220,31 @@ def run_experiment_set(laplacian, noise, simplex_H, seed = 0, max_iter=1000, l = times = np.array(times) i = np.argmin(final_losses) experiment_param_m = deepcopy(experiment_param) - experiment_param_m["max_iter"] = experiment_param_m["max_iter"]*3 - loss, l_infty, W, H, _, _ = one_experiment(Y, experiment_param_m, params[i][1], global_param) - np.testing.assert_array_equal(loss[:len(losses[i])], losses[i]) + experiment_param_m["max_iter"] = experiment_param_m["max_iter"] * 3 + loss, l_infty, W, H, _, _ = one_experiment( + Y, experiment_param_m, params[i][1], global_param + ) + np.testing.assert_array_equal(loss[: len(losses[i])], losses[i]) # plt.plot(loss) # plt.plot(losses[i]) # plt.yscale("log") - return losses, final_losses, Ws, Hs, params, captions, gammas, l_infty, W, H, true_D, true_H, X, Xdot, times - + return ( + losses, + final_losses, + Ws, + Hs, + params, + captions, + gammas, + l_infty, + W, + H, + true_D, + true_H, + X, + Xdot, + times, + ) def test_create_toy_problem(): @@ -203,19 +252,27 @@ def test_create_toy_problem(): shape_2d = (25, 25) l = 25 c = 10 - n_poisson=200 + n_poisson = 200 for seed in range(3): - G, D, H, Xtrue, X = create_toy_problem(shape_2d=shape_2d, k=k, l=l, c=c, n_poisson=n_poisson, seed=seed) - G2, D2, H2, Xtrue2, X2 = create_toy_problem(shape_2d=shape_2d, k=k, l=l, c=c, n_poisson=n_poisson, seed=seed) + G, D, H, Xtrue, X = create_toy_problem( + shape_2d=shape_2d, k=k, l=l, c=c, n_poisson=n_poisson, seed=seed + ) + G2, D2, H2, Xtrue2, X2 = create_toy_problem( + shape_2d=shape_2d, k=k, l=l, c=c, n_poisson=n_poisson, seed=seed + ) np.testing.assert_array_equal(G, G2) np.testing.assert_array_equal(D, D2) np.testing.assert_array_equal(H, H2) np.testing.assert_array_equal(Xtrue, Xtrue2) np.testing.assert_array_equal(X, X2) - G3, D3, H3, Xtrue3, X3 = create_laplacian_problem(shape_2d=shape_2d, k=k, l=l, c=c, n_poisson=n_poisson, seed=seed) - G4, D4, H4, Xtrue4, X4 = create_laplacian_problem(shape_2d=shape_2d, k=k, l=l, c=c, n_poisson=n_poisson, seed=seed) + G3, D3, H3, Xtrue3, X3 = create_laplacian_problem( + shape_2d=shape_2d, k=k, l=l, c=c, n_poisson=n_poisson, seed=seed + ) + G4, D4, H4, Xtrue4, X4 = create_laplacian_problem( + shape_2d=shape_2d, k=k, l=l, c=c, n_poisson=n_poisson, seed=seed + ) np.testing.assert_array_equal(G3, G4) np.testing.assert_array_equal(D3, D4) np.testing.assert_array_equal(H3, H4) @@ -226,9 +283,9 @@ def test_create_toy_problem(): assert D.shape == D3.shape == (l, k) assert H.shape == H3.shape == (k, np.prod(shape_2d)) assert Xtrue.shape == Xtrue3.shape == (l, np.prod(shape_2d)) - -if __name__ == "__main__": + +if __name__ == "__main__": # import argparse # # get parameter from command line @@ -244,7 +301,6 @@ def test_create_toy_problem(): # simplex_H = args.simplex_H # repetitions = args.repetitions - test_create_toy_problem() # Time test simplex_H = True @@ -256,7 +312,25 @@ def test_create_toy_problem(): l_infty_l = [] times_l = [] for seed in tqdm(range(repetitions), total=repetitions): - losses, final_losses, Ws, Hs, params, captions, gammas, l_infty, W, H, true_D, true_H, X, Xdot, times = run_experiment_set(laplacian, noise, simplex_H, seed=seed, max_iter=100, l=l) + ( + losses, + final_losses, + Ws, + Hs, + params, + captions, + gammas, + l_infty, + W, + H, + true_D, + true_H, + X, + Xdot, + times, + ) = run_experiment_set( + laplacian, noise, simplex_H, seed=seed, max_iter=100, l=l + ) losses_l.append(np.array(losses)) l_infty_l.append(l_infty) times_l.append(times) @@ -267,8 +341,21 @@ def test_create_toy_problem(): # Save the results filename = f"losses_{laplacian}_{noise}_{simplex_H}_{l}.npz" - np.savez(filename, losses=losses, l_infty=l_infty, params=params, captions=captions, true_D=true_D, true_H=true_H, X=X, Xdot=Xdot, H=H, W=W, gammas=gammas, times=times) - + np.savez( + filename, + losses=losses, + l_infty=l_infty, + params=params, + captions=captions, + true_D=true_D, + true_H=true_H, + X=X, + Xdot=Xdot, + H=H, + W=W, + gammas=gammas, + times=times, + ) # Convergence test simplex_H = True @@ -279,7 +366,25 @@ def test_create_toy_problem(): l_infty_l = [] times_l = [] for seed in tqdm(range(repetitions), total=repetitions): - losses, final_losses, Ws, Hs, params, captions, gammas, l_infty, W, H, true_D, true_H, X, Xdot, times = run_experiment_set(laplacian, noise, simplex_H, seed=seed, max_iter=1000, l=25) + ( + losses, + final_losses, + Ws, + Hs, + params, + captions, + gammas, + l_infty, + W, + H, + true_D, + true_H, + X, + Xdot, + times, + ) = run_experiment_set( + laplacian, noise, simplex_H, seed=seed, max_iter=1000, l=25 + ) losses_l.append(np.array(losses)) l_infty_l.append(l_infty) times_l.append(times) @@ -290,5 +395,18 @@ def test_create_toy_problem(): # Save the results filename = f"losses_{laplacian}_{noise}_{simplex_H}.npz" - np.savez(filename, losses=losses, l_infty=l_infty, params=params, captions=captions, true_D=true_D, true_H=true_H, X=X, Xdot=Xdot, H=H, W=W, gammas=gammas, times=times) - + np.savez( + filename, + losses=losses, + l_infty=l_infty, + params=params, + captions=captions, + true_D=true_D, + true_H=true_H, + X=X, + Xdot=Xdot, + H=H, + W=W, + gammas=gammas, + times=times, + ) diff --git a/notebooks/generate_data.ipynb b/notebooks/generate_data.ipynb index 4404cd3..74f3cc6 100644 --- a/notebooks/generate_data.ipynb +++ b/notebooks/generate_data.ipynb @@ -32,17 +32,16 @@ "%matplotlib inline\n", "\n", "# espm imports\n", - "from espm.datasets.base import generate_dataset\n", - "from espm.weights.generate_weights import chemical_map_weights\n", - "from espm.models.generate_EDXS_phases import generate_modular_phases\n", - "from espm.models.EDXS_function import elts_list_from_dict_list\n", - "\n", - "# Generic imports \n", + "# Generic imports\n", "from pathlib import Path\n", + "\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "import hyperspy.api as hs\n", - "\n" + "\n", + "from espm.datasets.base import generate_dataset\n", + "from espm.models.EDXS_function import elts_list_from_dict_list\n", + "from espm.models.generate_EDXS_phases import generate_modular_phases\n", + "from espm.weights.generate_weights import chemical_map_weights" ] }, { @@ -63,17 +62,20 @@ "source": [ "def get_repo_path():\n", " \"\"\"Get the path to the git repository.\n", - " \n", + "\n", " This is a bit of a hack, but it works.\n", " \"\"\"\n", " this_path = Path.cwd() / Path(\"generate_data.ipynb\")\n", " return this_path.resolve().parent.parent\n", "\n", + "\n", "# Path of the experimental dataset\n", "path = get_repo_path() / Path(\"generated_datasets/71GPa_experimental_data.hspy\")\n", "\n", - "# Creation of the weights\n", - "weights = chemical_map_weights( file = path, line_list = [\"Fe_Ka\",\"Ca_Ka\"], conc_list = [0.5,0.5])" + "# Creation of the weights\n", + "weights = chemical_map_weights(\n", + " file=path, line_list=[\"Fe_Ka\", \"Ca_Ka\"], conc_list=[0.5, 0.5]\n", + ")" ] }, { @@ -94,15 +96,15 @@ }, "outputs": [], "source": [ - "fig,axs = plt.subplots(1,3,figsize = (20,20*3))\n", - "for j in range(axs.shape[0]) :\n", - " im = axs[j].imshow(weights[:,:,j],vmin = 0, vmax = 1.0)\n", - " axs[j].tick_params(axis = \"both\",width = 0,labelbottom = False,labelleft = False) \n", - " axs[j].set_title(\"Phase {}\".format(j),fontsize = 22)\n", + "fig, axs = plt.subplots(1, 3, figsize=(20, 20 * 3))\n", + "for j in range(axs.shape[0]):\n", + " im = axs[j].imshow(weights[:, :, j], vmin=0, vmax=1.0)\n", + " axs[j].tick_params(axis=\"both\", width=0, labelbottom=False, labelleft=False)\n", + " axs[j].set_title(f\"Phase {j}\", fontsize=22)\n", "\n", "fig.subplots_adjust(right=0.84)\n", "cbar_ax = fig.add_axes([0.85, 0.47, 0.01, 0.045])\n", - "fig.colorbar(im,cax=cbar_ax)\n", + "fig.colorbar(im, cax=cbar_ax)\n", "cbar_ax.tick_params(labelsize=22)" ] }, @@ -135,52 +137,68 @@ "# Elemental concetration of each phase\n", "elts_dicts = [\n", " # Pseudo ferropericlase\n", - " {\n", - " \"Mg\" : 0.522, \"Fe\" : 0.104, \"O\" : 0.374, \"Cu\" : 0.05\n", - " },\n", + " {\"Mg\": 0.522, \"Fe\": 0.104, \"O\": 0.374, \"Cu\": 0.05},\n", " # Pseudo Ca-Perovskite\n", " {\n", - " \"Mg\" : 0.020, \"Fe\" : 0.018, \"Ca\" : 0.188, \"Si\" : 0.173, \"Al\" : 0.010, \"O\" : 0.572, \"Ti\" : 0.004, \"Cu\" : 0.05, \"Sm\" : 0.007, \"Lu\" : 0.006, \"Nd\" : 0.006 \n", + " \"Mg\": 0.020,\n", + " \"Fe\": 0.018,\n", + " \"Ca\": 0.188,\n", + " \"Si\": 0.173,\n", + " \"Al\": 0.010,\n", + " \"O\": 0.572,\n", + " \"Ti\": 0.004,\n", + " \"Cu\": 0.05,\n", + " \"Sm\": 0.007,\n", + " \"Lu\": 0.006,\n", + " \"Nd\": 0.006,\n", " },\n", " # Pseudo Bridgmanite\n", " {\n", - " \"Mg\" : 0.445, \"Fe\" : 0.035, \"Ca\" : 0.031, \"Si\" : 0.419, \"Al\" : 0.074, \"O\" : 1.136, \"Cu\" : 0.05, \"Hf\" : 0.01\n", - " }]\n", + " \"Mg\": 0.445,\n", + " \"Fe\": 0.035,\n", + " \"Ca\": 0.031,\n", + " \"Si\": 0.419,\n", + " \"Al\": 0.074,\n", + " \"O\": 1.136,\n", + " \"Cu\": 0.05,\n", + " \"Hf\": 0.01,\n", + " },\n", + "]\n", "\n", "# Parameters of the bremsstrahlung\n", "brstlg_pars = [\n", - " {\"b0\" : 0.0001629, \"b1\" : 0.0009812},\n", - " {\"b0\" : 0.0007853, \"b1\" : 0.0003658},\n", - " {\"b0\" : 0.0003458, \"b1\" : 0.0006268}\n", + " {\"b0\": 0.0001629, \"b1\": 0.0009812},\n", + " {\"b0\": 0.0007853, \"b1\": 0.0003658},\n", + " {\"b0\": 0.0003458, \"b1\": 0.0006268},\n", "]\n", "\n", "# Model parameters : energy scale, detector broadening, x-ray emission database, beam energy, absorption parameters, detector efficiency\n", "model_params = {\n", - " \"e_offset\" : 0.3,\n", - " \"e_size\" : 1980,\n", - " \"e_scale\" : 0.01,\n", - " \"width_slope\" : 0.01,\n", - " \"width_intercept\" : 0.065,\n", - " \"db_name\" : \"200keV_xrays.json\",\n", - " \"E0\" : 200,\n", - " \"params_dict\" : {\n", - " \"Abs\" : {\n", - " \"thickness\" : 100.0e-7,\n", - " \"toa\" : 35,\n", - " \"density\" : 4.5,\n", - " \"atomic_fraction\" : False\n", - " },\n", - " \"Det\" : \"SDD_efficiency.txt\"\n", - " }\n", - " }\n", + " \"e_offset\": 0.3,\n", + " \"e_size\": 1980,\n", + " \"e_scale\": 0.01,\n", + " \"width_slope\": 0.01,\n", + " \"width_intercept\": 0.065,\n", + " \"db_name\": \"200keV_xrays.json\",\n", + " \"E0\": 200,\n", + " \"params_dict\": {\n", + " \"Abs\": {\n", + " \"thickness\": 100.0e-7,\n", + " \"toa\": 35,\n", + " \"density\": 4.5,\n", + " \"atomic_fraction\": False,\n", + " },\n", + " \"Det\": \"SDD_efficiency.txt\",\n", + " },\n", + "}\n", "\n", "# miscellaneous paramaters : average detected number of X-rays per pixel, phases densities, output folder, model name, random seed\n", "data_dict = {\n", - " \"N\" : 100,\n", - " \"densities\" : [1.2,1.0,0.8],\n", - " \"data_folder\" : \"71GPa_synthetic_N100\",\n", - " \"model\" : \"EDXS\",\n", - " \"seed\" : 42\n", + " \"N\": 100,\n", + " \"densities\": [1.2, 1.0, 0.8],\n", + " \"data_folder\": \"71GPa_synthetic_N100\",\n", + " \"model\": \"EDXS\",\n", + " \"seed\": 42,\n", "}" ] }, @@ -199,11 +217,12 @@ "outputs": [], "source": [ "phases = generate_modular_phases(\n", - " elts_dicts=elts_dicts, brstlg_pars = brstlg_pars,\n", - " scales = [1, 1, 1],\n", - " model_params= model_params,\n", - " seed = 42\n", - " )\n", + " elts_dicts=elts_dicts,\n", + " brstlg_pars=brstlg_pars,\n", + " scales=[1, 1, 1],\n", + " model_params=model_params,\n", + " seed=42,\n", + ")\n", "# scales : bremsstrahlung parameters modifiers\n", "\n", "elements = elts_list_from_dict_list(elts_dicts)" @@ -225,19 +244,20 @@ }, "outputs": [], "source": [ - "fig,axs = plt.subplots(3,1,figsize = (20,15))\n", + "fig, axs = plt.subplots(3, 1, figsize=(20, 15))\n", "\n", "# Build the energy scale\n", "x = np.linspace(\n", " model_params[\"e_offset\"],\n", - " model_params[\"e_offset\"]+model_params[\"e_scale\"]*model_params[\"e_size\"],\n", - " num=model_params[\"e_size\"])\n", + " model_params[\"e_offset\"] + model_params[\"e_scale\"] * model_params[\"e_size\"],\n", + " num=model_params[\"e_size\"],\n", + ")\n", "\n", - "for j in range(axs.shape[0]) :\n", - " axs[j].plot(x,phases[j])\n", - " axs[j].set_title(\"Phase {}\".format(j),fontsize = 22)\n", + "for j in range(axs.shape[0]):\n", + " axs[j].plot(x, phases[j])\n", + " axs[j].set_title(f\"Phase {j}\", fontsize=22)\n", "\n", - "axs[-1].set_xlabel(\"Energy loss (eV)\",fontsize = 16)" + "axs[-1].set_xlabel(\"Energy loss (eV)\", fontsize=16)" ] }, { @@ -258,13 +278,15 @@ "metadata": {}, "outputs": [], "source": [ - "generate_dataset( phases = phases,\n", - " weights = weights,\n", - " model_params = model_params,\n", - " misc_params = data_dict,\n", - " base_seed=data_dict[\"seed\"],\n", - " sample_number=1,\n", - " elements = elements)" + "generate_dataset(\n", + " phases=phases,\n", + " weights=weights,\n", + " model_params=model_params,\n", + " misc_params=data_dict,\n", + " base_seed=data_dict[\"seed\"],\n", + " sample_number=1,\n", + " elements=elements,\n", + ")" ] }, { diff --git a/notebooks/papers_figures.ipynb b/notebooks/papers_figures.ipynb index c0269dd..7f8745e 100644 --- a/notebooks/papers_figures.ipynb +++ b/notebooks/papers_figures.ipynb @@ -6,18 +6,19 @@ "metadata": {}, "outputs": [], "source": [ - "from espm.weights import generate_weights as gw\n", - "from espm.models import generate_EDXS_phases as edx\n", - "from espm.datasets import base as gen\n", - "from espm.conf import DATASETS_PATH\n", - "from espm.estimators.smooth_nmf import SmoothNMF\n", - "from espm.measures import find_min_config\n", - "from sklearn.decomposition import NMF\n", + "import hyperspy.api as hs\n", "import matplotlib.pyplot as plt\n", "import numpy as np\n", - "import hyperspy.api as hs\n", "from mpl_toolkits.axes_grid1.anchored_artists import AnchoredSizeBar\n", - "from espm.utils import rescaled_DH" + "from sklearn.decomposition import NMF\n", + "\n", + "from espm.conf import DATASETS_PATH\n", + "from espm.datasets import base as gen\n", + "from espm.estimators.smooth_nmf import SmoothNMF\n", + "from espm.measures import find_min_config\n", + "from espm.models import generate_EDXS_phases as edx\n", + "from espm.utils import rescaled_DH\n", + "from espm.weights import generate_weights as gw" ] }, { @@ -36,7 +37,7 @@ "metadata": {}, "outputs": [], "source": [ - "spheres = gw.generate_weights(\"sphere\", [128,128],3,6549, radius =25.0)" + "spheres = gw.generate_weights(\"sphere\", [128, 128], 3, 6549, radius=25.0)" ] }, { @@ -53,13 +54,22 @@ "metadata": {}, "outputs": [], "source": [ - "fig, ax = plt.subplots(1,3,figsize=(15,5))\n", + "fig, ax = plt.subplots(1, 3, figsize=(15, 5))\n", "cax = fig.add_axes([0.15, 0.1, 0.2, 0.03])\n", - "for i,weight in enumerate(spheres.T):\n", - " im = ax[i].imshow(weight[:,:], cmap = 'hot', vmin = 0.0, vmax = 1.0)\n", - " ax[i].set_title(\"Map of phase {}\".format(i+1))\n", - " ax[i].tick_params(axis='both', which='both', bottom=False, top=False, labelbottom=False, right=False, left=False, labelleft=False)\n", - "fig.colorbar(im, cax=cax, orientation='horizontal')" + "for i, weight in enumerate(spheres.T):\n", + " im = ax[i].imshow(weight[:, :], cmap=\"hot\", vmin=0.0, vmax=1.0)\n", + " ax[i].set_title(f\"Map of phase {i + 1}\")\n", + " ax[i].tick_params(\n", + " axis=\"both\",\n", + " which=\"both\",\n", + " bottom=False,\n", + " top=False,\n", + " labelbottom=False,\n", + " right=False,\n", + " left=False,\n", + " labelleft=False,\n", + " )\n", + "fig.colorbar(im, cax=cax, orientation=\"horizontal\")" ] }, { @@ -80,41 +90,63 @@ "source": [ "elts_dicts = [\n", " {\n", - " \"Mg\" : 0.245, \"Fe\" : 0.035, \"Ca\" : 0.031, \"Si\" : 0.219, \"Al\" : 0.024, \"O\" : 0.436, \"Cu\" : 0.05, \"Hf\" : 0.01\n", + " \"Mg\": 0.245,\n", + " \"Fe\": 0.035,\n", + " \"Ca\": 0.031,\n", + " \"Si\": 0.219,\n", + " \"Al\": 0.024,\n", + " \"O\": 0.436,\n", + " \"Cu\": 0.05,\n", + " \"Hf\": 0.01,\n", " },\n", + " {\"Mg\": 0.522, \"Fe\": 0.104, \"O\": 0.374, \"Cu\": 0.05},\n", " {\n", - " \"Mg\" : 0.522, \"Fe\" : 0.104, \"O\" : 0.374, \"Cu\" : 0.05\n", + " \"Mg\": 0.020,\n", + " \"Fe\": 0.018,\n", + " \"Ca\": 0.188,\n", + " \"Si\": 0.173,\n", + " \"Al\": 0.010,\n", + " \"O\": 0.572,\n", + " \"Ti\": 0.004,\n", + " \"Cu\": 0.05,\n", + " \"Sm\": 0.007,\n", + " \"Lu\": 0.006,\n", + " \"Nd\": 0.006,\n", " },\n", - " {\n", - " \"Mg\" : 0.020, \"Fe\" : 0.018, \"Ca\" : 0.188, \"Si\" : 0.173, \"Al\" : 0.010, \"O\" : 0.572, \"Ti\" : 0.004, \"Cu\" : 0.05, \"Sm\" : 0.007, \"Lu\" : 0.006, \"Nd\" : 0.006 \n", - " }]\n", + "]\n", "\n", "brstlg_pars = [\n", - " {\"b0\" : 0.0003458, \"b1\" : 0.0006268},\n", - " {\"b0\" : 0.0001629, \"b1\" : 0.0009812},\n", - " {\"b0\" : 0.0007853, \"b1\" : 0.0003658}\n", + " {\"b0\": 0.0003458, \"b1\": 0.0006268},\n", + " {\"b0\": 0.0001629, \"b1\": 0.0009812},\n", + " {\"b0\": 0.0007853, \"b1\": 0.0003658},\n", "]\n", "\n", "model_params = {\n", - " \"e_offset\" : 0.3,\n", - " \"e_size\" : 1980,\n", - " \"e_scale\" : 0.01,\n", - " \"width_slope\" : 0.01,\n", - " \"width_intercept\" : 0.065,\n", - " \"db_name\" : \"200keV_xrays.json\",\n", - " \"E0\" : 200,\n", - " \"params_dict\" : {\n", - " \"Abs\" : {\n", - " \"thickness\" : 100.0e-7,\n", - " \"toa\" : 35,\n", - " \"density\" : 4.5,\n", - " \"atomic_fraction\" : False\n", - " },\n", - " \"Det\" : \"SDD_efficiency.txt\"\n", - " }\n", - " }\n", + " \"e_offset\": 0.3,\n", + " \"e_size\": 1980,\n", + " \"e_scale\": 0.01,\n", + " \"width_slope\": 0.01,\n", + " \"width_intercept\": 0.065,\n", + " \"db_name\": \"200keV_xrays.json\",\n", + " \"E0\": 200,\n", + " \"params_dict\": {\n", + " \"Abs\": {\n", + " \"thickness\": 100.0e-7,\n", + " \"toa\": 35,\n", + " \"density\": 4.5,\n", + " \"atomic_fraction\": False,\n", + " },\n", + " \"Det\": \"SDD_efficiency.txt\",\n", + " },\n", + "}\n", "\n", - "phases = edx.generate_modular_phases (elts_dicts = elts_dicts, brstlg_pars = brstlg_pars, scales = [1,1,1], model_params = model_params, seed = 0)" + "phases = edx.generate_modular_phases(\n", + " elts_dicts=elts_dicts,\n", + " brstlg_pars=brstlg_pars,\n", + " scales=[1, 1, 1],\n", + " model_params=model_params,\n", + " seed=0,\n", + ")" ] }, { @@ -131,13 +163,17 @@ "metadata": {}, "outputs": [], "source": [ - "x = np.linspace(model_params[\"e_offset\"], model_params[\"e_offset\"] + model_params[\"e_size\"]*model_params[\"e_scale\"], model_params[\"e_size\"])\n", - "fig, ax = plt.subplots(1,3,figsize=(15,5))\n", - "for i in range(3): \n", - " ax[i].plot(x,phases[i],'k')\n", - " ax[i].set_title(\"Phase {}\".format(i+1))\n", + "x = np.linspace(\n", + " model_params[\"e_offset\"],\n", + " model_params[\"e_offset\"] + model_params[\"e_size\"] * model_params[\"e_scale\"],\n", + " model_params[\"e_size\"],\n", + ")\n", + "fig, ax = plt.subplots(1, 3, figsize=(15, 5))\n", + "for i in range(3):\n", + " ax[i].plot(x, phases[i], \"k\")\n", + " ax[i].set_title(f\"Phase {i + 1}\")\n", " ax[i].set_ylabel(\"Intensity (a.u.)\")\n", - " ax[i].set_xlim([0,10])\n", + " ax[i].set_xlim([0, 10])\n", " ax[i].set_xlabel(\"Energy (keV)\")" ] }, @@ -162,16 +198,23 @@ "outputs": [], "source": [ "misc_params = {\n", - " \"data_folder\" : \"spheres_paper_N293\",\n", - " \"shape_2d\" : (128,128),\n", - " \"N\" : 293,\n", - " \"densities\" : [1.0,0.8,1.2],\n", - " \"model\" : \"EDXS\",\n", - " \"seed\" : 42\n", + " \"data_folder\": \"spheres_paper_N293\",\n", + " \"shape_2d\": (128, 128),\n", + " \"N\": 293,\n", + " \"densities\": [1.0, 0.8, 1.2],\n", + " \"model\": \"EDXS\",\n", + " \"seed\": 42,\n", "}\n", - " \n", - "gen.generate_dataset(phases, spheres, model_params, misc_params, sample_number=5, base_seed = 42,\n", - " elements = [\"O\",\"Ca\",\"Si\",\"Fe\",\"Mg\",\"Al\",\"Cu\",\"Hf\",\"Ti\",\"Sm\",\"Lu\",\"Nd\"])" + "\n", + "gen.generate_dataset(\n", + " phases,\n", + " spheres,\n", + " model_params,\n", + " misc_params,\n", + " sample_number=5,\n", + " base_seed=42,\n", + " elements=[\"O\", \"Ca\", \"Si\", \"Fe\", \"Mg\", \"Al\", \"Cu\", \"Hf\", \"Ti\", \"Sm\", \"Lu\", \"Nd\"],\n", + ")" ] }, { @@ -191,16 +234,23 @@ "outputs": [], "source": [ "misc_params = {\n", - " \"data_folder\" : \"spheres_paper_N18\",\n", - " \"shape_2d\" : (128,128),\n", - " \"N\" : 18,\n", - " \"densities\" : [1.0,0.8,1.2],\n", - " \"model\" : \"EDXS\",\n", - " \"seed\" : 42\n", + " \"data_folder\": \"spheres_paper_N18\",\n", + " \"shape_2d\": (128, 128),\n", + " \"N\": 18,\n", + " \"densities\": [1.0, 0.8, 1.2],\n", + " \"model\": \"EDXS\",\n", + " \"seed\": 42,\n", "}\n", - " \n", - "gen.generate_dataset(phases, spheres, model_params, misc_params, sample_number=5, base_seed = 42,\n", - " elements = [\"O\",\"Ca\",\"Si\",\"Fe\",\"Mg\",\"Al\",\"Cu\",\"Hf\",\"Ti\",\"Sm\",\"Lu\",\"Nd\"])" + "\n", + "gen.generate_dataset(\n", + " phases,\n", + " spheres,\n", + " model_params,\n", + " misc_params,\n", + " sample_number=5,\n", + " base_seed=42,\n", + " elements=[\"O\", \"Ca\", \"Si\", \"Fe\", \"Mg\", \"Al\", \"Cu\", \"Hf\", \"Ti\", \"Sm\", \"Lu\", \"Nd\"],\n", + ")" ] }, { @@ -265,19 +315,21 @@ "metadata": {}, "outputs": [], "source": [ - "espm_nmf = SmoothNMF(n_components=3,\n", - " G = espm_decomp_spim.model,\n", - " max_iter=100000,\n", - " simplex_W = True,\n", - " tol=1e-8,\n", - " init = 'nndsvdar',\n", - " random_state = 42,\n", - " mu = 0.004,\n", - " epsilon_reg = 0.01,\n", - " lambda_L = 1.0,\n", - " shape_2d = (128,128),\n", - " normalize = True,\n", - " hspy_comp = True)" + "espm_nmf = SmoothNMF(\n", + " n_components=3,\n", + " G=espm_decomp_spim.model,\n", + " max_iter=100000,\n", + " simplex_W=True,\n", + " tol=1e-8,\n", + " init=\"nndsvdar\",\n", + " random_state=42,\n", + " mu=0.004,\n", + " epsilon_reg=0.01,\n", + " lambda_L=1.0,\n", + " shape_2d=(128, 128),\n", + " normalize=True,\n", + " hspy_comp=True,\n", + ")" ] }, { @@ -294,7 +346,7 @@ "metadata": {}, "outputs": [], "source": [ - "espm_decomp_spim.decomposition(algorithm = espm_nmf)" + "espm_decomp_spim.decomposition(algorithm=espm_nmf)" ] }, { @@ -311,13 +363,15 @@ "metadata": {}, "outputs": [], "source": [ - "sk_nmf = NMF(n_components=3,\n", - " init='nndsvdar',\n", - " max_iter=10000,\n", - " tol=1e-8,\n", - " random_state = 42,\n", - " beta_loss='kullback-leibler',\n", - " solver='mu')" + "sk_nmf = NMF(\n", + " n_components=3,\n", + " init=\"nndsvdar\",\n", + " max_iter=10000,\n", + " tol=1e-8,\n", + " random_state=42,\n", + " beta_loss=\"kullback-leibler\",\n", + " solver=\"mu\",\n", + ")" ] }, { @@ -334,7 +388,7 @@ "metadata": {}, "outputs": [], "source": [ - "scikit_decomp_spim.decomposition(algorithm = sk_nmf)" + "scikit_decomp_spim.decomposition(algorithm=sk_nmf)" ] }, { @@ -353,19 +407,29 @@ "metadata": {}, "outputs": [], "source": [ - "espm_decomp_maps = espm_decomp_spim.get_decomposition_loadings().data.reshape((3,128*128))\n", + "espm_decomp_maps = espm_decomp_spim.get_decomposition_loadings().data.reshape(\n", + " (3, 128 * 128)\n", + ")\n", "espm_decomp_spectra = espm_decomp_spim.get_decomposition_factors().data\n", "\n", - "sk_decomp_maps = scikit_decomp_spim.get_decomposition_loadings().data.reshape((3,128*128))\n", + "sk_decomp_maps = scikit_decomp_spim.get_decomposition_loadings().data.reshape(\n", + " (3, 128 * 128)\n", + ")\n", "sk_decomp_spectra = scikit_decomp_spim.get_decomposition_factors().data\n", "\n", "true_maps = espm_decomp_spim.maps\n", "true_spectra = espm_decomp_spim.phases.T\n", "\n", - "espm_config = find_min_config(true_maps, true_spectra, espm_decomp_maps, espm_decomp_spectra)[2]\n", - "sk_config = find_min_config(true_maps, true_spectra, sk_decomp_maps, sk_decomp_spectra)[2]\n", + "espm_config = find_min_config(\n", + " true_maps, true_spectra, espm_decomp_maps, espm_decomp_spectra\n", + ")[2]\n", + "sk_config = find_min_config(true_maps, true_spectra, sk_decomp_maps, sk_decomp_spectra)[\n", + " 2\n", + "]\n", "\n", - "espm_decomp_spectra, espm_decomp_maps = rescaled_DH(espm_decomp_spectra.T, espm_decomp_maps)\n", + "espm_decomp_spectra, espm_decomp_maps = rescaled_DH(\n", + " espm_decomp_spectra.T, espm_decomp_maps\n", + ")\n", "sk_decomp_spectra, sk_decomp_maps = rescaled_DH(sk_decomp_spectra.T, sk_decomp_maps)\n", "\n", "espm_decomp_spectra = espm_decomp_spectra.T\n", @@ -386,29 +450,74 @@ "metadata": {}, "outputs": [], "source": [ - "fig, axs = plt.subplots(3,4,figsize=(15,9), width_ratios=[1,1,1,3])\n", + "fig, axs = plt.subplots(3, 4, figsize=(15, 9), width_ratios=[1, 1, 1, 3])\n", "cax = fig.add_axes([0.23, 0.08, 0.2, 0.01])\n", - "for i in range(3) : \n", - " im = axs[i,0].imshow(true_maps[i].reshape((128,128)),cmap='hot', vmin = 0, vmax = 1, aspect = 'auto')\n", - " axs[i,0].tick_params(axis='both', which='both', bottom=False, top=False, labelbottom=False, right=False, left=False, labelleft=False)\n", - " axs[i,1].tick_params(axis='both', which='both', bottom=False, top=False, labelbottom=False, right=False, left=False, labelleft=False)\n", - " axs[i,2].tick_params(axis='both', which='both', bottom=False, top=False, labelbottom=False, right=False, left=False, labelleft=False)\n", - " axs[0,0].set_title(\"Ground truth\")\n", - " axs[i,0].set_ylabel(\"Phase {}\".format(i+1))\n", - " axs[i,2].imshow(espm_decomp_maps[espm_config[i]].reshape((128,128)),cmap='hot',vmin = 0, vmax = 1.0, aspect = 'auto')\n", - " axs[0,1].set_title(\"Scikit decomposition\")\n", - " axs[i,1].imshow(sk_decomp_maps[sk_config[i]].reshape((128,128)),cmap='hot',vmin = 0, vmax = 1.0, aspect = 'auto')\n", - " axs[0,2].set_title(\"espm decomposition\")\n", - " axs[i,3].plot(x,true_spectra[i] + 10,'k', label = 'Ground truth')\n", - " axs[i,3].plot(x,sk_decomp_spectra[sk_config[i]] + 10,'r--', label = 'Scikit decomposition')\n", - " axs[i,3].plot(x,true_spectra[i],'k')\n", - " axs[i,3].plot(x,espm_decomp_spectra[espm_config[i]],'b--', label = 'espm decomposition')\n", - " axs[i,3].set_xlim([0,10])\n", - " axs[i,3].tick_params(axis = 'y', which = 'both', left = False, labelleft = False)\n", - " axs[i,3].set_ylabel(\"Intensity (a.u.)\")\n", - " axs[0,3].legend()\n", - " axs[2,3].set_xlabel(\"Energy (keV)\")\n", - "fig.colorbar(im, cax=cax, orientation='horizontal')" + "for i in range(3):\n", + " im = axs[i, 0].imshow(\n", + " true_maps[i].reshape((128, 128)), cmap=\"hot\", vmin=0, vmax=1, aspect=\"auto\"\n", + " )\n", + " axs[i, 0].tick_params(\n", + " axis=\"both\",\n", + " which=\"both\",\n", + " bottom=False,\n", + " top=False,\n", + " labelbottom=False,\n", + " right=False,\n", + " left=False,\n", + " labelleft=False,\n", + " )\n", + " axs[i, 1].tick_params(\n", + " axis=\"both\",\n", + " which=\"both\",\n", + " bottom=False,\n", + " top=False,\n", + " labelbottom=False,\n", + " right=False,\n", + " left=False,\n", + " labelleft=False,\n", + " )\n", + " axs[i, 2].tick_params(\n", + " axis=\"both\",\n", + " which=\"both\",\n", + " bottom=False,\n", + " top=False,\n", + " labelbottom=False,\n", + " right=False,\n", + " left=False,\n", + " labelleft=False,\n", + " )\n", + " axs[0, 0].set_title(\"Ground truth\")\n", + " axs[i, 0].set_ylabel(f\"Phase {i + 1}\")\n", + " axs[i, 2].imshow(\n", + " espm_decomp_maps[espm_config[i]].reshape((128, 128)),\n", + " cmap=\"hot\",\n", + " vmin=0,\n", + " vmax=1.0,\n", + " aspect=\"auto\",\n", + " )\n", + " axs[0, 1].set_title(\"Scikit decomposition\")\n", + " axs[i, 1].imshow(\n", + " sk_decomp_maps[sk_config[i]].reshape((128, 128)),\n", + " cmap=\"hot\",\n", + " vmin=0,\n", + " vmax=1.0,\n", + " aspect=\"auto\",\n", + " )\n", + " axs[0, 2].set_title(\"espm decomposition\")\n", + " axs[i, 3].plot(x, true_spectra[i] + 10, \"k\", label=\"Ground truth\")\n", + " axs[i, 3].plot(\n", + " x, sk_decomp_spectra[sk_config[i]] + 10, \"r--\", label=\"Scikit decomposition\"\n", + " )\n", + " axs[i, 3].plot(x, true_spectra[i], \"k\")\n", + " axs[i, 3].plot(\n", + " x, espm_decomp_spectra[espm_config[i]], \"b--\", label=\"espm decomposition\"\n", + " )\n", + " axs[i, 3].set_xlim([0, 10])\n", + " axs[i, 3].tick_params(axis=\"y\", which=\"both\", left=False, labelleft=False)\n", + " axs[i, 3].set_ylabel(\"Intensity (a.u.)\")\n", + " axs[0, 3].legend()\n", + " axs[2, 3].set_xlabel(\"Energy (keV)\")\n", + "fig.colorbar(im, cax=cax, orientation=\"horizontal\")" ] }, { @@ -470,18 +579,20 @@ "metadata": {}, "outputs": [], "source": [ - "espm_nmf_hn = SmoothNMF(n_components=3,\n", - " G = espm_decomp_spim_hn.model,\n", - " max_iter=10000,\n", - " tol=1e-8,\n", - " init = 'random',\n", - " random_state = 294,\n", - " mu = 0.000,\n", - " epsilon_reg = 1.0,\n", - " lambda_L = 0.0,\n", - " normalize = True,\n", - " simplex_W= False,\n", - " hspy_comp = True)" + "espm_nmf_hn = SmoothNMF(\n", + " n_components=3,\n", + " G=espm_decomp_spim_hn.model,\n", + " max_iter=10000,\n", + " tol=1e-8,\n", + " init=\"random\",\n", + " random_state=294,\n", + " mu=0.000,\n", + " epsilon_reg=1.0,\n", + " lambda_L=0.0,\n", + " normalize=True,\n", + " simplex_W=False,\n", + " hspy_comp=True,\n", + ")" ] }, { @@ -498,7 +609,7 @@ "metadata": {}, "outputs": [], "source": [ - "espm_decomp_spim_hn.decomposition(algorithm = espm_nmf_hn)" + "espm_decomp_spim_hn.decomposition(algorithm=espm_nmf_hn)" ] }, { @@ -515,12 +626,15 @@ "metadata": {}, "outputs": [], "source": [ - "sk_nmf_hn = NMF(n_components=3,\n", - " init='random',\n", - " max_iter=10000,tol=1e-8,\n", - " random_state = 294,\n", - " beta_loss='kullback-leibler',\n", - " solver='mu')" + "sk_nmf_hn = NMF(\n", + " n_components=3,\n", + " init=\"random\",\n", + " max_iter=10000,\n", + " tol=1e-8,\n", + " random_state=294,\n", + " beta_loss=\"kullback-leibler\",\n", + " solver=\"mu\",\n", + ")" ] }, { @@ -537,7 +651,7 @@ "metadata": {}, "outputs": [], "source": [ - "scikit_decomp_spim_hn.decomposition(algorithm = sk_nmf_hn)" + "scikit_decomp_spim_hn.decomposition(algorithm=sk_nmf_hn)" ] }, { @@ -554,20 +668,36 @@ "metadata": {}, "outputs": [], "source": [ - "espm_decomp_maps_hn = espm_decomp_spim_hn.get_decomposition_loadings().data.reshape((3,128*128))\n", + "espm_decomp_maps_hn = espm_decomp_spim_hn.get_decomposition_loadings().data.reshape(\n", + " (3, 128 * 128)\n", + ")\n", "espm_decomp_spectra_hn = espm_decomp_spim_hn.get_decomposition_factors().data\n", "\n", - "sk_decomp_maps_hn = scikit_decomp_spim_hn.get_decomposition_loadings().data.reshape((3,128*128))\n", + "sk_decomp_maps_hn = scikit_decomp_spim_hn.get_decomposition_loadings().data.reshape(\n", + " (3, 128 * 128)\n", + ")\n", "sk_decomp_spectra_hn = scikit_decomp_spim_hn.get_decomposition_factors().data\n", "\n", "true_maps_hn = espm_decomp_spim_hn.maps\n", "true_spectra_hn = espm_decomp_spim_hn.phases.T\n", "\n", - "espm_config_hn = find_min_config(true_maps_hn, true_spectra_hn, espm_decomp_maps_hn, espm_decomp_spectra_hn, angles = False)[2]\n", - "sk_config_hn = find_min_config(true_maps_hn, true_spectra_hn, sk_decomp_maps_hn, sk_decomp_spectra_hn, angles = False)[2]\n", - "\n", - "espm_decomp_spectra_hn, espm_decomp_maps_hn = rescaled_DH(espm_decomp_spectra_hn.T, espm_decomp_maps_hn)\n", - "sk_decomp_spectra_hn, sk_decomp_maps_hn = rescaled_DH(sk_decomp_spectra_hn.T, sk_decomp_maps_hn)\n", + "espm_config_hn = find_min_config(\n", + " true_maps_hn,\n", + " true_spectra_hn,\n", + " espm_decomp_maps_hn,\n", + " espm_decomp_spectra_hn,\n", + " angles=False,\n", + ")[2]\n", + "sk_config_hn = find_min_config(\n", + " true_maps_hn, true_spectra_hn, sk_decomp_maps_hn, sk_decomp_spectra_hn, angles=False\n", + ")[2]\n", + "\n", + "espm_decomp_spectra_hn, espm_decomp_maps_hn = rescaled_DH(\n", + " espm_decomp_spectra_hn.T, espm_decomp_maps_hn\n", + ")\n", + "sk_decomp_spectra_hn, sk_decomp_maps_hn = rescaled_DH(\n", + " sk_decomp_spectra_hn.T, sk_decomp_maps_hn\n", + ")\n", "\n", "espm_decomp_spectra_hn = espm_decomp_spectra_hn.T\n", "sk_decomp_spectra_hn = sk_decomp_spectra_hn.T" @@ -587,29 +717,77 @@ "metadata": {}, "outputs": [], "source": [ - "fig, axs = plt.subplots(3,4,figsize=(15,9), width_ratios=[1,1,1,3])\n", + "fig, axs = plt.subplots(3, 4, figsize=(15, 9), width_ratios=[1, 1, 1, 3])\n", "cax = fig.add_axes([0.23, 0.08, 0.2, 0.01])\n", - "for i in range(3) : \n", - " im = axs[i,0].imshow(true_maps_hn[i].reshape((128,128)),cmap='hot',vmin = 0, vmax = 1.0, aspect = 'auto')\n", - " axs[i,0].tick_params(axis='both', which='both', bottom=False, top=False, labelbottom=False, right=False, left=False, labelleft=False)\n", - " axs[i,1].tick_params(axis='both', which='both', bottom=False, top=False, labelbottom=False, right=False, left=False, labelleft=False)\n", - " axs[i,2].tick_params(axis='both', which='both', bottom=False, top=False, labelbottom=False, right=False, left=False, labelleft=False)\n", - " axs[0,0].set_title(\"Ground truth\")\n", - " axs[i,0].set_ylabel(\"Phase {}\".format(i+1))\n", - " axs[i,2].imshow(espm_decomp_maps_hn[espm_config_hn[i]].reshape((128,128)),cmap='hot',vmin = 0, vmax = 1.0, aspect = 'auto')\n", - " axs[0,1].set_title(\"Scikit decomposition\")\n", - " axs[i,1].imshow(sk_decomp_maps_hn[sk_config_hn[i]].reshape((128,128)),cmap='hot',vmin = 0, vmax = 1.0, aspect = 'auto')\n", - " axs[0,2].set_title(\"espm decomposition\")\n", - " axs[i,3].plot(x,true_spectra_hn[i] + 2,'k', label = 'Ground truth')\n", - " axs[i,3].plot(x,sk_decomp_spectra_hn[sk_config_hn[i]] + 2,'r--', label = 'Scikit decomposition')\n", - " axs[i,3].plot(x,true_spectra_hn[i],'k')\n", - " axs[i,3].plot(x,espm_decomp_spectra_hn[espm_config_hn[i]],'b--', label = 'espm decomposition')\n", - " axs[i,3].set_xlim([0,10])\n", - " axs[i,3].tick_params(axis = 'y', which = 'both', left = False, labelleft = False)\n", - " axs[i,3].set_ylabel(\"Intensity (a.u.)\")\n", - " axs[0,3].legend()\n", - " axs[2,3].set_xlabel(\"Energy (keV)\")\n", - "fig.colorbar(im, cax=cax, orientation='horizontal')" + "for i in range(3):\n", + " im = axs[i, 0].imshow(\n", + " true_maps_hn[i].reshape((128, 128)), cmap=\"hot\", vmin=0, vmax=1.0, aspect=\"auto\"\n", + " )\n", + " axs[i, 0].tick_params(\n", + " axis=\"both\",\n", + " which=\"both\",\n", + " bottom=False,\n", + " top=False,\n", + " labelbottom=False,\n", + " right=False,\n", + " left=False,\n", + " labelleft=False,\n", + " )\n", + " axs[i, 1].tick_params(\n", + " axis=\"both\",\n", + " which=\"both\",\n", + " bottom=False,\n", + " top=False,\n", + " labelbottom=False,\n", + " right=False,\n", + " left=False,\n", + " labelleft=False,\n", + " )\n", + " axs[i, 2].tick_params(\n", + " axis=\"both\",\n", + " which=\"both\",\n", + " bottom=False,\n", + " top=False,\n", + " labelbottom=False,\n", + " right=False,\n", + " left=False,\n", + " labelleft=False,\n", + " )\n", + " axs[0, 0].set_title(\"Ground truth\")\n", + " axs[i, 0].set_ylabel(f\"Phase {i + 1}\")\n", + " axs[i, 2].imshow(\n", + " espm_decomp_maps_hn[espm_config_hn[i]].reshape((128, 128)),\n", + " cmap=\"hot\",\n", + " vmin=0,\n", + " vmax=1.0,\n", + " aspect=\"auto\",\n", + " )\n", + " axs[0, 1].set_title(\"Scikit decomposition\")\n", + " axs[i, 1].imshow(\n", + " sk_decomp_maps_hn[sk_config_hn[i]].reshape((128, 128)),\n", + " cmap=\"hot\",\n", + " vmin=0,\n", + " vmax=1.0,\n", + " aspect=\"auto\",\n", + " )\n", + " axs[0, 2].set_title(\"espm decomposition\")\n", + " axs[i, 3].plot(x, true_spectra_hn[i] + 2, \"k\", label=\"Ground truth\")\n", + " axs[i, 3].plot(\n", + " x,\n", + " sk_decomp_spectra_hn[sk_config_hn[i]] + 2,\n", + " \"r--\",\n", + " label=\"Scikit decomposition\",\n", + " )\n", + " axs[i, 3].plot(x, true_spectra_hn[i], \"k\")\n", + " axs[i, 3].plot(\n", + " x, espm_decomp_spectra_hn[espm_config_hn[i]], \"b--\", label=\"espm decomposition\"\n", + " )\n", + " axs[i, 3].set_xlim([0, 10])\n", + " axs[i, 3].tick_params(axis=\"y\", which=\"both\", left=False, labelleft=False)\n", + " axs[i, 3].set_ylabel(\"Intensity (a.u.)\")\n", + " axs[0, 3].legend()\n", + " axs[2, 3].set_xlabel(\"Energy (keV)\")\n", + "fig.colorbar(im, cax=cax, orientation=\"horizontal\")" ] }, { @@ -663,10 +841,14 @@ "metadata": {}, "outputs": [], "source": [ - "#Download the data (1MB)\n", - "from urllib.request import urlretrieve, urlopen\n", + "# Download the data (1MB)\n", + "from urllib.request import urlretrieve\n", "from zipfile import ZipFile\n", - "files = urlretrieve(\"https://www.dropbox.com/s/ecdlgwxjq04m5mx/HyperSpy_demos_EDS_TEM_files.zip?raw=1\", \"./HyperSpy_demos_EDX_TEM_files.zip\")\n", + "\n", + "files = urlretrieve(\n", + " \"https://www.dropbox.com/s/ecdlgwxjq04m5mx/HyperSpy_demos_EDS_TEM_files.zip?raw=1\",\n", + " \"./HyperSpy_demos_EDX_TEM_files.zip\",\n", + ")\n", "with ZipFile(\"HyperSpy_demos_EDX_TEM_files.zip\") as z:\n", " z.extractall()" ] @@ -694,54 +876,54 @@ "outputs": [], "source": [ "# Loading\n", - "core_shell_i = hs.load('core_shell.hdf5', reader = 'hspy')\n", - "core_shell_f = hs.load('core_shell.hdf5', reader = 'hspy')\n", - "bare_core = hs.load('bare_core.hdf5', reader = 'hspy')\n", + "core_shell_i = hs.load(\"core_shell.hdf5\", reader=\"hspy\")\n", + "core_shell_f = hs.load(\"core_shell.hdf5\", reader=\"hspy\")\n", + "bare_core = hs.load(\"bare_core.hdf5\", reader=\"hspy\")\n", "\n", "# Setting signal type and dtype\n", - "core_shell_i.set_signal_type('EDS_espm')\n", - "core_shell_f.set_signal_type('EDS_espm')\n", - "bare_core.set_signal_type('EDS_espm')\n", + "core_shell_i.set_signal_type(\"EDS_espm\")\n", + "core_shell_f.set_signal_type(\"EDS_espm\")\n", + "bare_core.set_signal_type(\"EDS_espm\")\n", "\n", - "core_shell_i.change_dtype('float')\n", - "core_shell_f.change_dtype('float')\n", - "bare_core.change_dtype('float')\n", + "core_shell_i.change_dtype(\"float\")\n", + "core_shell_f.change_dtype(\"float\")\n", + "bare_core.change_dtype(\"float\")\n", "\n", "# Setting the metadata\n", "core_shell_i.metadata.Acquisition_instrument.TEM.Stage.tilt_beta = 0.0\n", "core_shell_i.set_analysis_parameters(\n", " thickness=2e-05,\n", " density=5.2,\n", - " detector_type='SDD_efficiency.txt',\n", + " detector_type=\"SDD_efficiency.txt\",\n", " width_slope=0.01,\n", " width_intercept=0.065,\n", - " xray_db='200keV_xrays.json'\n", - " )\n", - "core_shell_i.set_elements(elements = ['Fe', 'Pt', 'O', 'Si', 'C'])\n", + " xray_db=\"200keV_xrays.json\",\n", + ")\n", + "core_shell_i.set_elements(elements=[\"Fe\", \"Pt\", \"O\", \"Si\", \"C\"])\n", "\n", "\n", "core_shell_f.metadata.Acquisition_instrument.TEM.Stage.tilt_beta = 0.0\n", "core_shell_f.set_analysis_parameters(\n", " thickness=0.3e-05,\n", " density=5.2,\n", - " detector_type='SDD_efficiency.txt',\n", + " detector_type=\"SDD_efficiency.txt\",\n", " width_slope=0.01,\n", " width_intercept=0.065,\n", - " xray_db='200keV_xrays.json'\n", - " )\n", - "core_shell_f.set_elements(elements = ['Fe', 'Pt', 'O', 'Si', 'C'])\n", + " xray_db=\"200keV_xrays.json\",\n", + ")\n", + "core_shell_f.set_elements(elements=[\"Fe\", \"Pt\", \"O\", \"Si\", \"C\"])\n", "\n", "\n", "bare_core.metadata.Acquisition_instrument.TEM.Stage.tilt_beta = 0.0\n", "bare_core.set_analysis_parameters(\n", " thickness=0.3e-05,\n", " density=5.2,\n", - " detector_type='SDD_efficiency.txt',\n", + " detector_type=\"SDD_efficiency.txt\",\n", " width_slope=0.01,\n", " width_intercept=0.065,\n", - " xray_db='200keV_xrays.json'\n", - " )\n", - "bare_core.set_elements(elements = ['Fe', 'Pt', 'O', 'Si', 'C'])\n" + " xray_db=\"200keV_xrays.json\",\n", + ")\n", + "bare_core.set_elements(elements=[\"Fe\", \"Pt\", \"O\", \"Si\", \"C\"])" ] }, { @@ -763,7 +945,7 @@ "metadata": {}, "outputs": [], "source": [ - "bare_core.build_G('bremsstrahlung', elements_dict = {'Fe' : 4.0, 'Pt' : 4.0})" + "bare_core.build_G(\"bremsstrahlung\", elements_dict={\"Fe\": 4.0, \"Pt\": 4.0})" ] }, { @@ -781,9 +963,7 @@ "metadata": {}, "outputs": [], "source": [ - "bare_core_est = SmoothNMF(n_components=2,\n", - " G = bare_core.model,\n", - " hspy_comp = True)" + "bare_core_est = SmoothNMF(n_components=2, G=bare_core.model, hspy_comp=True)" ] }, { @@ -799,7 +979,7 @@ "metadata": {}, "outputs": [], "source": [ - "bare_core.decomposition(algorithm = bare_core_est)" + "bare_core.decomposition(algorithm=bare_core_est)" ] }, { @@ -823,33 +1003,47 @@ "# energy scale\n", "x = np.linspace(\n", " bare_core.axes_manager.signal_axes[0].offset,\n", - " bare_core.axes_manager.signal_axes[0].offset + bare_core.axes_manager.signal_axes[0].scale * bare_core.axes_manager.signal_axes[0].size,\n", - " bare_core.axes_manager.signal_axes[0].size\n", + " bare_core.axes_manager.signal_axes[0].offset\n", + " + bare_core.axes_manager.signal_axes[0].scale\n", + " * bare_core.axes_manager.signal_axes[0].size,\n", + " bare_core.axes_manager.signal_axes[0].size,\n", ")\n", "\n", "\n", - "fig, axs = plt.subplots(2,2,figsize=(15,9), width_ratios=[1,3])\n", + "fig, axs = plt.subplots(2, 2, figsize=(15, 9), width_ratios=[1, 3])\n", "cax = fig.add_axes([0.13, 0.08, 0.17, 0.01])\n", - "for i in range(2) : \n", - " im = axs[i,0].imshow(bare_core_maps.data[i],cmap='hot',vmin = 0, aspect = 'auto')\n", - " axs[i,1].plot(x, bare_core_spectra.data[i])\n", - " axs[i,0].tick_params(axis='both', which='both', bottom=False, top=False, labelbottom=False, right=False, left=False, labelleft=False)\n", - " axs[i,1].set_xlim([0,14])\n", - " axs[i,1].tick_params(axis = 'y', which = 'both', left = False, labelleft = False)\n", - " axs[i,1].set_ylabel(\"Intensity (a.u.)\")\n", - " axs[1,1].set_xlabel(\"Energy (keV)\")\n", - "\n", - "scalebar = AnchoredSizeBar(axs[1,0].transData,\n", - " 20, '{} {}'.format(20*bare_core.axes_manager[0].scale, bare_core.axes_manager[0].units),\n", - " 'lower left', \n", - " pad=0.1,\n", - " color='white',\n", - " frameon=False,\n", - " size_vertical=1)\n", + "for i in range(2):\n", + " im = axs[i, 0].imshow(bare_core_maps.data[i], cmap=\"hot\", vmin=0, aspect=\"auto\")\n", + " axs[i, 1].plot(x, bare_core_spectra.data[i])\n", + " axs[i, 0].tick_params(\n", + " axis=\"both\",\n", + " which=\"both\",\n", + " bottom=False,\n", + " top=False,\n", + " labelbottom=False,\n", + " right=False,\n", + " left=False,\n", + " labelleft=False,\n", + " )\n", + " axs[i, 1].set_xlim([0, 14])\n", + " axs[i, 1].tick_params(axis=\"y\", which=\"both\", left=False, labelleft=False)\n", + " axs[i, 1].set_ylabel(\"Intensity (a.u.)\")\n", + " axs[1, 1].set_xlabel(\"Energy (keV)\")\n", + "\n", + "scalebar = AnchoredSizeBar(\n", + " axs[1, 0].transData,\n", + " 20,\n", + " f\"{20 * bare_core.axes_manager[0].scale} {bare_core.axes_manager[0].units}\",\n", + " \"lower left\",\n", + " pad=0.1,\n", + " color=\"white\",\n", + " frameon=False,\n", + " size_vertical=1,\n", + ")\n", "\n", - "axs[1,0].add_artist(scalebar)\n", + "axs[1, 0].add_artist(scalebar)\n", "\n", - "fig.colorbar(im, cax=cax, orientation='horizontal')" + "fig.colorbar(im, cax=cax, orientation=\"horizontal\")" ] }, { @@ -889,7 +1083,7 @@ "metadata": {}, "outputs": [], "source": [ - "bare_core.print_concentration_report(selected_elts = ['Fe', 'Pt'])" + "bare_core.print_concentration_report(selected_elts=[\"Fe\", \"Pt\"])" ] }, { @@ -908,22 +1102,24 @@ "outputs": [], "source": [ "bare_core_exp = bare_core.mean()\n", - "bare_core_model = ((bare_core_spectra.data.T)@bare_core_maps.data.reshape((2,84*84))).mean(axis = 1)\n", + "bare_core_model = (\n", + " (bare_core_spectra.data.T) @ bare_core_maps.data.reshape((2, 84 * 84))\n", + ").mean(axis=1)\n", "\n", "\n", - "fig, axs = plt.subplots(1,2, figsize = (15,6))\n", - "axs[0].plot(x, bare_core_exp,'.', label = \"Experimental\")\n", - "axs[0].plot(x, bare_core_model, label = \"Model\")\n", + "fig, axs = plt.subplots(1, 2, figsize=(15, 6))\n", + "axs[0].plot(x, bare_core_exp, \".\", label=\"Experimental\")\n", + "axs[0].plot(x, bare_core_model, label=\"Model\")\n", "axs[0].set_xlabel(\"Energy (keV)\")\n", "axs[0].set_ylabel(\"Counts\")\n", "axs[0].legend()\n", - "axs[0].set_xlim(0,14.0)\n", - "axs[1].plot(x,bare_core_exp,'.')\n", - "axs[1].plot(x,bare_core_model)\n", + "axs[0].set_xlim(0, 14.0)\n", + "axs[1].plot(x, bare_core_exp, \".\")\n", + "axs[1].plot(x, bare_core_model)\n", "axs[1].set_xlabel(\"Energy (keV)\")\n", "axs[1].set_ylabel(\"Counts\")\n", - "axs[1].set_ylim(0,1.5)\n", - "axs[1].set_xlim(0,14.0)\n" + "axs[1].set_ylim(0, 1.5)\n", + "axs[1].set_xlim(0, 14.0)" ] }, { @@ -945,7 +1141,7 @@ "metadata": {}, "outputs": [], "source": [ - "core_shell_i.build_G('bremsstrahlung', elements_dict= {'Fe' : 4.0, 'Pt' : 4.0})" + "core_shell_i.build_G(\"bremsstrahlung\", elements_dict={\"Fe\": 4.0, \"Pt\": 4.0})" ] }, { @@ -963,9 +1159,9 @@ "metadata": {}, "outputs": [], "source": [ - "cs_i_est = SmoothNMF(n_components=3, simplex_W = True,\n", - " G = core_shell_i.model,\n", - " hspy_comp = True)" + "cs_i_est = SmoothNMF(\n", + " n_components=3, simplex_W=True, G=core_shell_i.model, hspy_comp=True\n", + ")" ] }, { @@ -981,7 +1177,7 @@ "metadata": {}, "outputs": [], "source": [ - "core_shell_i.decomposition(algorithm = cs_i_est)" + "core_shell_i.decomposition(algorithm=cs_i_est)" ] }, { @@ -1005,34 +1201,48 @@ "# energy scale\n", "x = np.linspace(\n", " core_shell_i.axes_manager.signal_axes[0].offset,\n", - " core_shell_i.axes_manager.signal_axes[0].offset + core_shell_i.axes_manager.signal_axes[0].scale * core_shell_i.axes_manager.signal_axes[0].size,\n", - " core_shell_i.axes_manager.signal_axes[0].size\n", + " core_shell_i.axes_manager.signal_axes[0].offset\n", + " + core_shell_i.axes_manager.signal_axes[0].scale\n", + " * core_shell_i.axes_manager.signal_axes[0].size,\n", + " core_shell_i.axes_manager.signal_axes[0].size,\n", ")\n", "\n", "\n", - "fig, axs = plt.subplots(3,2,figsize=(15,9), width_ratios=[1,3])\n", + "fig, axs = plt.subplots(3, 2, figsize=(15, 9), width_ratios=[1, 3])\n", "cax = fig.add_axes([0.13, 0.08, 0.17, 0.01])\n", - "for i in range(3) : \n", - " im = axs[i,0].imshow(cs_i_maps.data[i],cmap='hot',vmin = 0, aspect = 'auto')\n", - " axs[i,1].plot(x, cs_i_spectra.data[i])\n", - " axs[i,0].tick_params(axis='both', which='both', bottom=False, top=False, labelbottom=False, right=False, left=False, labelleft=False)\n", - " axs[i,1].set_xlim([0,14])\n", - " axs[i,1].tick_params(axis = 'y', which = 'both', left = False, labelleft = False)\n", - " axs[i,1].set_ylabel(\"Intensity (a.u.)\")\n", - " axs[1,1].set_xlabel(\"Energy (keV)\")\n", - "\n", - "scalebar = AnchoredSizeBar(axs[1,0].transData,\n", - " 20, '{} {}'.format(20*core_shell_i.axes_manager[0].scale, core_shell_i.axes_manager[0].units),\n", - " 'lower right', \n", - " pad=0.1,\n", - " color='white',\n", - " frameon=False,\n", - " size_vertical=1)\n", + "for i in range(3):\n", + " im = axs[i, 0].imshow(cs_i_maps.data[i], cmap=\"hot\", vmin=0, aspect=\"auto\")\n", + " axs[i, 1].plot(x, cs_i_spectra.data[i])\n", + " axs[i, 0].tick_params(\n", + " axis=\"both\",\n", + " which=\"both\",\n", + " bottom=False,\n", + " top=False,\n", + " labelbottom=False,\n", + " right=False,\n", + " left=False,\n", + " labelleft=False,\n", + " )\n", + " axs[i, 1].set_xlim([0, 14])\n", + " axs[i, 1].tick_params(axis=\"y\", which=\"both\", left=False, labelleft=False)\n", + " axs[i, 1].set_ylabel(\"Intensity (a.u.)\")\n", + " axs[1, 1].set_xlabel(\"Energy (keV)\")\n", + "\n", + "scalebar = AnchoredSizeBar(\n", + " axs[1, 0].transData,\n", + " 20,\n", + " f\"{20 * core_shell_i.axes_manager[0].scale} {core_shell_i.axes_manager[0].units}\",\n", + " \"lower right\",\n", + " pad=0.1,\n", + " color=\"white\",\n", + " frameon=False,\n", + " size_vertical=1,\n", + ")\n", "\n", - "axs[2,0].add_artist(scalebar)\n", + "axs[2, 0].add_artist(scalebar)\n", "\n", - "fig.colorbar(im, cax=cax, orientation='horizontal')\n", - "core_shell_i.print_concentration_report(selected_elts=['Fe', 'Pt'])" + "fig.colorbar(im, cax=cax, orientation=\"horizontal\")\n", + "core_shell_i.print_concentration_report(selected_elts=[\"Fe\", \"Pt\"])" ] }, { @@ -1069,12 +1279,15 @@ "metadata": {}, "outputs": [], "source": [ - "core_shell_f.build_G('bremsstrahlung', elements_dict= {'Fe' : 4.0, 'Pt' : 4.0})\n", + "core_shell_f.build_G(\"bremsstrahlung\", elements_dict={\"Fe\": 4.0, \"Pt\": 4.0})\n", "\n", "fW = core_shell_f.set_fixed_W(\n", - " {'background' : {'Pt' : 0.0, 'Fe' : 0.0},\n", - " 'core' : {'Pt' : 0.84, 'Fe' : 0.16, 'Si' : 0.0, 'C' : 0.0, 'O' : 0.0},\n", - " 'shell' : { 'Pt' : 0.0, 'Si' : 0.0, 'C' : 0.0}})" + " {\n", + " \"background\": {\"Pt\": 0.0, \"Fe\": 0.0},\n", + " \"core\": {\"Pt\": 0.84, \"Fe\": 0.16, \"Si\": 0.0, \"C\": 0.0, \"O\": 0.0},\n", + " \"shell\": {\"Pt\": 0.0, \"Si\": 0.0, \"C\": 0.0},\n", + " }\n", + ")" ] }, { @@ -1092,11 +1305,9 @@ "metadata": {}, "outputs": [], "source": [ - "cs_f_est = SmoothNMF(n_components=3,\n", - " G = core_shell_f.model,\n", - " fixed_W = fW,\n", - " simplex_W= True,\n", - " hspy_comp = True)" + "cs_f_est = SmoothNMF(\n", + " n_components=3, G=core_shell_f.model, fixed_W=fW, simplex_W=True, hspy_comp=True\n", + ")" ] }, { @@ -1112,7 +1323,7 @@ "metadata": {}, "outputs": [], "source": [ - "core_shell_f.decomposition(algorithm = cs_f_est)" + "core_shell_f.decomposition(algorithm=cs_f_est)" ] }, { @@ -1145,33 +1356,47 @@ "# energy scale\n", "x = np.linspace(\n", " core_shell_f.axes_manager.signal_axes[0].offset,\n", - " core_shell_f.axes_manager.signal_axes[0].offset + core_shell_f.axes_manager.signal_axes[0].scale * core_shell_f.axes_manager.signal_axes[0].size,\n", - " core_shell_f.axes_manager.signal_axes[0].size\n", + " core_shell_f.axes_manager.signal_axes[0].offset\n", + " + core_shell_f.axes_manager.signal_axes[0].scale\n", + " * core_shell_f.axes_manager.signal_axes[0].size,\n", + " core_shell_f.axes_manager.signal_axes[0].size,\n", ")\n", "\n", "\n", - "fig, axs = plt.subplots(3,2,figsize=(15,9), width_ratios=[1,3])\n", + "fig, axs = plt.subplots(3, 2, figsize=(15, 9), width_ratios=[1, 3])\n", "cax = fig.add_axes([0.13, 0.08, 0.17, 0.01])\n", - "for i in range(3) : \n", - " im = axs[i,0].imshow(cs_f_maps.data[i],cmap='hot',vmin = 0, aspect = 'auto')\n", - " axs[i,1].plot(x, cs_f_spectra.data[i])\n", - " axs[i,0].tick_params(axis='both', which='both', bottom=False, top=False, labelbottom=False, right=False, left=False, labelleft=False)\n", - " axs[i,1].set_xlim([0,14])\n", - " axs[i,1].tick_params(axis = 'y', which = 'both', left = False, labelleft = False)\n", - " axs[i,1].set_ylabel(\"Intensity (a.u.)\")\n", - " axs[1,1].set_xlabel(\"Energy (keV)\")\n", - "\n", - "scalebar = AnchoredSizeBar(axs[2,0].transData,\n", - " 20, '{} {}'.format(20*core_shell_f.axes_manager[0].scale, core_shell_f.axes_manager[0].units),\n", - " 'lower right', \n", - " pad=0.1,\n", - " color='white',\n", - " frameon=False,\n", - " size_vertical=1)\n", + "for i in range(3):\n", + " im = axs[i, 0].imshow(cs_f_maps.data[i], cmap=\"hot\", vmin=0, aspect=\"auto\")\n", + " axs[i, 1].plot(x, cs_f_spectra.data[i])\n", + " axs[i, 0].tick_params(\n", + " axis=\"both\",\n", + " which=\"both\",\n", + " bottom=False,\n", + " top=False,\n", + " labelbottom=False,\n", + " right=False,\n", + " left=False,\n", + " labelleft=False,\n", + " )\n", + " axs[i, 1].set_xlim([0, 14])\n", + " axs[i, 1].tick_params(axis=\"y\", which=\"both\", left=False, labelleft=False)\n", + " axs[i, 1].set_ylabel(\"Intensity (a.u.)\")\n", + " axs[1, 1].set_xlabel(\"Energy (keV)\")\n", + "\n", + "scalebar = AnchoredSizeBar(\n", + " axs[2, 0].transData,\n", + " 20,\n", + " f\"{20 * core_shell_f.axes_manager[0].scale} {core_shell_f.axes_manager[0].units}\",\n", + " \"lower right\",\n", + " pad=0.1,\n", + " color=\"white\",\n", + " frameon=False,\n", + " size_vertical=1,\n", + ")\n", "\n", - "axs[1,0].add_artist(scalebar)\n", + "axs[1, 0].add_artist(scalebar)\n", "\n", - "fig.colorbar(im, cax=cax, orientation='horizontal')" + "fig.colorbar(im, cax=cax, orientation=\"horizontal\")" ] }, { @@ -1187,7 +1412,7 @@ "metadata": {}, "outputs": [], "source": [ - "core_shell_f.print_concentration_report(selected_elts = ['Fe','O'])" + "core_shell_f.print_concentration_report(selected_elts=[\"Fe\", \"O\"])" ] }, { @@ -1210,22 +1435,24 @@ "outputs": [], "source": [ "core_shell_exp = core_shell_f.mean()\n", - "core_shell_model = ((cs_f_spectra.data.T)@cs_f_maps.data.reshape((3,64*64))).mean(axis = 1)\n", + "core_shell_model = ((cs_f_spectra.data.T) @ cs_f_maps.data.reshape((3, 64 * 64))).mean(\n", + " axis=1\n", + ")\n", "\n", "\n", - "fig, axs = plt.subplots(1,2, figsize = (15,6))\n", - "axs[0].plot(x, core_shell_exp,'.', label = \"Experimental\")\n", - "axs[0].plot(x, core_shell_model, label = \"Model\")\n", + "fig, axs = plt.subplots(1, 2, figsize=(15, 6))\n", + "axs[0].plot(x, core_shell_exp, \".\", label=\"Experimental\")\n", + "axs[0].plot(x, core_shell_model, label=\"Model\")\n", "axs[0].set_xlabel(\"Energy (keV)\")\n", "axs[0].set_ylabel(\"Counts\")\n", "axs[0].legend()\n", - "axs[0].set_xlim(0,14.0)\n", - "axs[1].plot(x,core_shell_exp,'.')\n", - "axs[1].plot(x,core_shell_model)\n", + "axs[0].set_xlim(0, 14.0)\n", + "axs[1].plot(x, core_shell_exp, \".\")\n", + "axs[1].plot(x, core_shell_model)\n", "axs[1].set_xlabel(\"Energy (keV)\")\n", "axs[1].set_ylabel(\"Counts\")\n", - "axs[1].set_ylim(0,3)\n", - "axs[1].set_xlim(0,14.0)\n" + "axs[1].set_ylim(0, 3)\n", + "axs[1].set_xlim(0, 14.0)" ] }, { diff --git a/notebooks/toy-ML.ipynb b/notebooks/toy-ML.ipynb index 56b8486..ede2712 100644 --- a/notebooks/toy-ML.ipynb +++ b/notebooks/toy-ML.ipynb @@ -79,14 +79,15 @@ "metadata": {}, "outputs": [], "source": [ - "import numpy as np\n", "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "\n", + "from espm.datasets.base import generate_spim_sample\n", "from espm.estimators import SmoothNMF\n", - "from espm.measures import find_min_angle, ordered_mse, ordered_mae, ordered_r2\n", + "from espm.measures import find_min_angle, ordered_mae, ordered_mse, ordered_r2\n", "from espm.models import ToyModel\n", - "from espm.weights import generate_weights as gw\n", - "from espm.datasets.base import generate_spim_sample\n", - "from espm.utils import process_losses" + "from espm.utils import process_losses\n", + "from espm.weights import generate_weights as gw" ] }, { @@ -103,15 +104,14 @@ "metadata": {}, "outputs": [], "source": [ + "seed = 42 # for reproducibility\n", "\n", - "seed = 42 # for reproducibility\n", - "\n", - "m = 15 # Number of components\n", - "n = 200 # Length of the phases\n", + "m = 15 # Number of components\n", + "n = 200 # Length of the phases\n", "\n", - "n_poisson = 300 # Average poisson number per pixel (this number will be splitted on the m dimension)\n", + "n_poisson = 300 # Average poisson number per pixel (this number will be splitted on the m dimension)\n", "\n", - "densities = np.random.uniform(0.1, 2.0, 3) # Random densities\n" + "densities = np.random.uniform(0.1, 2.0, 3) # Random densities" ] }, { @@ -122,19 +122,26 @@ "source": [ "def get_toy_sample():\n", " model_params = {\"L\": n, \"C\": m, \"K\": 3, \"seed\": seed}\n", - " misc_params = {\"N\": n_poisson, \"seed\": seed, 'densities' : densities, \"model\": \"ToyModel\"}\n", + " misc_params = {\n", + " \"N\": n_poisson,\n", + " \"seed\": seed,\n", + " \"densities\": densities,\n", + " \"model\": \"ToyModel\",\n", + " }\n", "\n", " toy_model = ToyModel(**model_params)\n", " toy_model.generate_phases()\n", " phases = toy_model.phases.T\n", " weights = gw.generate_weights(\"toy_problem\", None)\n", "\n", - " sample = generate_spim_sample(phases, weights, model_params,misc_params, seed = seed)\n", + " sample = generate_spim_sample(phases, weights, model_params, misc_params, seed=seed)\n", " return sample\n", "\n", + "\n", "def to_vec(X):\n", - " n = X.shape[2] \n", - " return X.transpose(2,0,1).reshape(n, -1)\n", + " n = X.shape[2]\n", + " return X.transpose(2, 0, 1).reshape(n, -1)\n", + "\n", "\n", "sample = get_toy_sample()\n", "\n", @@ -143,8 +150,7 @@ "H = to_vec(sample[\"H\"])\n", "X = to_vec(sample[\"X\"])\n", "Xdot = to_vec(sample[\"Xdot\"])\n", - "shape_2d = sample[\"shape_2d\"]\n", - "\n" + "shape_2d = sample[\"shape_2d\"]" ] }, { @@ -203,15 +209,14 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "vmin, vmax = 0,1\n", + "vmin, vmax = 0, 1\n", "cmap = plt.cm.gray_r\n", "plt.figure(figsize=(10, 3))\n", "for i, hdot in enumerate(H):\n", - " plt.subplot(1,3,i+1)\n", - " plt.imshow(H[i].reshape(shape_2d), cmap=cmap, vmin=vmin, vmax=vmax)\n", + " plt.subplot(1, 3, i + 1)\n", + " plt.imshow(hdot.reshape(shape_2d), cmap=cmap, vmin=vmin, vmax=vmax)\n", " plt.axis(\"off\")\n", - " plt.title(f\"GT - Map {i+1}\")" + " plt.title(f\"GT - Map {i + 1}\")" ] }, { @@ -228,11 +233,11 @@ "metadata": {}, "outputs": [], "source": [ - "e = np.linspace(0,1, GW.shape[0])\n", + "e = np.linspace(0, 1, GW.shape[0])\n", "plt.plot(e, GW)\n", "plt.title(\"GT - Phases\")\n", "plt.xlabel(\"Frequency [normalized]\")\n", - "plt.legend([f\"phase {i+1}\" for i in range(3)]);" + "plt.legend([f\"phase {i + 1}\" for i in range(3)]);" ] }, { @@ -249,8 +254,8 @@ "metadata": {}, "outputs": [], "source": [ - "l = np.linspace(0, 1,n)\n", - "plt.plot(l, G[:,:5]);" + "l = np.linspace(0, 1, n)\n", + "plt.plot(l, G[:, :5]);" ] }, { @@ -305,8 +310,8 @@ "metadata": {}, "outputs": [], "source": [ - "lambda_L = 2 # Smoothness of the maps\n", - "mu = 0.05 # Sparsity of the maps" + "lambda_L = 2 # Smoothness of the maps\n", + "mu = 0.05 # Sparsity of the maps" ] }, { @@ -324,7 +329,7 @@ "outputs": [], "source": [ "simplex_H = True\n", - "simplex_W = False\n" + "simplex_W = False" ] }, { @@ -374,21 +379,29 @@ "metadata": {}, "outputs": [], "source": [ - "\n", - "K = len(H) # Number of components / let assume that we know it\n", + "K = len(H) # Number of components / let assume that we know it\n", "params = {}\n", - "params[\"tol\"]=1e-6 # Tolerance for the stopping criterion.\n", - "params[\"max_iter\"] = 200 # Maximum number of iterations before timing out.\n", - "params[\"hspy_comp\"] = False # If should be set to True if hspy data format is used.\n", - "params[\"verbose\"] = 1 # Verbosity level.\n", - "params[\"eval_print\"] = 10 # Print the evaluation every eval_print iterations.\n", - "params[\"epsilon_reg\"] = 1 # Regularization parameter \n", - "params[\"linesearch\"] = False # Use linesearch to accelerate convergence\n", - "params[\"shape_2d\"] = shape_2d # Shape of the 2D maps\n", - "params[\"n_components\"] = K # Number of components\n", - "params[\"normalize\"] = True # Normalize the data. It helps to keep the range of the regularization parameters lambda_L and mu in a reasonable range.\n", - "\n", - "estimator = SmoothNMF(mu=mu, lambda_L=lambda_L, G = Gused, simplex_H=simplex_H, simplex_W=simplex_W, **params)" + "params[\"tol\"] = 1e-6 # Tolerance for the stopping criterion.\n", + "params[\"max_iter\"] = 200 # Maximum number of iterations before timing out.\n", + "params[\"hspy_comp\"] = False # If should be set to True if hspy data format is used.\n", + "params[\"verbose\"] = 1 # Verbosity level.\n", + "params[\"eval_print\"] = 10 # Print the evaluation every eval_print iterations.\n", + "params[\"epsilon_reg\"] = 1 # Regularization parameter\n", + "params[\"linesearch\"] = False # Use linesearch to accelerate convergence\n", + "params[\"shape_2d\"] = shape_2d # Shape of the 2D maps\n", + "params[\"n_components\"] = K # Number of components\n", + "params[\"normalize\"] = (\n", + " True # Normalize the data. It helps to keep the range of the regularization parameters lambda_L and mu in a reasonable range.\n", + ")\n", + "\n", + "estimator = SmoothNMF(\n", + " mu=mu,\n", + " lambda_L=lambda_L,\n", + " G=Gused,\n", + " simplex_H=simplex_H,\n", + " simplex_W=simplex_W,\n", + " **params,\n", + ")" ] }, { @@ -422,7 +435,6 @@ "metadata": {}, "outputs": [], "source": [ - "\n", "losses = estimator.get_losses()\n", "\n", "values, names = process_losses(losses)\n", @@ -432,7 +444,7 @@ " plt.plot(value, label=name)\n", " plt.xlabel(\"iteration\")\n", "plt.yscale(\"log\")\n", - "plt.legend()\n" + "plt.legend()" ] }, { @@ -518,54 +530,61 @@ }, "outputs": [], "source": [ - "\n", "def plot_results(Ddot, D, Hdotflat, Hflat):\n", " fontsize = 30\n", " scale = 15\n", " aspect_ratio = 1.4\n", - " marker_list = [\"-o\",\"-s\",\"->\",\"-<\",\"-^\",\"-v\",\"-d\"]\n", + " marker_list = [\"-o\", \"-s\", \"->\", \"-<\", \"-^\", \"-v\", \"-d\"]\n", " mark_space = 20\n", - " # cmap = plt.cm.hot_r \n", + " # cmap = plt.cm.hot_r\n", " cmap = plt.cm.gray_r\n", " vmax = 1\n", " vmin = 0\n", " K = len(H)\n", " L = len(D)\n", - " \n", + "\n", " angles, true_inds = find_min_angle(Ddot.T, D.T, unique=True, get_ind=True)\n", " mse = ordered_mse(Hdotflat, Hflat, true_inds)\n", " mae = ordered_mae(Hdotflat, Hflat, true_inds)\n", " r2 = ordered_r2(Hdotflat, Hflat, true_inds)\n", "\n", - "\n", - " fig, axes = plt.subplots(K,3,figsize = (scale/K * 3 * aspect_ratio,scale))\n", - " x = np.linspace(0,1, num = L)\n", - " for i in range(K): \n", - " axes[2,i].plot(x,Ddot.T[i,:],'g-',label='ground truth',linewidth=4)\n", - " axes[2,i].plot(x,D[:,true_inds[i]],'r--',label='reconstructed',linewidth=4)\n", - " axes[2,i].set_title(\"{:.2f} deg\".format(angles[i]),fontsize = fontsize-2)\n", - " axes[2,i].set_xlim(0,1)\n", - "\n", - " axes[1,i].imshow((Hflat[true_inds[i],:]).reshape(shape_2d),vmin = vmin, vmax = vmax , cmap=cmap)\n", - " axes[1,i].set_title(\"R2: {:.2f}\".format(r2[true_inds[i]]),fontsize = fontsize-2)\n", + " fig, axes = plt.subplots(K, 3, figsize=(scale / K * 3 * aspect_ratio, scale))\n", + " x = np.linspace(0, 1, num=L)\n", + " for i in range(K):\n", + " axes[2, i].plot(x, Ddot.T[i, :], \"g-\", label=\"ground truth\", linewidth=4)\n", + " axes[2, i].plot(\n", + " x, D[:, true_inds[i]], \"r--\", label=\"reconstructed\", linewidth=4\n", + " )\n", + " axes[2, i].set_title(f\"{angles[i]:.2f} deg\", fontsize=fontsize - 2)\n", + " axes[2, i].set_xlim(0, 1)\n", + "\n", + " axes[1, i].imshow(\n", + " (Hflat[true_inds[i], :]).reshape(shape_2d), vmin=vmin, vmax=vmax, cmap=cmap\n", + " )\n", + " axes[1, i].set_title(f\"R2: {r2[true_inds[i]]:.2f}\", fontsize=fontsize - 2)\n", " # axes[i,1].set_ylim(0.0,1.0)\n", - " axes[1,i].tick_params(axis = \"both\",labelleft = False, labelbottom = False,left = False, bottom = False)\n", - "\n", - " im = axes[0,i].imshow(Hdotflat[i].reshape(shape_2d),vmin = vmin, vmax = vmax, cmap=cmap)\n", - " axes[0,i].set_title(\"Phase {}\".format(i),fontsize = fontsize)\n", - " axes[0,i].tick_params(axis = \"both\",labelleft = False, labelbottom = False,left = False, bottom = False)\n", - " axes[2,0].legend()\n", - "\n", - " rows = [\"True maps\",\"Reconstructed maps\",\"Spectra\"]\n", - "\n", - " for ax, row in zip(axes[:,0], rows):\n", + " axes[1, i].tick_params(\n", + " axis=\"both\", labelleft=False, labelbottom=False, left=False, bottom=False\n", + " )\n", + "\n", + " im = axes[0, i].imshow(\n", + " Hdotflat[i].reshape(shape_2d), vmin=vmin, vmax=vmax, cmap=cmap\n", + " )\n", + " axes[0, i].set_title(f\"Phase {i}\", fontsize=fontsize)\n", + " axes[0, i].tick_params(\n", + " axis=\"both\", labelleft=False, labelbottom=False, left=False, bottom=False\n", + " )\n", + " axes[2, 0].legend()\n", + "\n", + " rows = [\"True maps\", \"Reconstructed maps\", \"Spectra\"]\n", + "\n", + " for ax, row in zip(axes[:, 0], rows):\n", " ax.set_ylabel(row, rotation=90, fontsize=fontsize)\n", "\n", - "\n", " fig.subplots_adjust(right=0.84)\n", " # put colorbar at desire position\n", " cbar_ax = fig.add_axes([0.85, 0.5, 0.01, 0.3])\n", - " fig.colorbar(im,cax=cbar_ax)\n", + " fig.colorbar(im, cax=cbar_ax)\n", "\n", " # fig.tight_layout()\n", "\n", @@ -576,8 +595,9 @@ "\n", " return fig\n", "\n", + "\n", "fig = plot_results(GW, GW_est, H, H_est)\n", - "plt.show() \n" + "plt.show()" ] } ], diff --git a/notebooks/version_history.ipynb b/notebooks/version_history.ipynb index bd9f362..b58072f 100644 --- a/notebooks/version_history.ipynb +++ b/notebooks/version_history.ipynb @@ -33,22 +33,20 @@ "%autoreload 2\n", "%matplotlib qt\n", "\n", - "# Generic imports \n", - "import hyperspy.api as hs\n", - "import numpy as np\n", + "# Generic imports\n", "\n", "# espm imports\n", - "from espm.estimators import SmoothNMF\n", "import espm.datasets as ds\n", + "from espm.estimators import SmoothNMF\n", "\n", "# Generate the data\n", "# Here `seeds_range` is the number of different samples to generate\n", "ds.generate_built_in_datasets(seeds_range=1)\n", "\n", "# Load the data\n", - "spim = ds.load_particules(sample = 0)\n", + "spim = ds.load_particules(sample=0)\n", "# We need to change the data to floats for the decomposition to run\n", - "spim.change_dtype('float64')\n", + "spim.change_dtype(\"float64\")\n", "spim.build_G()" ] }, @@ -69,7 +67,11 @@ "source": [ "from espm.cross_sections_utils import modify_cross_sections\n", "\n", - "modify_cross_sections(100, input_type = 'scaling_factors', lines_scaling_factors = {'Ca_Ka' : 0.9, 'W_La1' : 1.1})" + "modify_cross_sections(\n", + " 100,\n", + " input_type=\"scaling_factors\",\n", + " lines_scaling_factors={\"Ca_Ka\": 0.9, \"W_La1\": 1.1},\n", + ")" ] }, { @@ -103,9 +105,7 @@ "fH_1 = spim.generate_part_fixed_H_matrix(rois=None)\n", "fH_2 = spim.generate_part_fixed_H_matrix(rois=[])\n", "\n", - "fH_dict = {'0' : fH_0,\n", - " '1' : fH_1,\n", - " '2' : fH_2}\n", + "fH_dict = {\"0\": fH_0, \"1\": fH_1, \"2\": fH_2}\n", "\n", "fH = spim.set_fixed_H(fH_dict)" ] @@ -117,8 +117,15 @@ "metadata": {}, "outputs": [], "source": [ - "snmf = SmoothNMF(n_components=3,fixed_H=fH,hspy_comp=True,simplex_H=True,simplex_W=False,G=spim.model)\n", - "spim.decomposition(algorithm = snmf)" + "snmf = SmoothNMF(\n", + " n_components=3,\n", + " fixed_H=fH,\n", + " hspy_comp=True,\n", + " simplex_H=True,\n", + " simplex_W=False,\n", + " G=spim.model,\n", + ")\n", + "spim.decomposition(algorithm=snmf)" ] }, { @@ -157,7 +164,7 @@ "metadata": {}, "outputs": [], "source": [ - "a = spim.apply_interactive_calibration(4.64,14.96)" + "a = spim.apply_interactive_calibration(4.64, 14.96)" ] }, { @@ -224,7 +231,7 @@ "metadata": {}, "outputs": [], "source": [ - "spim.elemental_mapping(skipped_elements = ['Rb'])" + "spim.elemental_mapping(skipped_elements=[\"Rb\"])" ] }, { diff --git a/pyproject.toml b/pyproject.toml index 5abec88..3ddfc86 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,14 +1,17 @@ [build-system] -requires = ["setuptools>=64", "wheel"] +requires = ["setuptools>=64", "wheel", "setuptools-scm>=8"] build-backend = "setuptools.build_meta" [tool.setuptools.packages.find] where = ["."] include = ["espm*"] +[tool.setuptools_scm] +write_to = "espm/_version.py" + [project] name = "espm" -version = "1.1.4" +dynamic = ["version"] description = "Electron SPectro-Microscopy Python Library" requires-python = ">=3.10" readme = "README.rst" @@ -27,24 +30,24 @@ classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Science/Research", "Intended Audience :: End Users/Desktop", - "Intended Audience :: Developers", + "Intended Audience :: Developers", "Natural Language :: English", "License :: OSI Approved :: GNU General Public License v3 (GPLv3)", "Operating System :: MacOS :: MacOS X", - "Operating System :: POSIX :: Linux", - "Programming Language :: C", + "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Topic :: Scientific/Engineering :: Mathematics", - "Topic :: Scientific/Engineering" + "Topic :: Scientific/Engineering", ] dependencies = [ "exspy>=0.3.2", "hyperspy>=2.4.0", "intervaltree>=3.2.1", - "numba>=0.66.0", + "ipython>=8.39.0", + "numpy>=2.2.6", "scikit-image>=0.25.2", "scikit-learn>=1.7.2", "seaborn>=0.13.2", @@ -56,18 +59,21 @@ espm = "espm" [project.license] file = "Licence.txt" +[project.urls] +"Homepage" = "https://espm.readthedocs.io/en/latest/" +"Bug Reports" = "https://github.com/adriente/espm/issues" +"Source" = "https://github.com/adriente/espm" + [project.optional-dependencies] -dev = [ - "cython>=3.2.8", - "flake8>=7.3.0", - "jupyterlab>=4.6.1", - "memory-profiler>=0.61.0", +hyperspy_ui = ["hyperspy_gui_ipywidgets>=2.0", "hyperspy_gui_traitsui>2.0"] +dev_tools = ["build>=1.5.0", "ruff>=0.16.0", "twine>=7.0.0"] +test = ["pytest>=9.1.1"] +dev = ["espm[dev_tools]", "espm[test]", "espm[hyperspy_ui]"] +doc = [ + "ipykernel", "nbsphinx>=0.9.8", "numpydoc>=1.10.0", "pandoc>=2.4", - "pdoc>=16.0.0", - "pyqt5>=5.15.0", - "pytest>=9.1.1", "sphinx>=8.1.3", "sphinx-copybutton>=0.5.2", "sphinx-gallery>=0.21.0", @@ -75,12 +81,17 @@ dev = [ "sphinxcontrib-bibtex>=2.7.0", ] -hyperspy_ui = [ - "hyperspy_gui_ipywidgets>=2.0", - "hyperspy_gui_traitsui>2.0" -] +[tool.ruff.lint] +ignore = ["E711", "E712", "E741"] +extend-select = ["I"] -[project.urls] -"Homepage" = "https://espm.readthedocs.io/en/latest/" -"Bug Reports" = "https://github.com/adriente/espm/issues" -"Source" = "https://github.com/adriente/espm" +[tool.ruff.lint.per-file-ignores] +"__init__.py" = ["F401", "F403"] + +[tool.pytest] +testpaths = ["tests"] +filterwarnings = [ + "error", + "ignore::UserWarning", + 'ignore:function ham\(\) is deprecated:DeprecationWarning', +] diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index 1e95674..0000000 --- a/pytest.ini +++ /dev/null @@ -1,5 +0,0 @@ -[pytest] -filterwarnings = - error - ignore::UserWarning - ignore:function ham\(\) is deprecated:DeprecationWarning \ No newline at end of file diff --git a/tests/test_EDXS.py b/tests/test_EDXS.py new file mode 100644 index 0000000..528c337 --- /dev/null +++ b/tests/test_EDXS.py @@ -0,0 +1,332 @@ +import hyperspy as hs +import numpy as np + +import espm.models.EDXS_function as ef +from espm.models import EDXS +from espm.models.absorption_edxs import absorption_correction, det_efficiency +from espm.models.EDXS_function import ( + lifshin_bremsstrahlung, + lifshin_bremsstrahlung_b0, + lifshin_bremsstrahlung_b1, +) +from espm.models.generate_EDXS_phases import generate_elts_dict + +x = np.linspace(0.2, 19, num=2000) +det_dict = { + "detection": {"thickness": 450e-3, "density": 2.5}, + "layer1": { + "thickness": 1e-6, + "density": 2.3, + "atomic_fraction": True, + "elements_dict": {14: 1, 8: 2}, + }, + "layer2": {"thickness": 3e-6, "density": 2.7, "elements_dict": {"Al": 1.0}}, +} + +model_parameters = { + "e_offset": 0.2, + "e_size": 1900, + "e_scale": 0.01, + "width_slope": 0.01, + "width_intercept": 0.065, + "db_name": "default_xrays.json", + "E0": 200, + "params_dict": { + "Abs": {"thickness": 200.0e-7, "toa": 35, "density": 3.12}, + "Det": det_dict, + }, +} + + +def create_data(): + a = np.random.rand(50, 70, 100) + s = hs.signals.Signal1D(a) + s.axes_manager[-1].offset = 0.2 + s.axes_manager[-1].scale = 0.1 + s.set_signal_type("EDS_espm") + s.set_microscope_parameters( + beam_energy=200, tilt_stage=0.0, azimuth_angle=0.0, elevation_angle=35.0 + ) + s.metadata.Acquisition_instrument.TEM.Stage.tilt_beta = 0.0 + s.set_analysis_parameters(thickness=1e-6, density=2.7) + s.set_elements(elements=["Si", "O", "Fe", "Ca"]) + return s + + +def test_lifshin_bremsstrahlung(): + E0 = 200 + x = np.linspace(0.2, E0 - 0.1, num=2000) + for i in range(10): + b0 = np.random.rand() + b1 = np.random.rand() + y0 = lifshin_bremsstrahlung_b0(x, b0, E0) + y1 = lifshin_bremsstrahlung_b1(x, b1, E0) + y = lifshin_bremsstrahlung(x, b0, b1, E0) + + np.testing.assert_array_less(1e-30, y0) + np.testing.assert_array_less(1e-30, y1) + np.testing.assert_array_less(1e-30, y) + np.testing.assert_allclose(y, y1 + y0) + + +def test_efficiency(): + det = det_efficiency(x, det_dict) + np.testing.assert_array_less(1e-30, det) + np.testing.assert_array_less(det, 1.0) + + +def test_absorption(): + elts = generate_elts_dict(42) + abs_corr = absorption_correction( + x, thickness=1e-4, toa=35, density=5, elements_dict=elts + ) + + np.testing.assert_array_less(1e-30, abs_corr) + np.testing.assert_array_less(abs_corr, 1.0) + + +def test_elts_dict_from_dict_list(): + dict_list = [ + {"chou": 1, "carottes": 2, "navet": 3}, + {"chou": 2, "oignons": 3, "navet": 3}, + {"orange": 6, "citron": 4, "poireau": 8}, + ] + unique_dict = ef.elts_dict_from_dict_list(dict_list) + sum_elts = 32 + assert unique_dict == { + "chou": 3 / sum_elts, + "carottes": 2 / sum_elts, + "navet": 6 / sum_elts, + "orange": 6 / sum_elts, + "citron": 4 / sum_elts, + "poireau": 8 / sum_elts, + "oignons": 3 / sum_elts, + } + + +def test_continuum_xrays(): + E0 = 200 + b0 = np.random.uniform(0.0, 50.0) + b1 = np.random.uniform(0.0, 50.0) + cont_x = ef.continuum_xrays( + x, + model_parameters["params_dict"], + b0=b0, + b1=b1, + E0=E0, + elements_dict={"Si": 1.0, "C": 0.2, "Bi": 3.2}, + ) + + np.testing.assert_array_less(1e-30, cont_x) + + +def test_get_elements(): + s = create_data() + s.build_G(elements_dict={"Fe": 3.0}) + nelts = [] + for i in s.model.get_elements(False): + nelts.append(i) + assert nelts == ["20", "26", "8", "14"] + nelts = [] + for i in s.model.get_elements(True): + nelts.append(i) + print(nelts) + assert nelts == ["20", "26", "8", "14", "29"] + + +def test_carac_x_span(): + # TODO : Find a way to implement the test of this function + pass + + +def test_bremsstrahlung_only_tools(): + # TODO : Find a way to implement the test of this function + pass + + +def test_NMF_initialize_W(): + model = EDXS(**model_parameters) + model.generate_g_matr( + g_type="bremsstrahlung", + elements=["Na", "Sr", "Ge", "Nb"], + elements_dict={}, + ignored_elements=[], + ) + D = np.random.rand(1900, 2) + W = model.NMF_initialize_W(D) + assert W.shape == (6, 2) + + model = EDXS(**model_parameters) + model.generate_g_matr( + g_type="no_brstlg", + elements=["Na", "Sr", "Ge", "Nb"], + elements_dict={"Nb": 3.0}, + ignored_elements=[], + ) + D = np.random.rand(1900, 4) + W = model.NMF_initialize_W(D) + assert W.shape == (5, 4) + + model = EDXS(**model_parameters) + model.generate_g_matr( + g_type="identity", + elements=["Na", "Sr", "Ge", "Nb"], + elements_dict={}, + ignored_elements=[], + ) + with np.testing.assert_raises(ValueError): + D = np.random.rand(1900, 4) + W = model.NMF_initialize_W(D) + + model = EDXS(**model_parameters) + model.generate_g_matr( + g_type="bremsstrahlung", + elements=["Na", "Ge", "Nb"], + elements_dict={"Nb": 3.0}, + ignored_elements=["Sr"], + ) + D = np.random.rand(1900, 2) + W = model.NMF_initialize_W(D) + assert W.shape == (23, 2) + + +def test_NMF_simplex(): + model = EDXS(**model_parameters) + model.generate_g_matr( + g_type="bremsstrahlung", elements=["Na", "Sr", "Ge", "Nb"], elements_dict={} + ) + inds = model.NMF_simplex() + assert inds == [0, 1, 2, 3] + + model = EDXS(**model_parameters) + model.generate_g_matr( + g_type="no_brstlg", elements=["Na", "Sr", "Ge", "Nb"], elements_dict={"Nb": 3.0} + ) + inds = model.NMF_simplex() + assert inds == [0, 1, 2, 4] + + model = EDXS(**model_parameters) + model.generate_g_matr( + g_type="bremsstrahlung", + elements=["Sr", "Ge", "Nb"], + elements_dict={"Nb": 3.0}, + ignored_elements=["Na"], + ) + inds = model.NMF_simplex() + assert inds == [0, 1, 3] + + +def test_NMF_update(): + model = EDXS(**model_parameters) + model.generate_g_matr( + g_type="no_brstlg", elements=["Na", "Sr", "Ge", "Nb"], elements_dict={} + ) + temp_G = model.G.copy() + np.testing.assert_array_equal(model.NMF_update(), temp_G) + np.testing.assert_array_equal(model.NMF_update(np.random.rand(10, 10)), temp_G) + + model = EDXS(**model_parameters) + model.generate_g_matr( + g_type="bremsstrahlung", elements=["Na", "Sr", "Ge", "Nb"], elements_dict={} + ) + temp_G = model.G.copy() + np.testing.assert_array_equal(model.NMF_update(), temp_G) + assert model.NMF_update(np.random.rand(6, 10)).shape == model.G.shape + with np.testing.assert_raises(AssertionError): + np.testing.assert_array_equal( + model.NMF_update(np.random.rand(6, 10))[:, -2:], temp_G[:, -2:] + ) + + +def test_generate_g_matr(): + model1 = EDXS(**model_parameters) + model2 = EDXS(**model_parameters) + model3 = EDXS(**model_parameters) + model4 = EDXS(**model_parameters) + model5 = EDXS(**model_parameters) + size = model_parameters["e_size"] + elts_list = ["Na", "Sr", "Ge", "Nb"] + model1.generate_g_matr( + g_type="bremsstrahlung", + elements=elts_list, + elements_dict={}, + ignored_elements=[], + ) + G_brem = model1.G + model2.generate_g_matr( + g_type="identity", elements=elts_list, elements_dict={}, ignored_elements=[] + ) + G_id = model2.G + model3.generate_g_matr( + g_type="no_brstlg", elements=elts_list, elements_dict={}, ignored_elements=[] + ) + G_no_brstlg = model3.G + model4.generate_g_matr( + g_type="bremsstrahlung", + elements=elts_list, + elements_dict={"Ge": 3.0}, + ignored_elements=[], + ) + G_brem_ge = model4.G + model5.generate_g_matr( + g_type="bremsstrahlung", + elements=["Na", "Ge", "Nb"], + elements_dict={"Ge": 3.0}, + ignored_elements=["Sr"], + ) + G_brem_ge_ign = model5.G + + assert G_brem.shape == (size, 6) + assert G_no_brstlg.shape == (size, 4) + assert G_brem_ge.shape == (size, 7) + assert G_brem_ge_ign.shape == (size, 23) + + assert G_id is None + np.testing.assert_allclose(G_brem[:, :-2], G_no_brstlg) + np.testing.assert_allclose( + (G_brem_ge[:, 3] + G_brem_ge[:, 2]) * model4.norm[0, 0], + G_brem[:, 2] * model1.norm[0, 0], + ) + + # np.testing.assert_array_less(-1e-30, G_id) + np.testing.assert_array_less(-1e-30, G_brem) + np.testing.assert_array_less(-1e-30, G_no_brstlg) + assert model1.model_elts == ["11", "38", "32", "41"] + assert model2.model_elts == [] + assert model3.model_elts == ["11", "38", "32", "41"] + assert model4.model_elts == ["11", "38", "32_lo", "32_hi", "41"] + assert model5.model_elts == [ + "11", + "32_lo", + "32_hi", + "41", + "38_ign0", + "38_ign1", + "38_ign2", + "38_ign3", + "38_ign4", + "38_ign5", + "38_ign6", + "38_ign7", + "38_ign8", + "38_ign9", + "38_ign10", + "38_ign11", + "38_ign12", + "38_ign13", + "38_ign14", + "38_ign15", + "38_ign16", + ] + + +def test_G_bremsstrahlung(): + model = EDXS(**model_parameters) + size = model_parameters["e_size"] + elts_dict = {"V": 0.3, "Ti": 0.1, "Hf": 0.05, "Pb": 0.55} + B = ef.G_bremsstrahlung( + model.x, model.E0, model.params_dict, elements_dict=elts_dict + ) + + assert B.shape == (size, 2) + np.testing.assert_array_less(-1e-30, B) diff --git a/tests/test_datasets.py b/tests/test_datasets.py new file mode 100644 index 0000000..35d183d --- /dev/null +++ b/tests/test_datasets.py @@ -0,0 +1,606 @@ +import os +import shutil +from pathlib import Path + +import hyperspy.api as hs +import numpy as np +from exspy.utils.eds import take_off_angle + +import espm.weights.generate_weights as wts +from espm.conf import DATASETS_PATH +from espm.datasets.base import ( + generate_dataset, + generate_spim, + generate_spim_sample, + sample_to_EDSespm, +) +from espm.datasets.eds_spim import get_metadata +from espm.estimators import SmoothNMF +from espm.models import EDXS +from espm.models.EDXS_function import elts_list_from_dict_list +from espm.models.generate_EDXS_phases import ( + generate_modular_phases, +) +from espm.weights.generate_weights import generate_weights + +elts_dicts = [ + {"Fe": 0.54860348, "Pt": 0.38286879, "Mo": 0.03166235, "O": 0.03686538}, + {"Ca": 0.54860348, "Si": 0.38286879, "O": 0.15166235}, + {"Cu": 0.34, "Mo": 0.12, "Au": 0.54}, +] + +brstlg_pars = [ + {"b0": 5e-3, "b1": 3e-2}, + {"b0": 7e-3, "b1": 5e-2}, + {"b0": 3e-3, "b1": 5e-2}, +] + +scales = [0.05, 0.05, 0.05] + +model_params = { + "e_offset": 0.2, + "e_size": 1900, + "e_scale": 0.01, + "width_slope": 0.01, + "width_intercept": 0.065, + "db_name": "200keV_xrays.json", + "E0": 200, + "params_dict": { + "Abs": {"thickness": 100.0e-7, "toa": 35, "density": 5}, + "Det": "SDD_efficiency.txt", + }, +} + +misc_params = { + "N": 400, + "densities": [1.3, 1.6, 1.9], + "data_folder": "test_gen_data", + "seed": 42, + "shape_2d": (100, 120), + "model": "EDXS", +} + +elements = elts_list_from_dict_list(elts_dicts) + + +def test_generate(): + + # Generate the phases + model = EDXS(**model_params) + model.generate_g_matr(g_type="bremsstrahlung", elements=elements, elements_dict={}) + phases1 = generate_modular_phases( + elts_dicts=elts_dicts, + brstlg_pars=brstlg_pars, + scales=scales, + model_params=model_params, + ) + G = model.G + n_phases = len(elts_dicts) + maps = generate_weights( + weight_type="sphere", + shape_2d=misc_params["shape_2d"], + n_phases=n_phases, + seed=misc_params["seed"], + radius=15, + ) + densities = np.array(misc_params["densities"]) + spim_sample = generate_spim_sample( + phases1, maps, model_params=model_params, misc_params=misc_params + ) + spim = spim_sample["X"] + cont_spim = spim_sample["Xdot"] + Xdot = misc_params["N"] * maps @ np.diag(densities) @ phases1 + W = np.abs(np.linalg.lstsq(G, spim.sum(axis=(0, 1)), rcond=None)[0]) + + assert phases1.shape == (3, 1900) + assert maps.shape == (100, 120, 3) + assert spim.shape == (100, 120, 1900) + np.testing.assert_allclose(np.sum(phases1, axis=1), np.ones([3])) + np.testing.assert_allclose(Xdot, cont_spim) + np.testing.assert_allclose(Xdot.sum(axis=(0, 1)), G @ W, rtol=0.1) + + if os.path.exists("test.hspy"): + os.remove("test.hspy") + filename = "test.hspy" + hspy_spim = sample_to_EDSespm(spim_sample, elements=elements) + hspy_spim.save(filename) + si = hs.load(filename) + assert si.metadata.Signal.signal_type == "EDS_espm_Simulated" + si.build_G(problem_type="bremsstrahlung") + G = si.G + phases, maps = si.phases, si.maps_2d + # weights = weights.reshape((100,120,n_phases)) + X = si.data + W = np.linalg.lstsq(G, X.sum(axis=(0, 1)), rcond=None)[0] + + assert phases.shape == (1900, 3) + assert maps.shape == (100, 120, 3) + assert si.data.shape == (100, 120, 1900) + np.testing.assert_allclose(Xdot, maps @ phases.T) + np.testing.assert_allclose(Xdot.sum(axis=(0, 1)), G @ W, rtol=0.2) + + os.remove(filename) + + if os.path.exists(str(DATASETS_PATH / Path(misc_params["data_folder"]))): + shutil.rmtree(str(DATASETS_PATH / Path(misc_params["data_folder"]))) + + generate_dataset( + base_seed=misc_params["seed"], + sample_number=2, + model_params=model_params, + misc_params=misc_params, + phases=phases1, + weights=maps, + elements=elements, + ) + gen_folder = DATASETS_PATH / Path(misc_params["data_folder"]) + gen_si = hs.load(gen_folder / Path("sample_0.hspy")) + + np.testing.assert_allclose(X, gen_si.data) + + shutil.rmtree(str(gen_folder)) + + +def test_generate_spim(): + + k = 3 + ell = 7 + shape_2d = [4, 5] + pd = 0.3 + seed = 0 + + phases = np.random.rand(k, ell) + weights = np.random.rand(*shape_2d, k) + densities = 1 - pd + 2 * pd * np.random.rand(k) + + # Ns = np.linspace(10, 200, 30) + # vals = [] + # vals2 = [] + + # for N in Ns: + # Xdot = generate_spim(phases, weights, densities, N, seed=seed,continuous = True)/N + # X = generate_spim(phases, weights, densities, N, seed=seed,continuous = False)/N + # X2 = np.random.poisson(N * Xdot) / N + + # vals.append(np.mean(np.abs(X - Xdot))) + # vals2.append(np.mean(np.abs(X2 - Xdot))) + + # plt.plot(Ns, vals) + # plt.plot(Ns, vals2) + N = 10000 + X3 = np.zeros([*shape_2d, ell]) + for k, w in enumerate(densities): + # generating the spectroscopic events + for i in range(shape_2d[0]): + for j in range(shape_2d[1]): + # Draw a local_N based on the local density + local_N = np.random.poisson(N * w * weights[i, j, k]) + # draw local_N events from the ideal spectrum + counts = np.random.choice(ell, local_N, p=phases[k] / np.sum(phases[k])) + # Generate the spectrum based on the drawn events + hist = np.bincount(counts, minlength=ell) + X3[i, j] += hist + + Xdot = generate_spim(phases, weights, densities, N, seed=seed, continuous=True) / N + X = generate_spim(phases, weights, densities, N, seed=seed, continuous=False) / N + # X2 = np.random.poisson(N * Xdot) / N + + X3 /= N + assert np.mean(np.abs(X - Xdot)) < 0.005 + assert np.mean(np.abs(X3 - Xdot)) < 0.005 + assert np.mean(np.abs(X - X3)) < 0.006 + + +def test_generate_random_weights(): + shape_2d = [28, 36] + n_phases = 5 + + w = wts.random_weights(shape_2d=shape_2d, n_phases=n_phases) + + assert w.shape == (*shape_2d, n_phases) + np.testing.assert_array_less(-1e-30, w) + np.testing.assert_array_almost_equal(np.sum(w, axis=2), 1) + + +def test_generate_laplacian_weights(): + shape_2d = [28, 36] + n_phases = 5 + + w = wts.laplacian_weights(shape_2d=shape_2d, n_phases=n_phases) + + assert w.shape == (*shape_2d, n_phases) + np.testing.assert_array_less(-1e-30, w) + np.testing.assert_array_almost_equal(np.sum(w, axis=2), 1) + + +def test_generate_two_sphere(): + shape_2d = [80, 80] + n_phases = 3 + radius = 2 + + w = wts.spheres_weights(shape_2d=shape_2d, n_phases=n_phases, radius=radius) + + assert w.shape == (80, 80, 3) + np.testing.assert_array_less(-1e-30, w) + np.testing.assert_array_almost_equal(np.sum(w, axis=2), 1) + + +def test_generate_gaussian_ripple(): + shape_2d = [100, 40] + width = 10 + + w = wts.gaussian_ripple_weights(shape_2d, width=width) + + assert w.shape == (100, 40, 2) + np.testing.assert_array_less(-1e-30, w) + np.testing.assert_array_almost_equal(np.sum(w, axis=2), 1) + + +# def test_gen_EDXS () : + +# b_dict = generate_brem_params(42) +# assert b_dict["b0"] <= 1.0 +# assert b_dict["b1"] <= 1.0 + +# phases, dicts = generate_random_phases(n_phases=3,seed = 42) +# np.testing.assert_array_less(-1e-30, phases) +# model = EDXS(**DEFAULT_SYNTHETIC_DATA_DICT["model_parameters"]) +# model.generate_phases(dicts) +# np.testing.assert_almost_equal(model.phases,phases) + +# unique_list = unique_elts(dicts) +# assert len(unique_list) == len(set(unique_list)) + + +def test_decomposition(): + if os.path.exists(str(DATASETS_PATH / Path(misc_params["data_folder"]))): + shutil.rmtree(str(DATASETS_PATH / Path(misc_params["data_folder"]))) + + phases1 = generate_modular_phases( + elts_dicts=elts_dicts, + brstlg_pars=brstlg_pars, + scales=scales, + model_params=model_params, + ) + maps = generate_weights( + weight_type="sphere", + shape_2d=misc_params["shape_2d"], + n_phases=len(elts_dicts), + seed=misc_params["seed"], + radius=15, + ) + generate_dataset( + base_seed=misc_params["seed"], + sample_number=2, + model_params=model_params, + misc_params=misc_params, + phases=phases1, + weights=maps, + elements=elements, + ) + + gen_folder = DATASETS_PATH / Path(misc_params["data_folder"]) + gen_si = hs.load(gen_folder / Path("sample_0.hspy")) + gen_si.change_dtype("float64") + gen_si.build_G() + est = SmoothNMF(n_components=3, G=gen_si.model, tol=1e-7, hspy_comp=True) + gen_si.decomposition(algorithm=est) + assert gen_si.metadata.Signal.signal_type == "EDS_espm_Simulated" + np.testing.assert_allclose( + (est.G_ @ est.W_ @ est.H_).sum(axis=1), gen_si.X.sum(axis=1), rtol=0.5 + ) + indices = gen_si.model.NMF_simplex() + np.testing.assert_allclose(est.W_[indices, :].sum(axis=0), np.ones(3), rtol=0.1) + + shutil.rmtree(str(gen_folder)) + + +def test_spim(): + + if os.path.exists(str(DATASETS_PATH / Path(misc_params["data_folder"]))): + shutil.rmtree(str(DATASETS_PATH / Path(misc_params["data_folder"]))) + phases1 = generate_modular_phases( + elts_dicts=elts_dicts, + brstlg_pars=brstlg_pars, + scales=scales, + model_params=model_params, + ) + maps = generate_weights( + weight_type="sphere", + shape_2d=misc_params["shape_2d"], + n_phases=len(elts_dicts), + seed=misc_params["seed"], + radius=15, + ) + generate_dataset( + base_seed=misc_params["seed"], + sample_number=2, + model_params=model_params, + misc_params=misc_params, + phases=phases1, + weights=maps, + elements=elements, + ) + gen_folder = DATASETS_PATH / Path(misc_params["data_folder"]) + gen_si = hs.load(gen_folder / Path("sample_0.hspy")) + + assert gen_si.metadata.Signal.signal_type == "EDS_espm_Simulated" + + mod_pars = get_metadata(gen_si) + # mod_pars["params_dict"]["Abs"]["atomic_fraction"] = False + + assert model_params == mod_pars + + shape = gen_si.shape_2d + gen_si = hs.load(gen_folder / Path("sample_0.hspy")) + gen_si.build_G(problem_type="identity") + assert shape == (100, 120) + assert gen_si.G is None + + gen_si = hs.load(gen_folder / Path("sample_0.hspy")) + gen_si.build_G(problem_type="no_brstlg", elements_dict={}, ignored_elements=[]) + assert gen_si.G.shape == (1900, 8) + + gen_si = hs.load(gen_folder / Path("sample_0.hspy")) + gen_si.build_G(problem_type="bremsstrahlung", elements_dict={}, ignored_elements=[]) + assert gen_si.G.shape == (1900, 10) + + Xflat = gen_si.X + assert Xflat.shape == (1900, 120 * 100) + + gen_si = hs.load(gen_folder / Path("sample_0.hspy")) + gen_si.build_G( + problem_type="bremsstrahlung", elements_dict={"26": 3.0}, ignored_elements=[] + ) + assert gen_si.G.shape == (1900, 11) + + gen_si = hs.load(gen_folder / Path("sample_0.hspy")) + gen_si.build_G( + problem_type="no_brstlg", elements_dict={"26": 3.0}, ignored_elements=[] + ) + assert gen_si.G.shape == (1900, 9) + + gen_si = hs.load(gen_folder / Path("sample_0.hspy")) + gen_si.build_G( + problem_type="bremsstrahlung", + elements_dict={"26": 3.0}, + ignored_elements=["Cu"], + ) + assert gen_si.G.shape == (1900, 22) + + detector_dict = { + "detection": {"thickness": 45, "elements_dict": {"Si": 3, "Se": 4}}, + "layer1": {"thickness": 34, "elements_dict": {"Ge": 1, "O": 3}}, + } + + gen_si.set_microscope_parameters( + beam_energy=100, azimuth_angle=2.0, elevation_angle=3.0, tilt_stage=4.0 + ) + gen_si.set_elements(elements=["Si"]) + gen_si.metadata.Acquisition_instrument.TEM.Stage.tilt_beta = 0.0 + + gen_si.set_analysis_parameters( + thickness=500e-7, + density=4, + detector_type=detector_dict, + width_slope=0.3, + width_intercept=65.0, + xray_db="100keV_xrays.json", + ) + + assert gen_si.metadata.Sample.thickness == 500e-7 + assert gen_si.metadata.Sample.density == 4 + assert gen_si.metadata.Sample.elements == ["Si"] + assert gen_si.metadata.xray_db == "100keV_xrays.json" + assert gen_si.metadata.Acquisition_instrument.TEM.Stage.tilt_alpha == 4.0 + assert gen_si.metadata.Acquisition_instrument.TEM.beam_energy == 100 + assert ( + gen_si.metadata.Acquisition_instrument.TEM.Detector.EDS.take_off_angle + == take_off_angle(4.0, 2.0, 3.0) + ) + assert ( + gen_si.metadata.Acquisition_instrument.TEM.Detector.EDS.type.as_dictionary() + == detector_dict + ) + assert gen_si.metadata.Acquisition_instrument.TEM.Detector.EDS.width_slope == 0.3 + assert ( + gen_si.metadata.Acquisition_instrument.TEM.Detector.EDS.width_intercept == 65.0 + ) + + shutil.rmtree(str(gen_folder)) + + +def test_carto_fixed_W(): + + def create_data(): + a = np.random.rand(50, 70, 100) + s = hs.signals.Signal1D(a) + s.axes_manager[-1].offset = 0.2 + s.axes_manager[-1].scale = 0.1 + s.set_signal_type("EDS_espm") + s.add_elements(elements=["Si", "O", "Fe", "Ca"]) + s.set_microscope_parameters( + beam_energy=100, azimuth_angle=2.0, elevation_angle=3.0, tilt_stage=4.0 + ) + s.metadata.Acquisition_instrument.TEM.Stage.tilt_beta = 0.0 + s.set_analysis_parameters( + thickness=500e-7, + density=4, + detector_type="SDD_efficiency.txt", + width_slope=0.3, + width_intercept=65.0, + xray_db="100keV_xrays.json", + ) + return s + + s = create_data() + s.build_G(problem_type="bremsstrahlung", ignored_elements=[]) + fw1 = s.carto_fixed_W(brstlg_comps=2) + tw1_1 = np.diag(-1 * np.ones(4)) + tw1_2 = np.zeros((2, 4)) + tw1_prime = np.vstack((tw1_1, tw1_2)) + tw1_3 = np.zeros((4, 2)) + tw1_4 = -1 * np.ones((2, 2)) + tw1_second = np.vstack((tw1_3, tw1_4)) + tw1 = np.hstack((tw1_prime, tw1_second)) + + assert fw1.shape == (6, 6) + np.testing.assert_array_equal(fw1, tw1) + + s = create_data() + s.build_G(problem_type="no_brstlg", ignored_elements=[]) + fw2 = s.carto_fixed_W() + tw2 = np.diag(-1 * np.ones(4)) + + assert fw2.shape == (4, 4) + np.testing.assert_array_equal(fw2, tw2) + + s = create_data() + s.build_G(problem_type="no_brstlg", elements_dict={"26": 3.0}, ignored_elements=[]) + fw3 = s.carto_fixed_W() + tw3 = np.diag(-1 * np.ones(5)) + + assert fw3.shape == (5, 5) + np.testing.assert_array_equal(fw3, tw3) + + +def test_set_fixed_W(): + def create_data(): + a = np.random.rand(50, 70, 100) + s = hs.signals.Signal1D(a) + s.axes_manager[-1].offset = 0.2 + s.axes_manager[-1].scale = 0.1 + s.set_signal_type("EDS_espm") + s.add_elements(elements=["Si", "O", "Fe", "Ca"]) + s.set_microscope_parameters( + beam_energy=100, azimuth_angle=2.0, elevation_angle=3.0, tilt_stage=4.0 + ) + s.metadata.Acquisition_instrument.TEM.Stage.tilt_beta = 0.0 + s.set_analysis_parameters( + thickness=500e-7, + density=4, + detector_type="SDD_efficiency.txt", + width_slope=0.3, + width_intercept=65.0, + xray_db="100keV_xrays.json", + ) + return s + + s = create_data() + s.build_G(problem_type="bremsstrahlung", ignored_elements=[]) + fw1 = s.set_fixed_W( + { + "p0": {"Si": 0.0, "O": 0.5, "b1": 10.0}, + "p1": {}, + "p2": {"O": 0.4, "Fe": 0.6, "b0": 3.0}, + } + ) + tw1 = ( + np.array( + [ + [-1.0, -1.0, 0.5, 0.0, -1.0, 10.0], + [-1.0, -1.0, -1.0, -1.0, -1.0, -1.0], + [-1.0, 0.6, 0.4, -1.0, 3.0, -1.0], + ] + ) + ).T + assert fw1.shape == (6, 3) + np.testing.assert_array_equal(fw1, tw1) + + s = create_data() + s.build_G(problem_type="no_brstlg", elements_dict={"Fe": 3.0}, ignored_elements=[]) + fw2 = s.set_fixed_W( + {"p0": {"Si": 0.0, "O": 0.5}, "p1": {}, "p2": {"O": 0.4, "Fe": 0.6}} + ) + tw2 = ( + np.array( + [ + [-1.0, -1.0, -1.0, 0.5, 0.0], + [-1.0, -1.0, -1.0, -1.0, -1.0], + [-1.0, -1.0, 0.6, 0.4, -1.0], + ] + ) + ).T + + assert fw2.shape == (5, 3) + np.testing.assert_array_equal(fw2, tw2) + + s = create_data() + s.build_G(problem_type="bremsstrahlung", ignored_elements=["Cu"]) + fw3 = s.set_fixed_W( + { + "p0": {"Si": 0.0, "O": 0.5, "b1": 10.0}, + "p1": {}, + "p2": {"O": 0.4, "Fe": 0.6, "b0": 3.0}, + } + ) + tw3 = ( + np.array( + [ + [ + -1.0, + -1.0, + 0.5, + 0.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 10.0, + ], + [ + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + ], + [ + -1.0, + 0.6, + 0.4, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + -1.0, + 3.0, + -1.0, + ], + ] + ) + ).T + assert fw1.shape == (6, 3) + np.testing.assert_array_equal(fw3, tw3) + + +def test_estimate_best_binning(): + pass + # TODO : Implement this test diff --git a/espm/tests/test_docstring.py b/tests/test_docstring.py similarity index 70% rename from espm/tests/test_docstring.py rename to tests/test_docstring.py index 4f57809..6a33372 100644 --- a/espm/tests/test_docstring.py +++ b/tests/test_docstring.py @@ -1,11 +1,9 @@ -# -*- coding: utf-8 -*- - """ Test suite for the docstrings of the espm package. """ -import os import doctest +import os def gen_recursive_file(root, ext): @@ -17,32 +15,30 @@ def gen_recursive_file(root, ext): def func_test_docstrings(root, ext, setup=None): files = list(gen_recursive_file(root, ext)) - return doctest.DocFileSuite(*files, setUp=setup, tearDown=teardown, - module_relative=False) + return doctest.DocFileSuite( + *files, setUp=setup, tearDown=teardown, module_relative=False + ) def setup(doctest): - import numpy import hyperspy.api - doctest.globs = { - 'np': numpy, - 'hs': hyperspy.api - } + import numpy + + doctest.globs = {"np": numpy, "hs": hyperspy.api} def teardown(doctest): """Close matplotlib figures to avoid warning and save memory.""" from espm.utils import close_all + close_all() def test_docstrings_espm(): # Docstrings from API reference. - func_test_docstrings('espm', '.py', setup) + func_test_docstrings("espm", ".py", setup) def test_docstrings_rst(): # Docstrings from tutorials. No setup to not forget imports. - func_test_docstrings('.', '.rst') - - + func_test_docstrings(".", ".rst") diff --git a/tests/test_estimators.py b/tests/test_estimators.py new file mode 100644 index 0000000..3daeb65 --- /dev/null +++ b/tests/test_estimators.py @@ -0,0 +1,444 @@ +import numpy as np +from sklearn.utils.estimator_checks import check_estimator + +from espm.datasets.base import generate_spim_sample +from espm.estimators import SmoothNMF +from espm.estimators.base import normalization_factor +from espm.estimators.surrogates import ( + diff_surrogate, + smooth_dgkl_surrogate, + smooth_l2_surrogate, +) +from espm.measures import trace_xtLx +from espm.models import EDXS +from espm.models.generate_EDXS_phases import generate_modular_phases +from espm.utils import create_laplacian_matrix +from espm.weights import generate_weights + +phases_dict = { + "elts_dicts": [ + {"Fe": 0.54860348, "Pt": 0.38286879, "Mo": 0.03166235}, + {"Ca": 0.54860348, "Si": 0.38286879, "O": 0.03166235}, + ], + "brstlg_pars": [{"b0": 5e-5, "b1": 3e-4}, {"b0": 7e-5, "b1": 5e-4}], + "scales": [0.05, 0.05], + "model_params": { + "e_offset": 0.2, + "e_size": 2000, + "e_scale": 0.01, + "width_slope": 0.01, + "width_intercept": 0.065, + "db_name": "200keV_xrays.json", + "E0": 200, + "params_dict": { + "Abs": { + "thickness": 1e-5, + "toa": 35, + "density": 5, + "atomic_fraction": False, + }, + "Det": "SDD_efficiency.txt", + }, + }, +} + +misc_dict = { + "N": 100, + "seed": 42, + "data_folder": "built_in_particules", + "shape_2d": [10, 20], + "model": "EDXS", + "densities": [1.3, 1.6], +} + + +def generate_one_sample(): + # Generate the phases + model = EDXS(**phases_dict["model_params"]) + phases = generate_modular_phases(**phases_dict) + model.generate_g_matr( + g_type="bremsstrahlung", + elements=["Fe", "Mo", "Ca", "Si", "O", "Pt"], + elements_dict={}, + ignored_elements=[], + ) + G = model.G + + weights = generate_weights.generate_weights( + "laplacian", + misc_dict["shape_2d"], + n_phases=len(phases_dict["elts_dicts"]), + seed=misc_dict["seed"], + size_x=10, + size_y=10, + ) + + sample = generate_spim_sample( + phases, weights, phases_dict["model_params"], misc_dict, seed=42, g_params={} + ) + + X = sample["X"].reshape(-1, sample["X"].shape[-1]).T + X_cont = sample["Xdot"].reshape(-1, sample["Xdot"].shape[-1]).T + + D = sample["GW"].T + H = sample["H_flat"].T + + W = np.abs(np.linalg.lstsq(G, D, rcond=None)[0]) + for i in range(10): + G = model.NMF_update(W) + W = np.abs(np.linalg.lstsq(G, D, rcond=None)[0]) + + w = np.array(misc_dict["densities"]) + N = misc_dict["N"] + + return G, W, H, D, w, X, X_cont, N + + +def gen_fixed_mat(): + fixed_W = -1 * np.ones((8, 2)) + fixed_W[0, 0] = 0.0 + fixed_W[1, 0] = 0.0 + + fixed_H = -1 * np.ones((2, 200)) + fixed_H[0, 0:20] = 1.0 + fixed_H[1, 0:20] = 0.0 + + return fixed_W, fixed_H + + +def test_generate_one_sample(): + G, W, H, D, w, X, Xdot, N = generate_one_sample() + np.testing.assert_allclose(G @ W, D, atol=1e-3) + np.testing.assert_allclose(D @ H, Xdot) + np.testing.assert_allclose(G @ W @ H, Xdot, atol=1e-1) + + +def test_NMF_scikit(): + estimator = SmoothNMF( + n_components=5, + max_iter=200, + simplex_W=False, + simplex_H=True, + mu=1.0, + epsilon_reg=1.0, + hspy_comp=False, + ) + check_estimator(estimator) + estimator = SmoothNMF( + n_components=5, + lambda_L=2, + max_iter=200, + simplex_W=False, + simplex_H=True, + mu=1.0, + epsilon_reg=1.0, + hspy_comp=False, + ) + check_estimator(estimator) + + +def test_general(): + G, W, H, D, w, X, Xdot, N = generate_one_sample() + + # Check if we can recover D from H and Xdot + estimator = SmoothNMF( + G=G, + n_components=2, + max_iter=200, + simplex_W=False, + simplex_H=True, + mu=0, + epsilon_reg=1, + hspy_comp=False, + ) + D2 = estimator.fit_transform(H=H, X=Xdot) + np.testing.assert_allclose(D, D2, atol=6e-1) + + # Same without G + estimator = SmoothNMF( + n_components=2, + max_iter=200, + simplex_W=False, + simplex_H=True, + mu=0, + epsilon_reg=1, + hspy_comp=False, + ) + D2 = estimator.fit_transform(H=H, X=Xdot) + np.testing.assert_allclose(D, D2, atol=6e-2) + + # Check if we can recover D, H from W and Xdot + estimator = SmoothNMF( + G=G, + n_components=2, + max_iter=200, + simplex_W=False, + simplex_H=True, + mu=0, + epsilon_reg=1, + hspy_comp=False, + ) + D2 = estimator.fit_transform(W=W, X=Xdot) + H2 = estimator.H_ + np.testing.assert_allclose(D, D2, atol=3e-2) + np.testing.assert_allclose(H, H2, atol=3e-2) + + estimator = SmoothNMF( + G=G, + n_components=2, + max_iter=200, + simplex_W=False, + simplex_H=True, + mu=0, + epsilon_reg=1, + hspy_comp=False, + ) + D2 = estimator.fit_transform(W=W, X=Xdot) + np.testing.assert_allclose(D, D2, atol=3e-1) + + estimator = SmoothNMF( + G=G, + n_components=2, + max_iter=200, + simplex_W=False, + simplex_H=True, + mu=0, + epsilon_reg=1, + hspy_comp=False, + tol=1.0e-6, + ) + estimator.fit_transform(X=Xdot) + P2, A2 = estimator.W_, estimator.H_ + np.testing.assert_allclose(G @ P2 @ A2, Xdot, atol=1) + + estimator = SmoothNMF( + G=G, + n_components=2, + max_iter=200, + simplex_W=False, + simplex_H=True, + mu=0, + epsilon_reg=1, + hspy_comp=False, + tol=1.0e-6, + ) + estimator.fit_transform(X=X) + P2, A2 = estimator.W_, estimator.H_ + np.testing.assert_allclose(G @ P2 @ A2, Xdot, atol=1.1) + + estimator = SmoothNMF( + G=G, + shape_2d=[10, 20], + lambda_L=0, + n_components=2, + max_iter=200, + simplex_W=False, + simplex_H=True, + mu=0, + epsilon_reg=1, + hspy_comp=False, + tol=1.0e-6, + ) + estimator.fit_transform(X=X) + P2, A2 = estimator.W_, estimator.H_ + np.testing.assert_allclose(G @ P2 @ A2, Xdot, atol=1.1) + + estimator = SmoothNMF( + G=G, + lambda_L=100, + n_components=2, + max_iter=200, + simplex_W=False, + simplex_H=True, + mu=0, + epsilon_reg=1, + shape_2d=[10, 20], + hspy_comp=False, + tol=1.0e-6, + ) + estimator.fit_transform(X=X) + P3, A3 = estimator.W_, estimator.H_ + np.testing.assert_allclose(G @ P3 @ A3, Xdot, atol=1.1) + L = create_laplacian_matrix(10, 20) + + assert trace_xtLx(L, A3.T) < trace_xtLx(L, A2.T) + # assert(trace_xtLx(L, A.T) < trace_xtLx(L, A2.T) ) + assert trace_xtLx(L, A3.T) < trace_xtLx(L, H.T) * 1.01 + + +def test_fixed_mat(): + G, W, H, D, w, X, Xdot, N = generate_one_sample() + fW, fH = gen_fixed_mat() + estimator = SmoothNMF( + G=G, + n_components=2, + max_iter=200, + simplex_W=False, + simplex_H=True, + fixed_W=fW, + hspy_comp=False, + ) + estimator.fit_transform(X=X) + P2, A2 = estimator.W_, estimator.H_ + np.testing.assert_allclose(P2[fW >= 0], fW[fW >= 0]) + + estimator = SmoothNMF( + G=G, + n_components=2, + max_iter=200, + simplex_W=False, + simplex_H=True, + fixed_H=fH, + hspy_comp=False, + ) + estimator.fit_transform(X=X) + P2, A2 = estimator.W_, estimator.H_ + np.testing.assert_allclose(A2[fH >= 0], fH[fH >= 0]) + + +def test_surrogate_smooth_nmf(): + L = create_laplacian_matrix(4, 3) + for i in range(10): + A1 = np.random.randn(3, 12) + v1 = smooth_l2_surrogate(A1, L) + v2 = smooth_l2_surrogate(A1, L, A1) + v3 = trace_xtLx(L, A1.T) / 2 + np.testing.assert_almost_equal(v1, v2) + np.testing.assert_almost_equal(v1, v3) + + for j in range(10): + A2 = np.random.randn(3, 12) + v4 = smooth_l2_surrogate(A1, L, A2) + v5 = trace_xtLx(L, A2.T) / 2 + assert v4 >= v5 + d = diff_surrogate(A1, A2, L=L, algo="l2_surrogate") + np.testing.assert_almost_equal(v4 - v5, d) + + +def test_surrogate_smooth_dgkl_nmf(): + + L = create_laplacian_matrix(4, 3) + for i in range(10): + A1 = np.random.rand(3, 12) + v1 = smooth_dgkl_surrogate(A1, L) + v2 = smooth_dgkl_surrogate(A1, L, A1) + v3 = trace_xtLx(L, A1.T) / 2 + np.testing.assert_almost_equal(v1, v2) + np.testing.assert_almost_equal(v1, v3) + + for j in range(10): + A2 = np.random.rand(3, 12) + v4 = smooth_dgkl_surrogate(A1, L, A2) + v5 = trace_xtLx(L, A2.T) / 2 + assert v4 >= v5 + d = diff_surrogate(A1, A2, L=L, algo="log_surrogate") + np.testing.assert_almost_equal(v4 - v5, d) + + +def test_X_normalize(): + np.random.seed(0) + X = np.random.rand(10, 32) * 100 + fac = np.random.rand() * 50 + 0.1 + print(fac) + X_plus = np.concatenate([X / fac, X / fac], axis=0) + + nc = 5 + + estim = SmoothNMF( + n_components=nc, + lambda_L=1.0, + max_iter=10, + init="nndsvd", + normalize=True, + shape_2d=[8, 4], + random_state=0, + simplex_W=False, + simplex_H=True, + ) + GP = estim.fit_transform(X) + H = estim.H_ + GP_plus = estim.fit_transform(X_plus) + H_plus = estim.H_ + + def close_enough(a, b): + num = np.sum(np.abs(a - b)) + den = np.sum(np.abs(a)) + ratio = num / den + print(num, den, ratio) + return ratio < 1e-4 + + # plt.figure(figsize=(10,5)) + # plt.subplot(121) + # plt.imshow(H) + # plt.colorbar() + # plt.subplot(122) + # plt.imshow(H_plus) + # plt.colorbar() + + # plt.figure(figsize=(10,5)) + # plt.subplot(121) + # plt.imshow(np.concatenate([GP, GP], axis=0)) + # plt.colorbar() + # plt.subplot(122) + # plt.imshow(GP_plus*fac) + # plt.colorbar() + # np.testing.assert_allclose(GP_plus*fac, np.concatenate([GP, GP], axis=0), atol=1e-10) + # np.testing.assert_allclose(H_plus, H, atol=1e-10) + assert close_enough(GP_plus * fac, np.concatenate([GP, GP], axis=0)) + assert close_enough(H_plus, H) + + +def test_normalization_factor(): + X_high = np.random.rand(10, 32) + fac = np.random.rand() * 50 + X_low = X_high / fac + + nc = 5 + + X_high_norm = normalization_factor(X_high, nc) * X_high + X_low_norm = normalization_factor(X_low, nc) * X_low + + np.testing.assert_allclose(X_high_norm, X_low_norm) + + +# def test_losses(): +# G, P, A, D, w, X, Xdot, N = generate_one_sample() +# true_spectra = (G @ P @ np.diag(w)) +# true_maps = A + +# estimator = SmoothNMF(G = G, shape_2d = (10,20), n_components = 2, max_iter = 10, true_A = true_maps, true_D = true_spectra, lambda_L = 1, hspy_comp = False) + +# estimator.fit_transform(X = X) + +# loss = estimator.get_losses() +# # k = 2 +# # default_params = { +# # "n_components" : k, +# # "tol" : 1e-6, +# # "max_iter" : 10, +# # "init" : "random", +# # "random_state" : 1, +# # "verbose" : 0 +# # } + +# # params_snmf = { +# # "simplex_H" : True, +# # "mu": np.random.rand(k) +# # } + +# # params_evalution = { +# # "u" : True, +# # } + +# # # All parameters are contained here +# # exp = {"name": "snmfem smooth 30", "method": "SmoothNMF", "params": {**default_params, **params_snmf, "lambda_L" : 100.0}} + +# # spim = hs.signals.Signal1D(X) +# # spim.set_signal_type("EDXSsnmfem") + +# # estimator = SmoothNMF(**exp["params"]) + +# # m, (G, P, A), loss = run_experiment(spim,estimator,exp) + +# values = np.array([list(e) for e in loss]) +# np.testing.assert_allclose(KL(X, G@P @ A, average=True), values[-1,1]) diff --git a/espm/tests/test_laplacian.py b/tests/test_laplacian.py similarity index 94% rename from espm/tests/test_laplacian.py rename to tests/test_laplacian.py index 6d76121..5b8e99f 100644 --- a/espm/tests/test_laplacian.py +++ b/tests/test_laplacian.py @@ -1,8 +1,10 @@ import numpy as np -from espm.utils import create_laplacian_matrix -from espm.conf import sigmaL from scipy.sparse.linalg import eigs +from espm.conf import sigmaL +from espm.utils import create_laplacian_matrix + + def my_laplacian_op(image): nx, ny = image.shape @@ -14,10 +16,10 @@ def my_laplacian_op(image): return -(dxx + dyy) + def test_laplacian_matrix(): for nx in range(2, 5): - for ny in range(2,8): - + for ny in range(2, 8): L1 = [] for i in range(nx): for j in range(ny): @@ -30,9 +32,10 @@ def test_laplacian_matrix(): np.testing.assert_array_almost_equal(L2.todense(), L1) + def test_sigma_L(): for nx in range(4, 100, 20): for ny in range(5, 100, 30): Delta = create_laplacian_matrix(nx, ny) l2 = np.abs(eigs(Delta, k=1)[0][0]) - assert(l2 <= sigmaL) + assert l2 <= sigmaL diff --git a/espm/tests/test_measures.py b/tests/test_measures.py similarity index 51% rename from espm/tests/test_measures.py rename to tests/test_measures.py index b1cab55..440c719 100644 --- a/espm/tests/test_measures.py +++ b/tests/test_measures.py @@ -1,18 +1,34 @@ import numpy as np -from espm.measures import mse, spectral_angle, KLdiv_loss, KLdiv, find_min_MSE, find_min_angle, trace_xtLx, Frobenius_loss, ordered_angles, ordered_mse -from espm.measures import KL_loss_surrogate, log_reg, log_surrogate + from espm.conf import log_shift +from espm.measures import ( + Frobenius_loss, + KL_loss_surrogate, + KLdiv, + KLdiv_loss, + find_min_angle, + find_min_MSE, + log_reg, + log_surrogate, + mse, + ordered_angles, + ordered_mse, + spectral_angle, + trace_xtLx, +) from espm.utils import create_laplacian_matrix + def base_loss(x_matr, d_matr, a_matr, eps=log_shift): """ Evaluates the data-fitting part of the loss function. The d_matr argument is used for evaluation of temporary values of the d_matr that are not stored in self.d_matr """ d_a = d_matr @ a_matr - x_log = np.einsum("ij,ij", x_matr, np.log(d_a+eps)) + x_log = np.einsum("ij,ij", x_matr, np.log(d_a + eps)) return d_a.sum() - x_log + def MSE_map(map1, map2): """ Calculates the mean squared error between two 2D arrays. They have to have the same dimension. @@ -24,6 +40,7 @@ def MSE_map(map1, map2): tr_m1_m2 = np.trace(map1.T @ map2) return tr_m1_m1 - 2 * tr_m1_m2 + tr_m2_m2 + def spectral_angle_simple(v1, v2): """ Calculates the angle between two spectra. They have to have the same dimension. @@ -34,12 +51,13 @@ def spectral_angle_simple(v1, v2): v2_u = v2 / np.linalg.norm(v2) return np.arccos(np.clip(np.dot(v1_u, v2_u), -1.0, 1.0)) * 180 / np.pi + def test_mse(): a = np.random.randn(10, 34) b = np.random.randn(10, 34) - np.testing.assert_allclose(mse(a, b), MSE_map(a, b)/34/10) + np.testing.assert_allclose(mse(a, b), MSE_map(a, b) / 34 / 10) def test_spectral_angle(): @@ -53,14 +71,13 @@ def test_spectral_angle(): np.testing.assert_allclose(spectral_angle(v1, v1), 0, atol=1e-5) - v1 = np.array([1,0]) - v2 = np.array([0,1]) + v1 = np.array([1, 0]) + v2 = np.array([0, 1]) np.testing.assert_allclose(spectral_angle(v1, v2), 90) - - v1 = np.array([1,0]) - v2 = np.array([-1,0]) + v1 = np.array([1, 0]) + v2 = np.array([-1, 0]) np.testing.assert_allclose(spectral_angle(v1, v2), 180) @@ -78,92 +95,104 @@ def test_spectral_angle(): np.testing.assert_allclose(spectral_angle(a, b), res) -def test_find_min_angle () : +def test_find_min_angle(): np.random.seed(42) - p1 = np.random.rand(4,34) - p2 = np.random.rand(4,34) - - true_res_u = ([41.100923139683005, - 41.29114572254828, - 44.37587131209638, - 38.446115521694054], - (1, 2, 3, 0)) - - mins_u, ind_mins_u = find_min_angle(p1,p2,get_ind=True,unique=True) - np.testing.assert_allclose(mins_u,true_res_u[0]) - np.testing.assert_allclose(ind_mins_u,true_res_u[1]) - - true_res_g = ([38.446115521694054, - 38.95264025554747, - 41.22519848513551, - 38.446748219852694], - [3, 3, 0, 3]) - - mins_g, ind_mins_g = find_min_angle(p1,p2,get_ind=True) - np.testing.assert_allclose(mins_g,true_res_g[0]) - np.testing.assert_allclose(ind_mins_g,true_res_g[1]) - -def test_find_min_mse () : + p1 = np.random.rand(4, 34) + p2 = np.random.rand(4, 34) + + true_res_u = ( + [41.100923139683005, 41.29114572254828, 44.37587131209638, 38.446115521694054], + (1, 2, 3, 0), + ) + + mins_u, ind_mins_u = find_min_angle(p1, p2, get_ind=True, unique=True) + np.testing.assert_allclose(mins_u, true_res_u[0]) + np.testing.assert_allclose(ind_mins_u, true_res_u[1]) + + true_res_g = ( + [38.446115521694054, 38.95264025554747, 41.22519848513551, 38.446748219852694], + [3, 3, 0, 3], + ) + + mins_g, ind_mins_g = find_min_angle(p1, p2, get_ind=True) + np.testing.assert_allclose(mins_g, true_res_g[0]) + np.testing.assert_allclose(ind_mins_g, true_res_g[1]) + + +def test_find_min_mse(): np.random.seed(42) - p1 = np.random.rand(4,34) - p2 = np.random.rand(4,34) - - true_res_u = ([0.15630003054762612, - 0.17106418363825374, - 0.17991197120876662, - 0.17531522148609824], - (1, 2, 3, 0)) - - mins_u, ind_mins_u = find_min_MSE(p1,p2,get_ind=True,unique=True) - np.testing.assert_allclose(mins_u,true_res_u[0]) - np.testing.assert_allclose(ind_mins_u,true_res_u[1]) - - true_res_g = ([0.17516591343119908, - 0.15630003054762612, - 0.15133648097157715, - 0.1780263407013204], - [0, 0, 0, 3]) - - mins_g, ind_mins_g = find_min_MSE(p1,p2,get_ind=True) - np.testing.assert_allclose(mins_g,true_res_g[0]) - np.testing.assert_allclose(ind_mins_g,true_res_g[1]) - -def test_ordered_functions () : + p1 = np.random.rand(4, 34) + p2 = np.random.rand(4, 34) + + true_res_u = ( + [ + 0.15630003054762612, + 0.17106418363825374, + 0.17991197120876662, + 0.17531522148609824, + ], + (1, 2, 3, 0), + ) + + mins_u, ind_mins_u = find_min_MSE(p1, p2, get_ind=True, unique=True) + np.testing.assert_allclose(mins_u, true_res_u[0]) + np.testing.assert_allclose(ind_mins_u, true_res_u[1]) + + true_res_g = ( + [ + 0.17516591343119908, + 0.15630003054762612, + 0.15133648097157715, + 0.1780263407013204, + ], + [0, 0, 0, 3], + ) + + mins_g, ind_mins_g = find_min_MSE(p1, p2, get_ind=True) + np.testing.assert_allclose(mins_g, true_res_g[0]) + np.testing.assert_allclose(ind_mins_g, true_res_g[1]) + + +def test_ordered_functions(): np.random.seed(42) - p1 = np.random.rand(4,34) - p2 = np.random.rand(4,34) + p1 = np.random.rand(4, 34) + p2 = np.random.rand(4, 34) + + true_ang_u = ( + [41.100923139683005, 41.29114572254828, 44.37587131209638, 38.446115521694054], + (1, 2, 3, 0), + ) - true_ang_u = ([41.100923139683005, - 41.29114572254828, - 44.37587131209638, - 38.446115521694054], - (1, 2, 3, 0)) + angles = ordered_angles(p1, p2, true_ang_u[1]) + np.testing.assert_allclose(angles, true_ang_u[0]) - angles = ordered_angles(p1,p2,true_ang_u[1]) - np.testing.assert_allclose(angles,true_ang_u[0]) + true_mse_u = ( + [ + 0.15630003054762612, + 0.17106418363825374, + 0.17991197120876662, + 0.17531522148609824, + ], + (1, 2, 3, 0), + ) - true_mse_u = ([0.15630003054762612, - 0.17106418363825374, - 0.17991197120876662, - 0.17531522148609824], - (1, 2, 3, 0)) + mse = ordered_mse(p1, p2, true_mse_u[1]) + np.testing.assert_allclose(mse, true_mse_u[0]) - mse = ordered_mse(p1,p2,true_mse_u[1]) - np.testing.assert_allclose(mse,true_mse_u[0]) def test_base_loss(): l = 26 k = 5 p = 100 - A = np.random.rand(k,p) - A = A/np.sum(A, axis=1, keepdims=1) - - D = np.random.rand(l,k) - + A = np.random.rand(k, p) + A = A / np.sum(A, axis=1, keepdims=1) + + D = np.random.rand(l, k) + X = D @ A - - val = np.sum(X) - np.sum(X*np.log(X)) + + val = np.sum(X) - np.sum(X * np.log(X)) val2 = KLdiv_loss(X, D, A) val3 = base_loss(X, D, A) @@ -174,29 +203,28 @@ def test_base_loss(): np.testing.assert_almost_equal(KLdiv(X, D, A), 0) # with a different value, the divergence should be bigger - A = np.random.rand(k,p) - A = A/np.sum(A, axis=1, keepdims=1) + A = np.random.rand(k, p) + A = A / np.sum(A, axis=1, keepdims=1) val2 = KLdiv_loss(X, D, A) - assert(val2>val) + assert val2 > val val3 = KLdiv(X, D, A) - np.testing.assert_almost_equal(val2 -val, val3 ) + np.testing.assert_almost_equal(val2 - val, val3) - - A = np.random.rand(k,p) + A = np.random.rand(k, p) val2 = KLdiv_loss(X, D, A) - assert(val2>val) - + assert val2 > val + val3 = KLdiv(X, D, A, average=False) val4 = KLdiv(X, D, A, average=True) - np.testing.assert_almost_equal(val3, val4*X.shape[0]*X.shape[1] ) + np.testing.assert_almost_equal(val3, val4 * X.shape[0] * X.shape[1]) - A = 0.00001*np.random.rand(k,p) + A = 0.00001 * np.random.rand(k, p) val2 = KLdiv_loss(X, D, A) - assert(val2>val) + assert val2 > val - A = np.zeros([k,p]) + A = np.zeros([k, p]) val2 = KLdiv_loss(X, D, A) - assert(val2>val) + assert val2 > val # The code does this differently now... # with pytest.raises(Exception): @@ -209,6 +237,7 @@ def test_base_loss(): # D = np.random.randn(l,k) # val2 = KLdiv_loss(X, D, A) + def test_trace_xtLx(): nx = 4 ny = 6 @@ -217,45 +246,46 @@ def test_trace_xtLx(): np.testing.assert_allclose(trace_xtLx(L, x), 0) x = np.ones([nx, ny]) - x[2,3] = 2 + x[2, 3] = 2 np.testing.assert_allclose(trace_xtLx(L, x.flatten()), 4) - + k = 10 - x = np.random.randn(nx*ny, k) + x = np.random.randn(nx * ny, k) r1 = trace_xtLx(L, x) r2 = 0 for i in range(k): - r2 = r2 + trace_xtLx(L, x[:,i]) + r2 = r2 + trace_xtLx(L, x[:, i]) np.testing.assert_allclose(r1, r2) np.testing.assert_allclose(trace_xtLx(L, x), np.sum(np.diag(x.T @ (L @ x)))) r3 = trace_xtLx(L, x, average=True) - np.testing.assert_allclose(r1,r3*k*nx*ny) - + np.testing.assert_allclose(r1, r3 * k * nx * ny) + + def test_froebenius_loss(): l = 26 k = 5 p = 100 - A = np.random.rand(k,p) - A = A/np.sum(A, axis=1, keepdims=1) - - D = np.random.rand(l,k) - + A = np.random.rand(k, p) + A = A / np.sum(A, axis=1, keepdims=1) + + D = np.random.rand(l, k) + X = D @ A - + np.testing.assert_almost_equal(Frobenius_loss(X, D, A), 0) - D2 = np.random.rand(l,k) - res1 = np.linalg.norm(D2 @ A -X,"fro")**2 + D2 = np.random.rand(l, k) + res1 = np.linalg.norm(D2 @ A - X, "fro") ** 2 res2 = Frobenius_loss(X, D2, A) np.testing.assert_almost_equal(res1, res2) res1 = Frobenius_loss(X, D2, A) res2 = Frobenius_loss(X, D2, A, average=True) - np.testing.assert_almost_equal(res1/l/p, res2) + np.testing.assert_almost_equal(res1 / l / p, res2) def test_surrogates(): @@ -265,14 +295,16 @@ def test_surrogates(): mu = 10 epsilon = 0.01 - W0 = np.random.rand(l, k) + W0 = np.random.rand(l, k) H0 = np.random.rand(k, p) X = np.random.rand(l, p) log_shift = 0.0 np.testing.assert_allclose( - KL_loss_surrogate(X, W0, H0, H0, log_shift=log_shift), - KLdiv_loss(X, W0, H0, log_shift=log_shift)) + KL_loss_surrogate(X, W0, H0, H0, log_shift=log_shift), + KLdiv_loss(X, W0, H0, log_shift=log_shift), + ) np.testing.assert_allclose( log_surrogate(H0, H0, mu=mu, epsilon=epsilon), - log_reg(H0, mu=mu, epsilon=epsilon)) + log_reg(H0, mu=mu, epsilon=epsilon), + ) diff --git a/tests/test_updates.py b/tests/test_updates.py new file mode 100644 index 0000000..9ef4057 --- /dev/null +++ b/tests/test_updates.py @@ -0,0 +1,1011 @@ +import numpy as np +import pytest + +from espm.conf import dicotomy_tol, log_shift +from espm.estimators.updates import ( + dichotomy_simplex, + dichotomy_simplex_acc, + estimate_Lipschitz_bound_h, + estimate_Lipschitz_bound_w, + gradH, + gradW, + multiplicative_step_h, + multiplicative_step_hq, + multiplicative_step_w, + proj_grad_step_h, + proj_grad_step_w, + update_q, +) +from espm.measures import Frobenius_loss, KLdiv_loss, log_reg, trace_xtLx +from espm.utils import create_laplacian_matrix + + +def make_step_a( + x_matr, g_matr, p_matr, a_matr, mu_sparse=0, eps=log_shift, eps_sparse=1, mask=None +): + """ + Multiplicative step in A. + The main terms are calculated first. + With mu_sparse = 0, the steps in A are calculated once. For mu_sparse != 0, the steps in A are calculated first with particle regularization. Then only the entries allowed by the mask are calculaed, without particle regularization. + To calculate the regularized step, we make a linear approximation of the log. + """ + # Update of the d_matr (maybe it should be done in a dedicated function for clarity) + d_matr = g_matr @ p_matr + + # Multiplicative update numerator U and denominator V + d_a = d_matr @ a_matr + U = d_matr.T @ (x_matr.clip(min=1e-150) / d_a) + V = d_matr.sum(axis=0) + # Reset of the Lagrangian multiplier (Maybe non-necessary .. ?) + # lambda_s = np.zeros((x_shape[1] * x_shape[0],)) + + if mu_sparse != 0: + # Regularized part of the algorithm + if mask is None: + # In the linear approximation, the slope is constant. We modifiy this slope to approximate the log every 10 iterations. + # The number of iterations in between two slope changes is arbitrary. + # if num_iterations % 10 == 0: + fixed_a = a_matr.copy() + # Vectorized version of the regularization + vec_sparse = np.array([0] + (a_matr.shape[0] - 1) * [mu_sparse]) + denum = V[:, np.newaxis] + vec_sparse[:, np.newaxis] / ( + fixed_a + eps_sparse + ) + # Lagragian multiplier + lambda_s = dichotomy_simplex(a_matr * U, denum, 0, tol=dicotomy_tol) + # A update (regularized) + a_matr = ( + a_matr + / ( + V[:, np.newaxis] + + vec_sparse[:, np.newaxis] / (fixed_a + eps_sparse) + + lambda_s + ) + * U + ) + else: + # Lagragian multiplier + lambda_s = dichotomy_simplex( + a_matr * U, V[:, np.newaxis], 0, tol=dicotomy_tol + ) + # Update the entry that did not meet the sparsity requirements + n_mask = np.invert(mask) + # A update (masked) + a_matr[n_mask] = ( + a_matr[n_mask] / (V[:, np.newaxis] + lambda_s)[n_mask] * U[n_mask] + ) + + else: + # Lagragian multiplier + lambda_s = dichotomy_simplex(a_matr * U, V[:, np.newaxis], 0, tol=dicotomy_tol) + # A update (not regularized) + a_matr = a_matr / (V[:, np.newaxis] + lambda_s) * U + + return a_matr + + +def make_step_p(x_matr, g_matr, p_matr, a_matr, eps=log_shift): + """ + Multiplicative step in P. + """ + d_matr = g_matr @ p_matr + + d_a = d_matr @ a_matr + term1 = ( + g_matr.T @ (x_matr.clip(min=1e-150) / (d_a + eps)) @ a_matr.T + ) # The data are clipped to avoid issues during computation + term2 = g_matr.sum(axis=0)[:, np.newaxis] @ a_matr.sum(axis=1)[:, np.newaxis].T + # P update + p_matr = p_matr / term2 * term1 + return p_matr + + +def test_dichotomy_simplex(): + num = np.random.rand(1, 1) + 1 + denum = np.random.rand(1, 1) + sol = num - denum + tol = 1e-8 + sol2 = dichotomy_simplex(num, denum, 0, tol=tol) + assert np.abs(sol - sol2) < 2 * tol + + n = 10 + num = np.random.rand(1, n) + denum = np.random.rand(1, n) + sol = np.squeeze(num - denum) + sol2 = dichotomy_simplex(num, denum, 0, tol=tol) + np.testing.assert_allclose(sol2, sol, atol=tol) + + num = np.random.rand(n, 1) + denum = np.random.rand(n, 1) + tol = 1e-6 + sol = dichotomy_simplex(num, denum, 0, tol=tol) + np.sum(num / (denum + sol)) + np.testing.assert_allclose(np.sum(num / (denum + sol)), 1, atol=tol) + + num = np.random.rand(n, 6) + denum = np.random.rand(n, 6) + tol = 1e-6 + sol = dichotomy_simplex(num, denum, 0, tol=tol) + v = np.sum(num / (denum + sol), axis=0) + np.testing.assert_allclose(v, np.ones([6]), atol=tol) + + num = np.random.rand(n, 1) + denum = np.random.rand(n, 1) + num = num / (np.sum(num / denum)) + np.testing.assert_allclose(np.sum(num / (denum)), 1) + tol = 1e-6 + sol = dichotomy_simplex(num, denum, 0, tol=tol) + np.testing.assert_allclose(sol, 0, atol=tol) + + tol = 1e-6 + num = np.array([[1, 1, 0, 0, 0, 2]]).T + denum = np.array([[1, 1, 3, 5, 4, 2]]).T + sol = dichotomy_simplex(num, denum, 0, tol=tol) + np.testing.assert_allclose(np.sum(num / (denum + sol)), 1, atol=tol) + + tol = 1e-6 + num = np.array([[1, 1, 0, 0, 0, 2]]).T + denum = np.array([[1, 1, 0, 5, 0, 2]]).T + sol = dichotomy_simplex(num, denum, 0, tol=tol) + np.sum(num / (denum + sol)) + np.testing.assert_allclose(np.sum(num / (denum + sol)), 1, atol=tol) + + +def test_dichotomy_simplex_contraint(): + # Test the other constraint + tol = 1e-8 + num = np.array([[3, 0.5]]).T + denum = np.array([[1, 1]]).T + sol = np.array([[3]]).T + tol = 1e-8 + sol2 = dichotomy_simplex(num, denum, 1 / 4, tol=tol) + assert np.abs(sol - sol2) < 2 * tol + + n = 10 + num = np.random.rand(1, n) + denum = np.random.rand(1, n) + sol = np.squeeze(num - denum) + sol2 = dichotomy_simplex(num, denum, 0.5, tol=tol) + np.testing.assert_allclose(sol2, sol, atol=tol) + + with pytest.raises(ValueError) as _: + num = np.random.rand(1, n) + denum = np.random.rand(1, n) + dichotomy_simplex(num, denum, 1.1, tol=tol) + with pytest.raises(ValueError) as _: + num = np.random.rand(3, n) + denum = np.random.rand(3, n) + dichotomy_simplex(num, denum, 0.5, tol=tol) + + n = 10 + num = np.random.rand(1, n) + denum = np.random.rand(1, n) + sol = np.squeeze(num - denum) + sol2 = dichotomy_simplex(num, denum, 0.99, tol=tol) + np.testing.assert_allclose(sol2, sol, atol=tol) + + n = 10 + num = np.random.rand(n, 6) + denum = np.random.rand(n, 6) + tol = 1e-6 + log_shift = 0.05 + sol = dichotomy_simplex(num, denum, log_shift, tol=tol) + v = np.sum(np.maximum(num / (denum + sol), log_shift), axis=0) + np.testing.assert_allclose(v, np.ones([6]), atol=tol) + + +def test_dicotomy2(): + k = 5 + p = 6400 + span = np.logspace(-6, 6, num=17) + its = 25 + tol = 0 + maxit = 100 + np.random.seed(0) + for _ in range(its): + scale_num = np.random.choice(span, size=(k, p)) + num = scale_num * np.random.rand(k, p) + scale_denum = np.random.choice(span, size=(k, p)) + denum = scale_denum * np.random.rand(k, p) + sol = dichotomy_simplex(num, denum, 0, tol=tol, maxit=maxit) + v = np.sum(num / (denum + sol), axis=0) + np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) + + log_shift = 0.1 / k + sol = dichotomy_simplex(num, denum, log_shift, tol=tol, maxit=maxit) + v = np.sum(np.maximum(num / (denum + sol), log_shift), axis=0) + np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) + + for _ in range(its): + scale_num = np.random.choice(span, size=(k, p)) + num = scale_num * np.random.rand(k, p) + num[np.tile(np.arange(k), p // k), np.arange(p)] = 0 + scale_denum = np.random.choice(span, size=(k, p)) + denum = scale_denum * np.random.rand(k, p) + sol = dichotomy_simplex(num, denum, 0, tol=tol, maxit=maxit) + v = np.sum(num / (denum + sol), axis=0) + np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) + + log_shift = 0.1 / k + sol = dichotomy_simplex(num, denum, log_shift, tol=tol, maxit=maxit) + v = np.sum(np.maximum(num / (denum + sol), log_shift), axis=0) + np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) + + for _ in range(its): + scale_num = np.random.choice(span, size=(k, p)) + num = scale_num * np.random.rand(k, p) + scale_denum = np.random.choice(span, size=(k, 1)) + denum = scale_denum * np.random.rand(k, 1) + sol = dichotomy_simplex(num, denum, 0, tol=tol, maxit=maxit) + v = np.sum(num / (denum + sol), axis=0) + np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) + + log_shift = 0.1 / k + sol = dichotomy_simplex(num, denum, log_shift, tol=tol, maxit=maxit) + v = np.sum(np.maximum(num / (denum + sol), log_shift), axis=0) + np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) + + for _ in range(its): + scale_num = np.random.choice(span, size=(k, p)) + num = scale_num * np.random.rand(k, p) + num[np.tile(np.arange(k), p // k), np.arange(p)] = 0 + scale_denum = np.random.choice(span, size=(k, 1)) + denum = scale_denum * np.random.rand(k, 1) + sol = dichotomy_simplex(num, denum, 0, tol=tol, maxit=maxit) + v = np.sum(num / (denum + sol), axis=0) + np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) + + log_shift = 0.1 / k + sol = dichotomy_simplex(num, denum, log_shift, tol=tol, maxit=maxit) + v = np.sum(np.maximum(num / (denum + sol), log_shift), axis=0) + np.testing.assert_allclose(v, np.ones([v.shape[0]]), atol=1e-2) + + +def test_dicotomy_aq(): + def func_abc(x, a, b, minus_c): + n_p = len(b) + return ( + n_p * x + + 2 * a + + np.sum(-np.sqrt((b + x) ** 2 + 4 * a * minus_c) + b, axis=0) + ) + + # def func_abc_onedim(x, a, b, minus_c): + # return x + 2*a + b - np.sqrt( (b + x)**2 + 4*a*minus_c) + + a = np.random.rand() + b = np.random.rand(1, 1) + minus_c = np.random.rand(1, 1) + # x + 2*a + b - np.sqrt( (b + x)**2 + 4*a*minus_c) = 0 + # x + 2*a + b = np.sqrt( (b + x)**2 + 4*a*minus_c) + # (x + b)^2 + 4*a^2 + 4*a (x + b) = (b + x)**2 + 4*a*minus_c + # a + (x + b) = minus_c + # x = minus_c - a -b + sol = minus_c - a - b + tol = 1e-8 + sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol) + assert np.abs(sol - sol2) < 10 * tol + + a = np.random.rand() + b = np.zeros([1, 1]) + minus_c = np.random.rand(1, 1) + sol = minus_c - a - b + tol = 1e-8 + sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol) + assert np.abs(sol - sol2) < 10 * tol + + a = np.random.rand() + b = np.random.rand(1, 1) + minus_c = np.zeros([1, 1]) + sol = minus_c - a - b + tol = 1e-8 + sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol) + assert np.abs(sol - sol2) < 10 * tol + + a = np.random.rand() + b = np.zeros([1, 1]) + minus_c = np.zeros([1, 1]) + sol = minus_c - a - b + tol = 1e-8 + sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol) + assert np.abs(sol - sol2) < 10 * tol + + n = 10 + a = np.random.rand() + b = np.random.rand(1, n) + minus_c = np.random.rand(1, n) + sol = np.squeeze(minus_c - a - b) + sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol) + np.testing.assert_allclose(sol2, sol, atol=10 * tol) + + a = np.random.rand() + b = np.random.rand(n, 6) + minus_c = np.random.rand(n, 6) + tol = 1e-6 + sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol) + np.testing.assert_allclose( + func_abc(sol, a, b, minus_c), np.zeros([6]), atol=10 * tol + ) + + +def test_dichotomy_aq_constraint(): + # Test the other constraint + tol = 1e-8 + a = 4 + log_shift = 1 / 8 + b = np.array([[1, 1, 1, 8 / log_shift, 8 / log_shift]]).T + minus_c = np.array([[1, 1, 1, 1, 1]]).T + + sol = 2 + tol = 1e-8 + sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=log_shift, tol=tol) + assert np.abs(sol - sol2) < 10 * tol + + n = 10 + a = np.random.rand() + b = np.random.rand(1, n) + minus_c = np.random.rand(1, n) + sol = np.squeeze(minus_c - a - b) + sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0.5, tol=tol) + np.testing.assert_allclose(sol2, sol, atol=10 * tol) + + with pytest.raises(ValueError) as _: + a = np.random.rand(1, n) + b = np.random.rand(1, n) + minus_c = np.random.rand(1, n) + sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=1.1, tol=tol) + + with pytest.raises(ValueError) as _: + a = np.random.rand(3, n) + b = np.random.rand(3, n) + minus_c = np.random.rand(3, n) + sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0.5, tol=tol) + + n = 10 + a = np.random.rand() + b = np.random.rand(1, n) + minus_c = np.random.rand(1, n) + sol = np.squeeze(minus_c - a - b) + sol2 = dichotomy_simplex_acc(a, b, minus_c, log_shift=0.99, tol=tol) + np.testing.assert_allclose(sol2, sol, atol=10 * tol) + + def func_abc(x, a, b, minus_c, log_shift): + return ( + np.sum( + np.maximum( + np.sqrt((b + x) ** 2 + 4 * a * minus_c) - b - x, log_shift * 2 * a + ), + axis=0, + ) + - 2 * a + ) + + n = 10 + a = np.random.rand() + b = np.random.rand(6, n) + minus_c = np.random.rand(6, n) + log_shift = 0.1 + sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=log_shift, tol=tol) + v = func_abc(sol, a, b, minus_c, log_shift) + np.testing.assert_allclose(v, np.zeros([n]), atol=tol) + + +def test_dicotom_aq2(): + k = 5 + p = 6400 + maxit = 100 + span = np.logspace(-6, 5, num=17) + its = 25 + tol = 1e-6 + + # def func_abc(x, a, b, minus_c): + # n_p = len(b) + # return n_p * x + 2*a + np.sum( - np.sqrt( (b + x)**2 + 4*a*minus_c) + b, axis=0) + def func_abc(x, a, b, minus_c, log_shift=0): + return ( + np.sum( + np.maximum( + np.sqrt((b + x) ** 2 + 4 * a * minus_c) - b - x, log_shift * 2 * a + ), + axis=0, + ) + - 2 * a + ) + + np.random.seed(0) + for _ in range(its): + a = np.random.rand() + scale_num = np.random.choice(span, size=(k, p)) + b = scale_num * np.random.rand(k, p) + scale_denum = np.random.choice(span, size=(k, 1)) + minus_c = scale_denum * np.random.rand(k, 1) + sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol, maxit=maxit) + np.testing.assert_allclose( + func_abc(sol, a, b, minus_c), np.zeros([p]), atol=tol + ) + + log_shift = 0.1 / k + sol = dichotomy_simplex_acc( + a, b, minus_c, log_shift=log_shift, tol=tol, maxit=maxit + ) + np.testing.assert_allclose( + func_abc(sol, a, b, minus_c, log_shift=log_shift), np.zeros([p]), atol=tol + ) + + for _ in range(its): + a = np.random.rand() + scale_num = np.random.choice(span, size=(k, p)) + b = scale_num * np.random.rand(k, p) + scale_denum = np.random.choice(span, size=(k, p)) + minus_c = scale_denum * np.random.rand(k, p) + sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol, maxit=maxit) + np.testing.assert_allclose( + func_abc(sol, a, b, minus_c), np.zeros([p]), atol=tol + ) + + log_shift = 0.1 / k + sol = dichotomy_simplex_acc( + a, b, minus_c, log_shift=log_shift, tol=tol, maxit=maxit + ) + np.testing.assert_allclose( + func_abc(sol, a, b, minus_c, log_shift=log_shift), np.zeros([p]), atol=tol + ) + + for _ in range(its): + a = np.random.rand() + scale_num = np.random.choice(span, size=(k, p)) + b = scale_num * np.random.rand(k, p) + b[np.tile(np.arange(k), p // k), np.arange(p)] = 0 + scale_denum = np.random.choice(span, size=(k, p)) + minus_c = scale_denum * np.random.rand(k, p) + sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol, maxit=maxit) + np.testing.assert_allclose( + func_abc(sol, a, b, minus_c), np.zeros([p]), atol=tol + ) + + log_shift = 0.1 / k + sol = dichotomy_simplex_acc( + a, b, minus_c, log_shift=log_shift, tol=tol, maxit=maxit + ) + np.testing.assert_allclose( + func_abc(sol, a, b, minus_c, log_shift=log_shift), np.zeros([p]), atol=tol + ) + + for _ in range(its): + a = np.random.rand() + scale_num = np.random.choice(span, size=(k, p)) + b = scale_num * np.random.rand(k, p) + b[np.tile(np.arange(k), p // k), np.arange(p)] = 0 + scale_denum = np.random.choice(span, size=(k, 1)) + minus_c = scale_denum * np.random.rand(k, 1) + sol = dichotomy_simplex_acc(a, b, minus_c, log_shift=0, tol=tol, maxit=maxit) + np.testing.assert_allclose( + func_abc(sol, a, b, minus_c), np.zeros([p]), atol=tol + ) + + log_shift = 0.1 / k + sol = dichotomy_simplex_acc( + a, b, minus_c, log_shift=log_shift, tol=tol, maxit=maxit + ) + np.testing.assert_allclose( + func_abc(sol, a, b, minus_c, log_shift=log_shift), np.zeros([p]), atol=tol + ) + + +def test_multiplicative_step_w(): + l = 26 + k = 5 + p = 100 + c = 17 + + A = np.random.rand(k, p) + A = A / np.sum(A, axis=1, keepdims=True) + + G = np.random.rand(l, c) + P = np.random.rand(c, k) + GP = G @ P + + X = GP @ A + + Pp = multiplicative_step_w(X, G, P, A, log_shift=0, simplex_W=False) + np.testing.assert_array_almost_equal(Pp, P) + + Pp = multiplicative_step_w(X, G, P, A, log_shift=0, l2=True, simplex_W=False) + np.testing.assert_array_almost_equal(Pp, P) + + for _ in range(10): + P = np.random.rand(c, k) + Pp = multiplicative_step_w(X, G, P, A, simplex_W=False) + Pp2 = make_step_p(X, G, P, A) + np.testing.assert_array_almost_equal(Pp, Pp2) + GP = G @ P + GPp = G @ Pp + val1 = KLdiv_loss(X, GP, A) + val2 = KLdiv_loss(X, GPp, A) + np.testing.assert_array_less(0, Pp) + assert val1 > val2 + + for _ in range(10): + P = np.random.rand(c, k) + Pp = multiplicative_step_w(X, G, P, A, l2=True, simplex_W=False) + GP = G @ P + GPp = G @ Pp + val1 = Frobenius_loss(X, GP, A) + val2 = Frobenius_loss(X, GPp, A) + np.testing.assert_array_less(0, Pp) + assert val1 > val2 + + +def test_multiplicative_step_h(): + l = 26 + k = 5 + p = 100 + c = 17 + + A = np.random.rand(k, p) + A = A / np.sum(A, axis=0, keepdims=True) + + G = np.random.rand(l, c) + P = np.random.rand(c, k) + GP = G @ P + + X = GP @ A + np.testing.assert_allclose( + np.sum(A, axis=0), np.ones([A.shape[1]]), atol=dicotomy_tol + ) + + Ap = multiplicative_step_h( + X, G, P, A, simplex_H=False, mu=0, log_shift=0, epsilon_reg=1, safe=True + ) + np.testing.assert_array_almost_equal(A, Ap) + np.testing.assert_allclose( + np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol + ) + + Ap = multiplicative_step_h( + X, G, P, A, simplex_H=True, mu=0, log_shift=0, epsilon_reg=1, safe=True + ) + np.testing.assert_allclose(A, Ap, atol=dicotomy_tol) + + # Same test for l2 + Ap = multiplicative_step_h( + X, + G, + P, + A, + simplex_H=False, + mu=0, + log_shift=0, + epsilon_reg=1, + safe=True, + l2=True, + ) + np.testing.assert_array_almost_equal(A, Ap) + np.testing.assert_allclose( + np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol + ) + + Ap = multiplicative_step_h( + X, G, P, A, simplex_H=True, mu=0, log_shift=0, epsilon_reg=1, safe=True, l2=True + ) + np.testing.assert_allclose(A, Ap, atol=dicotomy_tol) + + for _ in range(10): + A = np.random.rand(k, p) + A = A / np.sum(A, axis=1, keepdims=True) + Ap = multiplicative_step_h( + X, G, P, A, simplex_H=False, mu=0, log_shift=0, epsilon_reg=1, safe=True + ) + val1 = KLdiv_loss(X, GP, A) + val2 = KLdiv_loss(X, GP, Ap) + np.testing.assert_array_less(0, Ap) + assert val1 > val2 + + Ap = multiplicative_step_h( + X, + G, + P, + A, + simplex_H=True, + mu=0, + log_shift=log_shift, + epsilon_reg=1, + safe=True, + ) + Ap2 = make_step_a( + X, G, P, A, mu_sparse=0, eps=log_shift, eps_sparse=1, mask=None + ) + np.testing.assert_array_almost_equal(Ap2, Ap) + np.testing.assert_allclose( + np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol + ) + + val1 = KLdiv_loss(X, GP, A) + val2 = KLdiv_loss(X, GP, Ap) + np.testing.assert_array_less(0, Ap) + assert val1 > val2 + + epsilon_reg = 1 + mu = np.ones(k) + mu[0] = 0 + Ap = multiplicative_step_h( + X, + G, + P, + A, + simplex_H=True, + mu=mu, + log_shift=log_shift, + epsilon_reg=epsilon_reg, + safe=True, + ) + Ap2 = make_step_a( + X, G, P, A, mu_sparse=1, eps=log_shift, eps_sparse=epsilon_reg, mask=None + ) + np.testing.assert_array_almost_equal(Ap2, Ap) + np.testing.assert_allclose( + np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol + ) + + Ap = multiplicative_step_h( + X, + G, + P, + A, + simplex_H=True, + mu=3 * mu, + log_shift=log_shift, + epsilon_reg=epsilon_reg, + safe=True, + ) + Ap2 = make_step_a( + X, G, P, A, mu_sparse=3, eps=log_shift, eps_sparse=epsilon_reg, mask=None + ) + np.testing.assert_array_almost_equal(Ap2, Ap) + np.testing.assert_allclose( + np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol + ) + + val1 = KLdiv_loss(X, GP, A) + log_reg(A, 3 * mu, epsilon_reg) + val2 = KLdiv_loss(X, GP, Ap) + log_reg(A, 3 * mu, epsilon_reg) + np.testing.assert_array_less(0, Ap) + assert val1 > val2 + + # Ap = multiplicative_step_h(X, G, P, A, simplex_H=True, mu=3, eps=log_shift, epsilon_reg=1, safe=True) + # Ap2 = make_step_a(X, G, P, A, mu_sparse=3, eps=log_shift, eps_sparse=1, mask=np.zeros([k])>0) + # np.testing.assert_array_almost_equal(Ap2, Ap) + # np.testing.assert_allclose(np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol) + + for _ in range(10): + A = np.random.rand(k, p) + A = A / np.sum(A, axis=1, keepdims=True) + Ap = multiplicative_step_h( + X, + G, + P, + A, + simplex_H=False, + mu=0, + log_shift=0, + epsilon_reg=1, + safe=True, + l2=True, + ) + val1 = Frobenius_loss(X, GP, A) + val2 = Frobenius_loss(X, GP, Ap) + np.testing.assert_array_less(0, Ap) + assert val1 > val2 + + Ap = multiplicative_step_h( + X, + G, + P, + A, + simplex_H=True, + mu=0, + log_shift=log_shift, + epsilon_reg=1, + safe=True, + l2=True, + ) + np.testing.assert_allclose( + np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol + ) + + val1 = Frobenius_loss(X, GP, A) + val2 = Frobenius_loss(X, GP, Ap) + np.testing.assert_array_less(0, Ap) + assert val1 > val2 + + +def test_multiplicative_step_hq(): + + l = 26 + k = 5 + p = 100 + c = 17 + + A = np.random.rand(k, p) + A = A / np.sum(A, axis=0, keepdims=True) + + G = np.random.rand(l, c) + P = np.random.rand(c, k) + GP = G @ P + + X = GP @ A + np.testing.assert_allclose( + np.sum(A, axis=0), np.ones([A.shape[1]]), atol=dicotomy_tol + ) + + Ap = multiplicative_step_hq(X, G, P, A, simplex_H=False, log_shift=0, safe=True) + np.testing.assert_array_almost_equal(A, Ap) + np.testing.assert_allclose( + np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol + ) + + Ap = multiplicative_step_hq(X, G, P, A, simplex_H=True, log_shift=0, safe=True) + np.testing.assert_allclose(A, Ap, atol=dicotomy_tol) + + for _ in range(10): + A = np.random.rand(k, p) + A = A / np.sum(A, axis=1, keepdims=True) + Ap = multiplicative_step_hq(X, G, P, A, simplex_H=False, log_shift=0, safe=True) + val1 = KLdiv_loss(X, GP, A) + val2 = KLdiv_loss(X, GP, Ap) + np.testing.assert_array_less(0, Ap) + assert val1 > val2 + + Ap = multiplicative_step_hq( + X, G, P, A, simplex_H=True, log_shift=log_shift, safe=True + ) + Ap2 = make_step_a( + X, G, P, A, mu_sparse=0, eps=log_shift, eps_sparse=1, mask=None + ) + np.testing.assert_array_almost_equal(Ap2, Ap) + np.testing.assert_allclose( + np.sum(Ap, axis=0), np.ones([Ap.shape[1]]), atol=dicotomy_tol + ) + + val1 = KLdiv_loss(X, GP, A) + val2 = KLdiv_loss(X, GP, Ap) + np.testing.assert_array_less(0, Ap) + assert val1 > val2 + + +def test_Q_step(): + l = 4 + p = 5 + i = 3 + + D = np.random.randn(l, i) + A = np.random.randn(i, p) + + Q = np.zeros([l, p, i]) + for a in range(l): + for b in range(p): + for c in range(i): + Q[a, b, c] = A[c, b] * D[a, c] / np.sum(A[:, b] * D[a, :]) + Q2 = update_q(D, A) + np.testing.assert_array_almost_equal(Q, Q2) + + +def test_Q_step2(): + l = 4 + p = 5 + i = 3 + + D = np.random.randn(l, i) + A = np.random.randn(i, p) + A[0, :] = 0 + A[:, 0] = 0 + D[0, :] = 0 + D[:, 0] = 0 + + Q = np.zeros([l, p, i]) + for a in range(l): + for b in range(p): + for c in range(i): + Q[a, b, c] = A[c, b] * D[a, c] / (np.sum(A[:, b] * D[a, :]) + log_shift) + Q2 = update_q(D, A) + np.testing.assert_array_almost_equal(Q, Q2) + assert (np.isnan(Q) == False).all() + + +def create_toy_problem(l=25, k=3, p=100, c=10, n_poisson=200, simplex_H=True): + + A = np.random.rand(k, p) + if simplex_H: + A = A / np.sum(A, axis=0, keepdims=True) + + G = np.random.rand(l, c) + P = np.random.rand(c, k) + GP = G @ P + + X = GP @ A + + Xdot = 1 / n_poisson * np.random.poisson(n_poisson * X) + + return G, P, A, X, Xdot + + +def test_proj_step_h(): + + def full_loss(X, G, W, H, log_shift, mu, lambda_L, L): + return ( + KLdiv_loss(X, G @ W, H, log_shift=log_shift) + + lambda_L * trace_xtLx(L, H.T) + + log_reg(H, mu) + ) + + shape_2d = [10, 15] + k = 5 + n_poisson = 200 + log_shift = 0.01 + epsilon_reg = 2 + L = create_laplacian_matrix(*shape_2d) + + for _ in range(10): + G, W, H, Xtrue, X = create_toy_problem( + p=shape_2d[0] * shape_2d[1], k=k, n_poisson=n_poisson + ) + + true_D = G @ W + true_H = H + + for simplex_H in [True, False]: + for lambda_L in [0, 1, 10]: + for mu in [0, 1, 10]: + W0 = np.random.rand(*W.shape) + H0 = np.random.rand(*H.shape) + W0 = np.maximum(W0, log_shift) + H0 = np.maximum(H0, log_shift) + X = np.maximum(X, log_shift) + + gamma_h = estimate_Lipschitz_bound_h( + log_shift, + X, + G, + k, + mu=mu, + lambda_L=lambda_L, + epsilon_reg=epsilon_reg, + ) + + loss1 = full_loss(X, G, W0, H0, log_shift, mu, lambda_L, L) + + grad = gradH( + X, + G, + W0, + H0, + log_shift=log_shift, + l2=False, + mu=mu, + lambda_L=lambda_L, + L=L, + epsilon_reg=epsilon_reg, + ) + H1 = H0 - 1 / gamma_h * grad + loss2 = full_loss(X, G, W0, H1, log_shift, mu, lambda_L, L) + H2 = proj_grad_step_h( + X, + G, + W0, + H0, + gamma_h, + log_shift=log_shift, + safe=True, + l2=False, + simplex_H=simplex_H, + mu=mu, + lambda_L=lambda_L, + L=L, + epsilon_reg=epsilon_reg, + ) + loss3 = full_loss(X, G, W0, H2, log_shift, mu, lambda_L, L) + assert loss1 >= loss2 + assert loss1 >= loss3 + assert (H2 >= log_shift).all() + + maxit = 10 + loss_old = loss3 + grad = gradH(X, G, W0, H2, log_shift=log_shift, l2=False) + n_grad_old = np.sum(grad**2) + for _ in range(maxit): + H2 = proj_grad_step_h( + X, + G, + W0, + H2, + gamma_h, + log_shift=log_shift, + safe=True, + l2=False, + simplex_H=simplex_H, + mu=mu, + lambda_L=lambda_L, + L=L, + epsilon_reg=epsilon_reg, + ) + grad = gradH( + X, + G, + W0, + H2, + log_shift=log_shift, + mu=mu, + lambda_L=lambda_L, + L=L, + l2=False, + ) + n_grad = np.sum(grad**2) + loss = full_loss(X, G, W0, H2, log_shift, mu, lambda_L, L) + assert loss <= loss_old + np.abs(loss_old) * 0.00001 + # assert n_grad<=n_grad_old + loss_old = loss + n_grad_old = n_grad + + +def test_proj_step_w(): + for _ in range(10): + shape_2d = [10, 15] + log_shift = 0.01 + + k = 5 + n_poisson = 200 + G, W, H, Xtrue, X = create_toy_problem( + p=shape_2d[0] * shape_2d[1], k=k, n_poisson=n_poisson + ) + + true_D = G @ W + true_H = H + + W0 = np.random.rand(*W.shape) + H0 = np.random.rand(*H.shape) + + gamma_w = estimate_Lipschitz_bound_w(log_shift, X, G, k) + + loss1 = KLdiv_loss(X, G @ W0, H0, log_shift=log_shift) + grad = gradW(X, G, W0, H0, log_shift=log_shift, l2=False) + W1 = W0 - 1 / gamma_w * grad + loss2 = KLdiv_loss(X, G @ W1, H0, log_shift=log_shift) + W2 = proj_grad_step_w( + X, + G, + W0, + H0, + gamma_w, + log_shift=log_shift, + safe=True, + l2=False, + simplex_W=False, + ) + loss3 = KLdiv_loss(X, G @ W2, H0, log_shift=log_shift) + assert loss1 >= loss2 + assert loss1 >= loss3 + assert (W2 >= log_shift).all() + + maxit = 10 + loss_old = loss3 + grad = gradW(X, G, W2, H0, log_shift=log_shift, l2=False) + n_grad_old = np.sum(grad**2) + for _ in range(maxit): + W2 = proj_grad_step_w( + X, + G, + W2, + H0, + gamma_w, + log_shift=log_shift, + safe=True, + l2=False, + simplex_W=False, + ) + grad = gradW(X, G, W2, H0, log_shift=log_shift, l2=False) + + n_grad = np.sum(grad**2) + loss = KLdiv_loss(X, G @ W2, H0, log_shift=log_shift) + assert loss <= loss_old + assert n_grad <= n_grad_old + loss_old = loss + n_grad_old = n_grad diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..6dba58b --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,133 @@ +import numpy as np +from exspy.material import density_of_mixture + +import espm.utils as u + + +def test_rescale(): + np.random.seed(0) + for _ in range(20): + # for k small + W = np.random.rand(27, 5) + H = np.random.rand(5, 150) + W_r, H_r = u.rescaled_DH(W, H) + assert np.abs(np.mean(H_r.sum(axis=0)) - 1) < np.abs(np.mean(H.sum(axis=0)) - 1) + np.testing.assert_array_almost_equal(W @ H, W_r @ H_r) + + # for k large + W = np.random.rand(50, 10) + H = np.random.rand(10, 5) + W_r, H_r = u.rescaled_DH(W, H) + np.testing.assert_array_almost_equal(W @ H, W_r @ H_r) + np.testing.assert_allclose(H_r.sum(axis=0), 1, atol=0.2) + + +def test_number_symbols(): + mixed_dict = {"Fe": 1.0, "Si": 2.0, 27: 3.0, "83": 4.0, 8: 5.0} + symbol_dict = {"Fe": 1.0, "Si": 2.0, "Co": 3.0, "Bi": 4.0, "O": 5.0} + number_dict = {26: 1.0, 14: 2.0, 27: 3.0, 83: 4.0, 8: 5.0} + + mixed_list = ["Fe", "Si", 27, "83", 8] + number_list = [26, 14, 27, 83, 8] + symbol_list = ["Fe", "Si", "Co", "Bi", "O"] + + @u.symbol_to_number_dict + def s_to_n(elements_dict={}): + return elements_dict + + @u.number_to_symbol_dict + def n_to_s(elements_dict={}): + return elements_dict + + @u.symbol_to_number_list + def s_to_n_list(elements=[]): + return elements + + @u.number_to_symbol_list + def n_to_s_list(elements=[]): + return elements + + assert s_to_n(elements_dict=mixed_dict) == number_dict + assert n_to_s(elements_dict=mixed_dict) == symbol_dict + assert s_to_n_list(elements=mixed_list) == number_list + assert n_to_s_list(elements=mixed_list) == symbol_list + + +def test_is_symbol(): + symbols = ["Si", "45", 34, "si", "Zbrra"] + bools = [True, False, False, False, False] + + bools_test = [] + for i in symbols: + bools_test.append(u.is_symbol(i)) + + assert bools == bools_test + + +def test_hspy_wrappers(): + atomic_fractions = {8: 0.2, 14: 0.3, 26: 0.1, 83: 0.4} + weight_fractions = { + "O": 0.03174415159035731, + "Si": 0.08358598161408992, + "Fe": 0.055400582070687154, + "Bi": 0.8292692847248655, + } + + res_wt = u.atomic_to_weight_dict(elements_dict=atomic_fractions) + assert res_wt == weight_fractions + + res_dens = u.approx_density(atomic_fraction=True, elements_dict=atomic_fractions) + hspy_dens = density_of_mixture( + [ + 0.03174415159035731, + 0.08358598161408992, + 0.055400582070687154, + 0.8292692847248655, + ], + ["O", "Si", "Fe", "Bi"], + ) + + assert res_dens == hspy_dens + + +def test_arg_helper(): + default_dict = { + "chou": "non", + "carottes": 3, + "salades": {"roquette": 2, "batavia": 5}, + "tubercules": { + "oignons": {"jaune": [1, 2, 3], "blanc": [3, 2, 1]}, + "echalottes": 5, + }, + } + + input_dict = { + "chou": 4, + "navets": 3, + "salades": {"clous": 3, "roquette": 2}, + "tubercules": { + "oignons": {"jaune": 9}, + "vis": {"cruciformes": 3, "hexagonales": 6}, + }, + } + + result_dict = { + "chou": 4, + "carottes": 3, + "navets": 3, + "salades": {"clous": 3, "roquette": 2, "batavia": 5}, + "tubercules": { + "oignons": {"jaune": 9, "blanc": [3, 2, 1]}, + "echalottes": 5, + "vis": {"cruciformes": 3, "hexagonales": 6}, + }, + } + + assert u.arg_helper(input_dict, default_dict) == result_dict + + +def test_bin_spim(): + array = np.random.rand(100, 20, 30) + + assert u.bin_spim(array, 50, 10).shape == (50, 10, 30) + assert u.bin_spim(array, 30, 6).shape == (30, 6, 30)