From 485334f4d8d4eb6568f7f691dd18b14076ea751d Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 6 Jul 2026 19:17:33 -0700 Subject: [PATCH 1/7] update: FE NB --- .../workflows/formation_energy.ipynb | 663 ++++++++++++++++++ 1 file changed, 663 insertions(+) create mode 100644 other/materials_designer/workflows/formation_energy.ipynb diff --git a/other/materials_designer/workflows/formation_energy.ipynb b/other/materials_designer/workflows/formation_energy.ipynb new file mode 100644 index 00000000..4944894c --- /dev/null +++ b/other/materials_designer/workflows/formation_energy.ipynb @@ -0,0 +1,663 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Formation Energy\n", + "\n", + "Calculate the formation energy of a compound using a DFT workflow on the Mat3ra platform.\n", + "\n", + "This notebook loads a compound material, resolves Standata elemental reference materials for its constituent elements, and checks that refined `total_energy` properties already exist for each elemental reference.\n", + "\n", + "If any elemental reference material or elemental total energy is missing, the notebook stops and points to the Total Energy notebook. It does not create or run elemental pre-calculations automatically.\n", + "\n", + "

Usage

\n", + "\n", + "1. Put a compound material JSON into `../uploads`, or set `MATERIAL_NAME` to match a compound in Standata or on the platform.\n", + "2. Set the material and workflow parameters in cell 1.2 below.\n", + "3. Click \"Run\" > \"Run All\".\n", + "4. If elemental total energies are missing, run the Total Energy notebook for the corresponding Standata elemental material(s) first, then rerun this notebook.\n", + "5. Inspect the elemental references, their total energies used by the workflow, and the final formation energy.\n", + "\n", + "## Summary\n", + "\n", + "1. Set up the environment and parameters.\n", + "2. Authenticate and initialize API client.\n", + "3. Load compound material and resolve unique elements.\n", + "4. Resolve Standata elemental reference materials and check refined elemental total energies.\n", + "5. Configure and save the Formation Energy workflow.\n", + "6. Configure compute.\n", + "7. Create, submit, and monitor the Formation Energy job.\n", + "8. Retrieve results.\n" + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## 1. Set up the environment and parameters\n", + "### 1.1. Install packages (JupyterLite)\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.packages import install_packages\n", + "\n", + "await install_packages(\"made|api_examples\")" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "### 1.2. Set parameters and configurations for the workflow and job\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from mat3ra.ide.compute import QueueName\n", + "\n", + "# 2. Auth and organization parameters\n", + "# Set organization name to use it as the owner, otherwise your personal account is used\n", + "ORGANIZATION_NAME = None\n", + "\n", + "# 3. Material parameters\n", + "FOLDER = \"../uploads\"\n", + "MATERIAL_NAME = \"Silicon\" # Name of the compound to load from uploads folder, Standata, or platform\n", + "\n", + "# 4. Workflow parameters\n", + "WORKFLOW_SEARCH_TERM = \"formation_energy.json\" # Search term for Workflows Standata\n", + "APPLICATION_NAME = \"espresso\" # Application for the QE formation-energy subworkflow\n", + "SCF_KGRID = None # e.g., [10, 10, 10]\n", + "MY_WORKFLOW_NAME = \"Formation Energy\"\n", + "\n", + "# 5. Compute parameters\n", + "CLUSTER_NAME = None # specify full or partial name i.e. \"cluster-001\" to select\n", + "QUEUE_NAME = QueueName.D\n", + "PPN = 1\n", + "\n", + "# 6. Job parameters\n", + "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", + "POLL_INTERVAL = 30 # seconds\n" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "## 2. Authenticate and initialize API client\n", + "### 2.1. Authenticate\n", + "Authenticate in the browser and have credentials stored in environment variable `OIDC_ACCESS_TOKEN`.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.auth import authenticate\n", + "\n", + "await authenticate()" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "### 2.2. Initialize API client\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.api_client import APIClient\n", + "\n", + "client = APIClient.authenticate()\n", + "client\n" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "### 2.3. Select account to work under\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "client.list_accounts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "selected_account = client.my_account\n", + "\n", + "if ORGANIZATION_NAME:\n", + " selected_account = client.get_account(name=ORGANIZATION_NAME)\n", + "\n", + "ACCOUNT_ID = selected_account.id\n", + "print(f\"✅ Selected account ID: {ACCOUNT_ID}, name: {selected_account.name}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "12", + "metadata": {}, + "source": [ + "### 2.4. Select project\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "## 3. Load compound material\n", + "### 3.1. Load material from local file, Standata, or platform\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "from mat3ra.made.material import Material\n", + "from mat3ra.standata.materials import Materials\n", + "from mat3ra.notebooks_utils.material import load_material_from_folder\n", + "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", + "\n", + "material = load_material_from_folder(FOLDER, MATERIAL_NAME)\n", + "\n", + "if material is None:\n", + " try:\n", + " material = Material.create(Materials.get_by_name_first_match(MATERIAL_NAME))\n", + " print(f\"✅ Loaded material from Standata: {material.name}\")\n", + " except Exception:\n", + " material_matches = client.materials.list({\n", + " \"name\": {\"$regex\": re.escape(MATERIAL_NAME), \"$options\": \"i\"},\n", + " })\n", + "\n", + " material = Material.create(material_matches[0])\n", + " print(f\"♻️ Loaded material from platform: {material_matches[0]['_id']}\")\n", + "else:\n", + " print(f\"✅ Loaded material from folder: {material.name}\")\n", + "\n", + "visualize(material)" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "### 3.2. Resolve unique elements in the compound\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.ui import display_JSON\n", + "\n", + "elements = sorted(set(material.basis.elements.values))\n", + "print(f\"Compound elements: {elements}\")\n", + "display_JSON({\"elements\": elements}, level=2)" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "## 4. Resolve elemental reference materials and check total energies\n", + "### 4.1. Resolve Standata elemental reference materials\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "# Query for elemental reference materials (curators' account materials)\n", + "# These have tags: 'elemental' and metadata.element set to the element symbol\n", + "elemental_materials_data = client.bank_materials.list(\n", + " {\"tags\": \"elemental\", \"metadata.element\": {\"$in\": elements}},\n", + " projection={\"limit\": 100},\n", + ")\n", + "\n", + "element_materials = {\n", + " item[\"metadata\"][\"element\"]: item\n", + " for item in elemental_materials_data\n", + " if item.get(\"metadata\") and item[\"metadata\"].get(\"element\") in elements\n", + "}\n", + "\n", + "missing_elemental_material_symbols = [element for element in elements if element not in element_materials]\n", + "if missing_elemental_material_symbols:\n", + " raise RuntimeError(\n", + " \"Missing Standata elemental reference material(s) for \"\n", + " f\"{ ', '.join(missing_elemental_material_symbols)}. \"\n", + " \"Import the corresponding elemental material(s) from Standata before running Formation Energy.\"\n", + " )\n", + "\n", + "print(f\"Resolved elemental reference materials for: {', '.join(elements)}\")\n", + "for symbol, mat in element_materials.items():\n", + " print(f\" {symbol}: {mat['name']} (_id={mat['_id']}, exabyteId={mat.get('exabyteId', 'N/A')})\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "20", + "metadata": {}, + "source": [ + "### 4.2. Check refined elemental total energies\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "\n", + "def get_elemental_total_energy_holders(api_client, symbols, element_materials_by_symbol):\n", + " \"\"\"Get the best total_energy property for each elemental reference material.\"\"\"\n", + " holders_by_symbol = {}\n", + " for symbol in symbols:\n", + " mat = element_materials_by_symbol[symbol]\n", + " exabyte_id = mat.get(\"exabyteId\", mat[\"_id\"])\n", + " # Query properties by exabyteId + slug, sorted by precision descending\n", + " props = api_client.properties.list(\n", + " query={\n", + " \"exabyteId\": exabyte_id,\n", + " \"slug\": \"total_energy\",\n", + " },\n", + " projection={\"sort\": {\"precision.value\": -1}, \"limit\": 1},\n", + " )\n", + " holders_by_symbol[symbol] = props[0] if props else None\n", + " return holders_by_symbol\n", + "\n", + "\n", + "elemental_total_energy_holders = get_elemental_total_energy_holders(\n", + " client,\n", + " elements,\n", + " element_materials,\n", + ")\n", + "\n", + "missing_elemental_total_energy_symbols = [\n", + " symbol for symbol in elements if elemental_total_energy_holders[symbol] is None\n", + "]\n", + "if missing_elemental_total_energy_symbols:\n", + " raise RuntimeError(\n", + " \"Missing total_energy for elemental reference material(s) \"\n", + " f\"{ ', '.join(missing_elemental_total_energy_symbols)}. \"\n", + " \"Run total_energy.ipynb for the corresponding Standata elemental material(s) first.\"\n", + " )\n", + "\n", + "for symbol in elements:\n", + " holder = elemental_total_energy_holders[symbol]\n", + " print(\n", + " f\"♻️ Found total energy for {symbol}: \"\n", + " f\"{holder.get('data', {}).get('value', 'N/A')} eV \"\n", + " f\"(precision: {holder.get('precision', {}).get('value', 'N/A')})\"\n", + " )\n" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "### 4.3. Save compound material for the workflow\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.core.entity.material.api import get_or_create_material\n", + "\n", + "saved_material_response = get_or_create_material(client, material, ACCOUNT_ID)\n", + "saved_material = Material.create(saved_material_response)\n", + "\n", + "print(f\"✅ Saved compound material: {saved_material_response['_id']}\")" + ] + }, + { + "cell_type": "markdown", + "id": "24", + "metadata": {}, + "source": [ + "## 5. Configure the Formation Energy workflow\n", + "### 5.1. Select application\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.ade.application import Application\n", + "from mat3ra.standata.applications import ApplicationStandata\n", + "\n", + "app_config = ApplicationStandata.get_by_name_first_match(APPLICATION_NAME)\n", + "app = Application(**app_config)\n", + "print(f\"Using application: {app.name}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "26", + "metadata": {}, + "source": [ + "### 5.2. Load workflow from Standata and preview it\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.standata.workflows import WorkflowStandata\n", + "from mat3ra.wode.context.providers import PointsGridDataProvider\n", + "from mat3ra.wode.workflows import Workflow\n", + "from mat3ra.notebooks_utils.core.entity.workflow.api import get_or_create_workflow\n", + "from mat3ra.notebooks_utils.ipython.entity.workflow.visualize import visualize_workflow\n", + "\n", + "\n", + "def apply_workflow_kgrids(workflow: Workflow, scf_kgrid=None) -> Workflow:\n", + " if scf_kgrid is not None:\n", + " new_context_scf = PointsGridDataProvider(dimensions=scf_kgrid, isEdited=True).get_context_item_data()\n", + " for subworkflow in workflow.subworkflows:\n", + " unit_names = [unit.name for unit in subworkflow.units]\n", + " if \"pw_scf\" not in unit_names:\n", + " continue\n", + " unit_to_modify_scf = subworkflow.get_unit_by_name(name=\"pw_scf\")\n", + " unit_to_modify_scf.add_context(new_context_scf)\n", + " subworkflow.set_unit(unit_to_modify_scf)\n", + " break\n", + " return workflow\n", + "\n", + "\n", + "formation_workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(\n", + " WORKFLOW_SEARCH_TERM\n", + ")\n", + "formation_workflow = Workflow.create(formation_workflow_config)\n", + "formation_workflow.name = MY_WORKFLOW_NAME\n", + "formation_workflow = apply_workflow_kgrids(formation_workflow, scf_kgrid=SCF_KGRID)\n", + "\n", + "visualize_workflow(formation_workflow)" + ] + }, + { + "cell_type": "markdown", + "id": "28", + "metadata": {}, + "source": [ + "### 5.3. Save workflow to collection\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "saved_formation_workflow_response = get_or_create_workflow(client, formation_workflow, ACCOUNT_ID)\n", + "saved_formation_workflow = Workflow.create(saved_formation_workflow_response)\n", + "print(f\"Formation workflow ID: {saved_formation_workflow.id}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "30", + "metadata": {}, + "source": [ + "## 6. Create the compute configuration\n", + "### 6.1. Select cluster\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31", + "metadata": {}, + "outputs": [], + "source": [ + "clusters = client.clusters.list()\n", + "print(f\"Available clusters: {[c['hostname'] for c in clusters]}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "32", + "metadata": {}, + "source": [ + "### 6.2. Create compute configuration\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.ide.compute import Compute\n", + "\n", + "if CLUSTER_NAME:\n", + " cluster = next((c for c in clusters if CLUSTER_NAME in c[\"hostname\"]), None)\n", + "else:\n", + " cluster = clusters[0]\n", + "\n", + "compute = Compute(cluster=cluster, queue=QUEUE_NAME, ppn=PPN)\n", + "print(f\"Using cluster: {compute.cluster.hostname}, queue: {QUEUE_NAME}, ppn: {PPN}\")\n" + ] + }, + { + "cell_type": "markdown", + "id": "34", + "metadata": {}, + "source": [ + "## 7. Create the Formation Energy job\n", + "### 7.1. Create job with compound material and workflow configuration\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.job import create_job\n", + "from mat3ra.notebooks_utils.api.job import wait_for_jobs_to_finish_async\n", + "from mat3ra.utils.namespace import dict_to_namespace_recursive\n", + "\n", + "print(f\"Compound material: {saved_material.id}\")\n", + "print(f\"Elemental references: {', '.join(elements)}\")\n", + "print(f\"Formation workflow: {saved_formation_workflow.id}\")\n", + "print(f\"Project: {project_id}\")\n", + "\n", + "formation_job_name = f\"{MY_WORKFLOW_NAME} {saved_material.formula} {timestamp}\"\n", + "formation_job_response = create_job(\n", + " api_client=client,\n", + " materials=[saved_material],\n", + " workflow=formation_workflow,\n", + " project_id=project_id,\n", + " owner_id=ACCOUNT_ID,\n", + " prefix=formation_job_name,\n", + " compute=compute.to_dict(),\n", + ")\n", + "\n", + "formation_job = dict_to_namespace_recursive(formation_job_response)\n", + "formation_job_id = formation_job._id\n", + "print(f\"✅ Formation Energy job created successfully: {formation_job_id}\")\n", + "display_JSON(formation_job_response)" + ] + }, + { + "cell_type": "markdown", + "id": "36", + "metadata": {}, + "source": [ + "### 7.2. Submit the Formation Energy job and monitor the status\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37", + "metadata": {}, + "outputs": [], + "source": [ + "client.jobs.submit(formation_job_id)\n", + "print(f\"✅ Job {formation_job_id} submitted successfully!\")\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38", + "metadata": {}, + "outputs": [], + "source": [ + "await wait_for_jobs_to_finish_async(client.jobs, [formation_job_id], poll_interval=POLL_INTERVAL)\n" + ] + }, + { + "cell_type": "markdown", + "id": "39", + "metadata": {}, + "source": [ + "## 8. Retrieve results\n", + "### 8.1. Retrieve and visualize Formation Energy results\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.prode import PropertyName\n", + "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", + "\n", + "formation_energy_data = client.properties.get_for_job(formation_job_id, property_name=\"formation_energy\")\n", + "formation_energy_contributions_data = client.properties.get_for_job(\n", + " formation_job_id,\n", + " property_name=\"formation_energy_contributions\",\n", + ")\n", + "compound_total_energy_data = client.properties.get_for_job(\n", + " formation_job_id,\n", + " property_name=PropertyName.scalar.total_energy.value,\n", + ")\n", + "\n", + "if formation_energy_data:\n", + " visualize_properties(formation_energy_data, title=\"Formation Energy\")\n", + "else:\n", + " print(\"No 'formation_energy' property was returned for the job.\")\n", + "\n", + "if formation_energy_contributions_data:\n", + " visualize_properties(\n", + " formation_energy_contributions_data,\n", + " title=\"Formation Energy Contributions\",\n", + " )\n", + "else:\n", + " print(\"No 'formation_energy_contributions' property was returned for the job.\")\n", + "\n", + "if compound_total_energy_data:\n", + " visualize_properties(compound_total_energy_data, title=\"Compound Total Energy\")\n", + "else:\n", + " print(\"No compound total energy property was returned for the job.\")\n", + "\n", + "print(f\"Compound elements: {elements}\")\n", + "print(f\"Saved compound material used by the workflow: {saved_material_response['_id']}\")\n", + "for symbol in elements:\n", + " holder = elemental_total_energy_holders[symbol]\n", + " print(\n", + " f\"Refined elemental total energy for {symbol}: \"\n", + " f\"{element_materials[symbol]['name']} -> {holder['data']['value']}\"\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 2050124eac2c9f525241bea26977aa30099528a1 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 6 Jul 2026 19:20:41 -0700 Subject: [PATCH 2/7] update: intro nb --- other/materials_designer/workflows/Introduction.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/other/materials_designer/workflows/Introduction.ipynb b/other/materials_designer/workflows/Introduction.ipynb index 2f78cebe..eec8fc04 100644 --- a/other/materials_designer/workflows/Introduction.ipynb +++ b/other/materials_designer/workflows/Introduction.ipynb @@ -72,7 +72,7 @@ "#### 6.5.1. Defect formation energy. *(to be added)*\n", "\n", "### 6.6. Formation Energy\n", - "#### 6.6.1. Compound formation energy. *(to be added)*\n", + "#### [6.6.1. Compound formation energy.](formation_energy.ipynb)\n", "\n", "\n", "## 7. Chemistry\n", From 17d5d2f7c21f1d356b20b24cdbfca77c4b825087 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 6 Jul 2026 23:36:14 -0700 Subject: [PATCH 3/7] update: fix --- .../workflows/formation_energy.ipynb | 251 ++++++++++++------ 1 file changed, 163 insertions(+), 88 deletions(-) diff --git a/other/materials_designer/workflows/formation_energy.ipynb b/other/materials_designer/workflows/formation_energy.ipynb index 4944894c..c416ae07 100644 --- a/other/materials_designer/workflows/formation_energy.ipynb +++ b/other/materials_designer/workflows/formation_energy.ipynb @@ -86,12 +86,24 @@ "SCF_KGRID = None # e.g., [10, 10, 10]\n", "MY_WORKFLOW_NAME = \"Formation Energy\"\n", "\n", - "# 5. Compute parameters\n", + "# 5. Elemental total energy source\n", + "# Controls whose total_energy properties are used for elemental references:\n", + "# 'public' — any owner (highest precision wins)\n", + "# 'curators' — only curators' properties\n", + "# 'my_account' — curators' or your own properties\n", + "ELEMENTAL_TE_SOURCE = \"public\"\n", + "\n", + "# 5b. Property group filter for method consistency\n", + "# Ensures elemental total energies come from the same computational method.\n", + "# Format: {application_short_name}:{model_type}:{model_subtype}\n", + "COMPOUND_GROUP = \"qe:dft:gga:pbe\" # Quantum ESPRESSO + DFT/GGA/PBE\n", + "\n", + "# 6. Compute parameters\n", "CLUSTER_NAME = None # specify full or partial name i.e. \"cluster-001\" to select\n", "QUEUE_NAME = QueueName.D\n", "PPN = 1\n", "\n", - "# 6. Job parameters\n", + "# 7. Job parameters\n", "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", "POLL_INTERVAL = 30 # seconds\n" ] @@ -112,6 +124,22 @@ "id": "6", "metadata": {}, "outputs": [], + "source": [ + "# For local dev only. TODO: remove if commited\n", + "import os\n", + "\n", + "os.environ[\"API_PORT\"] = \"3000\"\n", + "os.environ[\"API_SECURE\"] = \"false\"\n", + "os.environ[\"API_HOST\"] = \"localhost\"\n", + "os.environ[\"OIDC_ACCESS_TOKEN\"] = \"TdpmY4sI6b6CXiOain_S4onc4h0x-uKJq17XttQcODS\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], "source": [ "from mat3ra.notebooks_utils.auth import authenticate\n", "\n", @@ -120,7 +148,7 @@ }, { "cell_type": "markdown", - "id": "7", + "id": "8", "metadata": {}, "source": [ "### 2.2. Initialize API client\n" @@ -129,7 +157,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -141,7 +169,7 @@ }, { "cell_type": "markdown", - "id": "9", + "id": "10", "metadata": {}, "source": [ "### 2.3. Select account to work under\n" @@ -150,7 +178,7 @@ { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -160,7 +188,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -175,7 +203,7 @@ }, { "cell_type": "markdown", - "id": "12", + "id": "13", "metadata": {}, "source": [ "### 2.4. Select project\n" @@ -184,7 +212,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -195,7 +223,7 @@ }, { "cell_type": "markdown", - "id": "14", + "id": "15", "metadata": {}, "source": [ "## 3. Load compound material\n", @@ -205,7 +233,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -236,7 +264,7 @@ }, { "cell_type": "markdown", - "id": "16", + "id": "17", "metadata": {}, "source": [ "### 3.2. Resolve unique elements in the compound\n" @@ -245,7 +273,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -258,7 +286,7 @@ }, { "cell_type": "markdown", - "id": "18", + "id": "19", "metadata": {}, "source": [ "## 4. Resolve elemental reference materials and check total energies\n", @@ -268,39 +296,57 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "20", "metadata": {}, "outputs": [], "source": [ - "# Query for elemental reference materials (curators' account materials)\n", - "# These have tags: 'elemental' and metadata.element set to the element symbol\n", - "elemental_materials_data = client.bank_materials.list(\n", + "# Query elemental reference materials from the materials endpoint.\n", + "# Matches the workflow logic: tags='elemental', metadata.element in compound elements.\n", + "elemental_materials_data = client.materials.list(\n", " {\"tags\": \"elemental\", \"metadata.element\": {\"$in\": elements}},\n", - " projection={\"limit\": 100},\n", ")\n", "\n", + "# Group by symbol: collect ALL matching materials per element (like the workflow)\n", + "element_materials_by_symbol = {\n", + " el: [\n", + " m for m in elemental_materials_data\n", + " if m.get(\"metadata\") and m[\"metadata\"].get(\"element\") == el\n", + " ]\n", + " for el in elements\n", + "}\n", + "\n", + "# Collect all exabyteIds per symbol (workflow uses this for $in query)\n", + "element_material_ids_by_symbol = {\n", + " el: [m[\"exabyteId\"] for m in element_materials_by_symbol[el]]\n", + " for el in elements\n", + "}\n", + "\n", + "# First match per symbol (for display / metadata)\n", "element_materials = {\n", - " item[\"metadata\"][\"element\"]: item\n", - " for item in elemental_materials_data\n", - " if item.get(\"metadata\") and item[\"metadata\"].get(\"element\") in elements\n", + " el: element_materials_by_symbol[el][0]\n", + " for el in elements if element_materials_by_symbol[el]\n", "}\n", "\n", - "missing_elemental_material_symbols = [element for element in elements if element not in element_materials]\n", + "missing_elemental_material_symbols = [\n", + " el for el in elements if not element_material_ids_by_symbol.get(el)\n", + "]\n", "if missing_elemental_material_symbols:\n", " raise RuntimeError(\n", " \"Missing Standata elemental reference material(s) for \"\n", - " f\"{ ', '.join(missing_elemental_material_symbols)}. \"\n", - " \"Import the corresponding elemental material(s) from Standata before running Formation Energy.\"\n", + " f\"{', '.join(missing_elemental_material_symbols)}. \"\n", + " \"Add elemental material(s) with tags elemental and metadata.element \"\n", + " \"before running Formation Energy.\"\n", " )\n", "\n", "print(f\"Resolved elemental reference materials for: {', '.join(elements)}\")\n", - "for symbol, mat in element_materials.items():\n", - " print(f\" {symbol}: {mat['name']} (_id={mat['_id']}, exabyteId={mat.get('exabyteId', 'N/A')})\")\n" + "for el in elements:\n", + " ids = element_material_ids_by_symbol[el]\n", + " print(f\" {el}: {len(ids)} material(s), exabyteIds={ids}\")\n" ] }, { "cell_type": "markdown", - "id": "20", + "id": "21", "metadata": {}, "source": [ "### 4.2. Check refined elemental total energies\n" @@ -309,25 +355,47 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "22", "metadata": {}, "outputs": [], "source": [ "import json\n", "\n", "\n", - "def get_elemental_total_energy_holders(api_client, symbols, element_materials_by_symbol):\n", - " \"\"\"Get the best total_energy property for each elemental reference material.\"\"\"\n", + "def get_elemental_total_energy_holders(\n", + " api_client, symbols, element_material_ids_by_symbol,\n", + " owner_slug, compound_group, source=\"public\"\n", + "):\n", + " \"\"\"\n", + " Get the best total_energy property for each elemental reference material.\n", + "\n", + " Matches the workflow logic:\n", + " - Loops per element\n", + " - Queries properties with exabyteId $in ALL matching material IDs\n", + " - Filters by group (computational method) for consistency\n", + " - Filters by owner.slug based on source parameter\n", + " - Sorts by precision.value descending, limit 1\n", + " \"\"\"\n", " holders_by_symbol = {}\n", " for symbol in symbols:\n", - " mat = element_materials_by_symbol[symbol]\n", - " exabyte_id = mat.get(\"exabyteId\", mat[\"_id\"])\n", - " # Query properties by exabyteId + slug, sorted by precision descending\n", + " material_ids = element_material_ids_by_symbol[symbol]\n", + "\n", + " # Base query: exabyteId + slug + group (method filter)\n", + " base_query = {\n", + " \"exabyteId\": {\"$in\": material_ids},\n", + " \"slug\": \"total_energy\",\n", + " \"group\": compound_group,\n", + " }\n", + "\n", + " # Add owner filter based on ELEMENTAL_TE_SOURCE\n", + " if source == \"my_account\":\n", + " base_query[\"owner.slug\"] = {\"$in\": [owner_slug, \"curators\"]}\n", + " elif source == \"curators\":\n", + " base_query[\"owner.slug\"] = \"curators\"\n", + " # else: public — no owner filter\n", + "\n", " props = api_client.properties.list(\n", - " query={\n", - " \"exabyteId\": exabyte_id,\n", - " \"slug\": \"total_energy\",\n", - " },\n", + " query=base_query,\n", " projection={\"sort\": {\"precision.value\": -1}, \"limit\": 1},\n", " )\n", " holders_by_symbol[symbol] = props[0] if props else None\n", @@ -337,7 +405,10 @@ "elemental_total_energy_holders = get_elemental_total_energy_holders(\n", " client,\n", " elements,\n", - " element_materials,\n", + " element_material_ids_by_symbol,\n", + " owner_slug=selected_account.entity_cache.get(\"slug\", selected_account.name),\n", + " compound_group=COMPOUND_GROUP,\n", + " source=ELEMENTAL_TE_SOURCE,\n", ")\n", "\n", "missing_elemental_total_energy_symbols = [\n", @@ -346,7 +417,7 @@ "if missing_elemental_total_energy_symbols:\n", " raise RuntimeError(\n", " \"Missing total_energy for elemental reference material(s) \"\n", - " f\"{ ', '.join(missing_elemental_total_energy_symbols)}. \"\n", + " f\"{', '.join(missing_elemental_total_energy_symbols)}. \"\n", " \"Run total_energy.ipynb for the corresponding Standata elemental material(s) first.\"\n", " )\n", "\n", @@ -355,13 +426,14 @@ " print(\n", " f\"♻️ Found total energy for {symbol}: \"\n", " f\"{holder.get('data', {}).get('value', 'N/A')} eV \"\n", - " f\"(precision: {holder.get('precision', {}).get('value', 'N/A')})\"\n", + " f\"(precision: {holder.get('precision', {}).get('value', 'N/A')}, \"\n", + " f\"group: {COMPOUND_GROUP}, source: {ELEMENTAL_TE_SOURCE})\"\n", " )\n" ] }, { "cell_type": "markdown", - "id": "22", + "id": "23", "metadata": {}, "source": [ "### 4.3. Save compound material for the workflow\n" @@ -370,7 +442,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "24", "metadata": {}, "outputs": [], "source": [ @@ -384,7 +456,7 @@ }, { "cell_type": "markdown", - "id": "24", + "id": "25", "metadata": {}, "source": [ "## 5. Configure the Formation Energy workflow\n", @@ -394,7 +466,7 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "26", "metadata": {}, "outputs": [], "source": [ @@ -408,7 +480,7 @@ }, { "cell_type": "markdown", - "id": "26", + "id": "27", "metadata": {}, "source": [ "### 5.2. Load workflow from Standata and preview it\n" @@ -417,7 +489,7 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "28", "metadata": {}, "outputs": [], "source": [ @@ -454,7 +526,7 @@ }, { "cell_type": "markdown", - "id": "28", + "id": "29", "metadata": {}, "source": [ "### 5.3. Save workflow to collection\n" @@ -463,18 +535,35 @@ { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "30", "metadata": {}, "outputs": [], "source": [ - "saved_formation_workflow_response = get_or_create_workflow(client, formation_workflow, ACCOUNT_ID)\n", + "from mat3ra.notebooks_utils.core.entity.workflow.api import get_or_create_workflow\n", + "\n", + "# get_or_create_workflow matches by hash. Since the API list endpoint\n", + "# does not filter server-side, we must filter client-side.\n", + "all_workflows = client.workflows.list({})\n", + "existing = [w for w in all_workflows if w.get('hash') == formation_workflow.hash]\n", + "if existing:\n", + " print(f\"♻️ Reusing existing workflow: {existing[0]['_id']} ({existing[0]['name']})\")\n", + " saved_formation_workflow_response = existing[0]\n", + "else:\n", + " # No exact hash match - find a Formation Energy workflow and use it\n", + " fe_workflows = [w for w in all_workflows if 'Formation Energy' in w.get('name', '')]\n", + " if fe_workflows:\n", + " print(f\"♻️ Using existing FE workflow: {fe_workflows[0]['_id']} ({fe_workflows[0]['name']})\")\n", + " saved_formation_workflow_response = fe_workflows[0]\n", + " else:\n", + " saved_formation_workflow_response = get_or_create_workflow(client, formation_workflow, ACCOUNT_ID)\n", + "\n", "saved_formation_workflow = Workflow.create(saved_formation_workflow_response)\n", "print(f\"Formation workflow ID: {saved_formation_workflow.id}\")\n" ] }, { "cell_type": "markdown", - "id": "30", + "id": "31", "metadata": {}, "source": [ "## 6. Create the compute configuration\n", @@ -484,7 +573,7 @@ { "cell_type": "code", "execution_count": null, - "id": "31", + "id": "32", "metadata": {}, "outputs": [], "source": [ @@ -494,7 +583,7 @@ }, { "cell_type": "markdown", - "id": "32", + "id": "33", "metadata": {}, "source": [ "### 6.2. Create compute configuration\n" @@ -503,7 +592,7 @@ { "cell_type": "code", "execution_count": null, - "id": "33", + "id": "34", "metadata": {}, "outputs": [], "source": [ @@ -520,7 +609,7 @@ }, { "cell_type": "markdown", - "id": "34", + "id": "35", "metadata": {}, "source": [ "## 7. Create the Formation Energy job\n", @@ -530,12 +619,11 @@ { "cell_type": "code", "execution_count": null, - "id": "35", + "id": "36", "metadata": {}, "outputs": [], "source": [ "from mat3ra.notebooks_utils.job import create_job\n", - "from mat3ra.notebooks_utils.api.job import wait_for_jobs_to_finish_async\n", "from mat3ra.utils.namespace import dict_to_namespace_recursive\n", "\n", "print(f\"Compound material: {saved_material.id}\")\n", @@ -562,7 +650,7 @@ }, { "cell_type": "markdown", - "id": "36", + "id": "37", "metadata": {}, "source": [ "### 7.2. Submit the Formation Energy job and monitor the status\n" @@ -571,7 +659,7 @@ { "cell_type": "code", "execution_count": null, - "id": "37", + "id": "38", "metadata": {}, "outputs": [], "source": [ @@ -582,16 +670,18 @@ { "cell_type": "code", "execution_count": null, - "id": "38", + "id": "39", "metadata": {}, "outputs": [], "source": [ - "await wait_for_jobs_to_finish_async(client.jobs, [formation_job_id], poll_interval=POLL_INTERVAL)\n" + "from mat3ra.notebooks_utils.api.job import wait_for_jobs_to_finish_async\n", + "\n", + "await wait_for_jobs_to_finish_async(client.jobs, formation_job_id, poll_interval=POLL_INTERVAL)" ] }, { "cell_type": "markdown", - "id": "39", + "id": "40", "metadata": {}, "source": [ "## 8. Retrieve results\n", @@ -601,7 +691,7 @@ { "cell_type": "code", "execution_count": null, - "id": "40", + "id": "41", "metadata": {}, "outputs": [], "source": [ @@ -609,10 +699,6 @@ "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", "\n", "formation_energy_data = client.properties.get_for_job(formation_job_id, property_name=\"formation_energy\")\n", - "formation_energy_contributions_data = client.properties.get_for_job(\n", - " formation_job_id,\n", - " property_name=\"formation_energy_contributions\",\n", - ")\n", "compound_total_energy_data = client.properties.get_for_job(\n", " formation_job_id,\n", " property_name=PropertyName.scalar.total_energy.value,\n", @@ -623,28 +709,17 @@ "else:\n", " print(\"No 'formation_energy' property was returned for the job.\")\n", "\n", - "if formation_energy_contributions_data:\n", - " visualize_properties(\n", - " formation_energy_contributions_data,\n", - " title=\"Formation Energy Contributions\",\n", - " )\n", - "else:\n", - " print(\"No 'formation_energy_contributions' property was returned for the job.\")\n", - "\n", - "if compound_total_energy_data:\n", - " visualize_properties(compound_total_energy_data, title=\"Compound Total Energy\")\n", - "else:\n", - " print(\"No compound total energy property was returned for the job.\")\n", - "\n", - "print(f\"Compound elements: {elements}\")\n", - "print(f\"Saved compound material used by the workflow: {saved_material_response['_id']}\")\n", - "for symbol in elements:\n", - " holder = elemental_total_energy_holders[symbol]\n", - " print(\n", - " f\"Refined elemental total energy for {symbol}: \"\n", - " f\"{element_materials[symbol]['name']} -> {holder['data']['value']}\"\n", - " )" + "formation_energy_data = client.properties.get_for_job(formation_job_id)\n", + "visualize_properties(formation_energy_data)\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "42", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From 085843a9132af7e2feddcd06a228252de5ffdd49 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 6 Jul 2026 23:50:33 -0700 Subject: [PATCH 4/7] update: cleanup --- .../workflows/formation_energy.ipynb | 137 ++++++------------ 1 file changed, 43 insertions(+), 94 deletions(-) diff --git a/other/materials_designer/workflows/formation_energy.ipynb b/other/materials_designer/workflows/formation_energy.ipynb index c416ae07..0c54717a 100644 --- a/other/materials_designer/workflows/formation_energy.ipynb +++ b/other/materials_designer/workflows/formation_energy.ipynb @@ -124,22 +124,6 @@ "id": "6", "metadata": {}, "outputs": [], - "source": [ - "# For local dev only. TODO: remove if commited\n", - "import os\n", - "\n", - "os.environ[\"API_PORT\"] = \"3000\"\n", - "os.environ[\"API_SECURE\"] = \"false\"\n", - "os.environ[\"API_HOST\"] = \"localhost\"\n", - "os.environ[\"OIDC_ACCESS_TOKEN\"] = \"TdpmY4sI6b6CXiOain_S4onc4h0x-uKJq17XttQcODS\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7", - "metadata": {}, - "outputs": [], "source": [ "from mat3ra.notebooks_utils.auth import authenticate\n", "\n", @@ -148,7 +132,7 @@ }, { "cell_type": "markdown", - "id": "8", + "id": "7", "metadata": {}, "source": [ "### 2.2. Initialize API client\n" @@ -157,7 +141,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "8", "metadata": {}, "outputs": [], "source": [ @@ -169,7 +153,7 @@ }, { "cell_type": "markdown", - "id": "10", + "id": "9", "metadata": {}, "source": [ "### 2.3. Select account to work under\n" @@ -178,7 +162,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -188,7 +172,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -203,7 +187,7 @@ }, { "cell_type": "markdown", - "id": "13", + "id": "12", "metadata": {}, "source": [ "### 2.4. Select project\n" @@ -212,7 +196,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -223,7 +207,7 @@ }, { "cell_type": "markdown", - "id": "15", + "id": "14", "metadata": {}, "source": [ "## 3. Load compound material\n", @@ -233,7 +217,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -264,7 +248,7 @@ }, { "cell_type": "markdown", - "id": "17", + "id": "16", "metadata": {}, "source": [ "### 3.2. Resolve unique elements in the compound\n" @@ -273,7 +257,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -286,7 +270,7 @@ }, { "cell_type": "markdown", - "id": "19", + "id": "18", "metadata": {}, "source": [ "## 4. Resolve elemental reference materials and check total energies\n", @@ -296,7 +280,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -346,7 +330,7 @@ }, { "cell_type": "markdown", - "id": "21", + "id": "20", "metadata": {}, "source": [ "### 4.2. Check refined elemental total energies\n" @@ -355,7 +339,7 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "21", "metadata": {}, "outputs": [], "source": [ @@ -363,8 +347,8 @@ "\n", "\n", "def get_elemental_total_energy_holders(\n", - " api_client, symbols, element_material_ids_by_symbol,\n", - " owner_slug, compound_group, source=\"public\"\n", + " api_client, symbols, element_material_ids_by_symbol,\n", + " owner_slug, compound_group, source=\"public\"\n", "):\n", " \"\"\"\n", " Get the best total_energy property for each elemental reference material.\n", @@ -433,7 +417,7 @@ }, { "cell_type": "markdown", - "id": "23", + "id": "22", "metadata": {}, "source": [ "### 4.3. Save compound material for the workflow\n" @@ -442,7 +426,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "23", "metadata": {}, "outputs": [], "source": [ @@ -456,7 +440,7 @@ }, { "cell_type": "markdown", - "id": "25", + "id": "24", "metadata": {}, "source": [ "## 5. Configure the Formation Energy workflow\n", @@ -466,7 +450,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -480,7 +464,7 @@ }, { "cell_type": "markdown", - "id": "27", + "id": "26", "metadata": {}, "source": [ "### 5.2. Load workflow from Standata and preview it\n" @@ -489,7 +473,7 @@ { "cell_type": "code", "execution_count": null, - "id": "28", + "id": "27", "metadata": {}, "outputs": [], "source": [ @@ -526,7 +510,7 @@ }, { "cell_type": "markdown", - "id": "29", + "id": "28", "metadata": {}, "source": [ "### 5.3. Save workflow to collection\n" @@ -535,35 +519,20 @@ { "cell_type": "code", "execution_count": null, - "id": "30", + "id": "29", "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.workflow.api import get_or_create_workflow\n", - "\n", - "# get_or_create_workflow matches by hash. Since the API list endpoint\n", - "# does not filter server-side, we must filter client-side.\n", - "all_workflows = client.workflows.list({})\n", - "existing = [w for w in all_workflows if w.get('hash') == formation_workflow.hash]\n", - "if existing:\n", - " print(f\"♻️ Reusing existing workflow: {existing[0]['_id']} ({existing[0]['name']})\")\n", - " saved_formation_workflow_response = existing[0]\n", - "else:\n", - " # No exact hash match - find a Formation Energy workflow and use it\n", - " fe_workflows = [w for w in all_workflows if 'Formation Energy' in w.get('name', '')]\n", - " if fe_workflows:\n", - " print(f\"♻️ Using existing FE workflow: {fe_workflows[0]['_id']} ({fe_workflows[0]['name']})\")\n", - " saved_formation_workflow_response = fe_workflows[0]\n", - " else:\n", - " saved_formation_workflow_response = get_or_create_workflow(client, formation_workflow, ACCOUNT_ID)\n", - "\n", + "saved_formation_workflow_response = get_or_create_workflow(\n", + " client, formation_workflow, ACCOUNT_ID\n", + ")\n", "saved_formation_workflow = Workflow.create(saved_formation_workflow_response)\n", "print(f\"Formation workflow ID: {saved_formation_workflow.id}\")\n" ] }, { "cell_type": "markdown", - "id": "31", + "id": "30", "metadata": {}, "source": [ "## 6. Create the compute configuration\n", @@ -573,7 +542,7 @@ { "cell_type": "code", "execution_count": null, - "id": "32", + "id": "31", "metadata": {}, "outputs": [], "source": [ @@ -583,7 +552,7 @@ }, { "cell_type": "markdown", - "id": "33", + "id": "32", "metadata": {}, "source": [ "### 6.2. Create compute configuration\n" @@ -592,7 +561,7 @@ { "cell_type": "code", "execution_count": null, - "id": "34", + "id": "33", "metadata": {}, "outputs": [], "source": [ @@ -609,7 +578,7 @@ }, { "cell_type": "markdown", - "id": "35", + "id": "34", "metadata": {}, "source": [ "## 7. Create the Formation Energy job\n", @@ -619,17 +588,11 @@ { "cell_type": "code", "execution_count": null, - "id": "36", + "id": "35", "metadata": {}, "outputs": [], "source": [ "from mat3ra.notebooks_utils.job import create_job\n", - "from mat3ra.utils.namespace import dict_to_namespace_recursive\n", - "\n", - "print(f\"Compound material: {saved_material.id}\")\n", - "print(f\"Elemental references: {', '.join(elements)}\")\n", - "print(f\"Formation workflow: {saved_formation_workflow.id}\")\n", - "print(f\"Project: {project_id}\")\n", "\n", "formation_job_name = f\"{MY_WORKFLOW_NAME} {saved_material.formula} {timestamp}\"\n", "formation_job_response = create_job(\n", @@ -642,15 +605,13 @@ " compute=compute.to_dict(),\n", ")\n", "\n", - "formation_job = dict_to_namespace_recursive(formation_job_response)\n", - "formation_job_id = formation_job._id\n", - "print(f\"✅ Formation Energy job created successfully: {formation_job_id}\")\n", - "display_JSON(formation_job_response)" + "formation_job_id = formation_job_response[\"_id\"]\n", + "print(f\"✅ Formation Energy job created: {formation_job_id}\")\n" ] }, { "cell_type": "markdown", - "id": "37", + "id": "36", "metadata": {}, "source": [ "### 7.2. Submit the Formation Energy job and monitor the status\n" @@ -659,7 +620,7 @@ { "cell_type": "code", "execution_count": null, - "id": "38", + "id": "37", "metadata": {}, "outputs": [], "source": [ @@ -670,18 +631,18 @@ { "cell_type": "code", "execution_count": null, - "id": "39", + "id": "38", "metadata": {}, "outputs": [], "source": [ "from mat3ra.notebooks_utils.api.job import wait_for_jobs_to_finish_async\n", "\n", - "await wait_for_jobs_to_finish_async(client.jobs, formation_job_id, poll_interval=POLL_INTERVAL)" + "await wait_for_jobs_to_finish_async(client.jobs, formation_job_id, poll_interval=POLL_INTERVAL)\n" ] }, { "cell_type": "markdown", - "id": "40", + "id": "39", "metadata": {}, "source": [ "## 8. Retrieve results\n", @@ -691,24 +652,12 @@ { "cell_type": "code", "execution_count": null, - "id": "41", + "id": "40", "metadata": {}, "outputs": [], "source": [ - "from mat3ra.prode import PropertyName\n", "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", "\n", - "formation_energy_data = client.properties.get_for_job(formation_job_id, property_name=\"formation_energy\")\n", - "compound_total_energy_data = client.properties.get_for_job(\n", - " formation_job_id,\n", - " property_name=PropertyName.scalar.total_energy.value,\n", - ")\n", - "\n", - "if formation_energy_data:\n", - " visualize_properties(formation_energy_data, title=\"Formation Energy\")\n", - "else:\n", - " print(\"No 'formation_energy' property was returned for the job.\")\n", - "\n", "formation_energy_data = client.properties.get_for_job(formation_job_id)\n", "visualize_properties(formation_energy_data)\n" ] @@ -716,7 +665,7 @@ { "cell_type": "code", "execution_count": null, - "id": "42", + "id": "41", "metadata": {}, "outputs": [], "source": [] From 4bdeea2a97ec79f670a172249a18b167621b18f1 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Tue, 7 Jul 2026 09:02:26 -0700 Subject: [PATCH 5/7] update: cleanup --- .../workflows/formation_energy.ipynb | 157 +++++------------- 1 file changed, 38 insertions(+), 119 deletions(-) diff --git a/other/materials_designer/workflows/formation_energy.ipynb b/other/materials_designer/workflows/formation_energy.ipynb index 0c54717a..6c1830b2 100644 --- a/other/materials_designer/workflows/formation_energy.ipynb +++ b/other/materials_designer/workflows/formation_energy.ipynb @@ -221,29 +221,15 @@ "metadata": {}, "outputs": [], "source": [ - "import re\n", "from mat3ra.made.material import Material\n", "from mat3ra.standata.materials import Materials\n", - "from mat3ra.notebooks_utils.material import load_material_from_folder\n", "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", + "from mat3ra.notebooks_utils.material import load_material_from_folder\n", "\n", - "material = load_material_from_folder(FOLDER, MATERIAL_NAME)\n", - "\n", - "if material is None:\n", - " try:\n", - " material = Material.create(Materials.get_by_name_first_match(MATERIAL_NAME))\n", - " print(f\"✅ Loaded material from Standata: {material.name}\")\n", - " except Exception:\n", - " material_matches = client.materials.list({\n", - " \"name\": {\"$regex\": re.escape(MATERIAL_NAME), \"$options\": \"i\"},\n", - " })\n", + "material = load_material_from_folder(FOLDER, MATERIAL_NAME) or Material.create(\n", + " Materials.get_by_name_first_match(MATERIAL_NAME))\n", "\n", - " material = Material.create(material_matches[0])\n", - " print(f\"♻️ Loaded material from platform: {material_matches[0]['_id']}\")\n", - "else:\n", - " print(f\"✅ Loaded material from folder: {material.name}\")\n", - "\n", - "visualize(material)" + "visualize(material)\n" ] }, { @@ -261,11 +247,8 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.ui import display_JSON\n", - "\n", "elements = sorted(set(material.basis.elements.values))\n", - "print(f\"Compound elements: {elements}\")\n", - "display_JSON({\"elements\": elements}, level=2)" + "print(f\"Compound elements: {elements}\")\n" ] }, { @@ -284,48 +267,28 @@ "metadata": {}, "outputs": [], "source": [ - "# Query elemental reference materials from the materials endpoint.\n", - "# Matches the workflow logic: tags='elemental', metadata.element in compound elements.\n", "elemental_materials_data = client.materials.list(\n", " {\"tags\": \"elemental\", \"metadata.element\": {\"$in\": elements}},\n", ")\n", "\n", - "# Group by symbol: collect ALL matching materials per element (like the workflow)\n", "element_materials_by_symbol = {\n", - " el: [\n", - " m for m in elemental_materials_data\n", - " if m.get(\"metadata\") and m[\"metadata\"].get(\"element\") == el\n", - " ]\n", + " el: [m for m in elemental_materials_data if m.get(\"metadata\", {}).get(\"element\") == el]\n", " for el in elements\n", "}\n", - "\n", - "# Collect all exabyteIds per symbol (workflow uses this for $in query)\n", "element_material_ids_by_symbol = {\n", " el: [m[\"exabyteId\"] for m in element_materials_by_symbol[el]]\n", " for el in elements\n", "}\n", - "\n", - "# First match per symbol (for display / metadata)\n", "element_materials = {\n", " el: element_materials_by_symbol[el][0]\n", " for el in elements if element_materials_by_symbol[el]\n", "}\n", "\n", - "missing_elemental_material_symbols = [\n", - " el for el in elements if not element_material_ids_by_symbol.get(el)\n", - "]\n", - "if missing_elemental_material_symbols:\n", - " raise RuntimeError(\n", - " \"Missing Standata elemental reference material(s) for \"\n", - " f\"{', '.join(missing_elemental_material_symbols)}. \"\n", - " \"Add elemental material(s) with tags elemental and metadata.element \"\n", - " \"before running Formation Energy.\"\n", - " )\n", + "missing = [el for el in elements if not element_material_ids_by_symbol.get(el)]\n", + "if missing:\n", + " raise RuntimeError(f\"Missing elemental reference material(s) for {missing}\")\n", "\n", - "print(f\"Resolved elemental reference materials for: {', '.join(elements)}\")\n", - "for el in elements:\n", - " ids = element_material_ids_by_symbol[el]\n", - " print(f\" {el}: {len(ids)} material(s), exabyteIds={ids}\")\n" + "print(f\"Resolved elemental materials for: {', '.join(elements)}\")\n" ] }, { @@ -343,75 +306,46 @@ "metadata": {}, "outputs": [], "source": [ - "import json\n", - "\n", - "\n", "def get_elemental_total_energy_holders(\n", - " api_client, symbols, element_material_ids_by_symbol,\n", - " owner_slug, compound_group, source=\"public\"\n", + " api_client, symbols, element_material_ids_by_symbol,\n", + " owner_slug, compound_group, source=\"public\"\n", "):\n", - " \"\"\"\n", - " Get the best total_energy property for each elemental reference material.\n", - "\n", - " Matches the workflow logic:\n", - " - Loops per element\n", - " - Queries properties with exabyteId $in ALL matching material IDs\n", - " - Filters by group (computational method) for consistency\n", - " - Filters by owner.slug based on source parameter\n", - " - Sorts by precision.value descending, limit 1\n", - " \"\"\"\n", - " holders_by_symbol = {}\n", + " holders = {}\n", " for symbol in symbols:\n", - " material_ids = element_material_ids_by_symbol[symbol]\n", - "\n", - " # Base query: exabyteId + slug + group (method filter)\n", - " base_query = {\n", - " \"exabyteId\": {\"$in\": material_ids},\n", + " query = {\n", + " \"exabyteId\": {\"$in\": element_material_ids_by_symbol[symbol]},\n", " \"slug\": \"total_energy\",\n", " \"group\": compound_group,\n", " }\n", - "\n", - " # Add owner filter based on ELEMENTAL_TE_SOURCE\n", " if source == \"my_account\":\n", - " base_query[\"owner.slug\"] = {\"$in\": [owner_slug, \"curators\"]}\n", + " query[\"owner.slug\"] = {\"$in\": [owner_slug, \"curators\"]}\n", " elif source == \"curators\":\n", - " base_query[\"owner.slug\"] = \"curators\"\n", - " # else: public — no owner filter\n", + " query[\"owner.slug\"] = \"curators\"\n", "\n", " props = api_client.properties.list(\n", - " query=base_query,\n", + " query=query,\n", " projection={\"sort\": {\"precision.value\": -1}, \"limit\": 1},\n", " )\n", - " holders_by_symbol[symbol] = props[0] if props else None\n", - " return holders_by_symbol\n", + " holders[symbol] = props[0] if props else None\n", + " return holders\n", "\n", "\n", "elemental_total_energy_holders = get_elemental_total_energy_holders(\n", - " client,\n", - " elements,\n", - " element_material_ids_by_symbol,\n", + " client, elements, element_material_ids_by_symbol,\n", " owner_slug=selected_account.entity_cache.get(\"slug\", selected_account.name),\n", " compound_group=COMPOUND_GROUP,\n", " source=ELEMENTAL_TE_SOURCE,\n", ")\n", "\n", - "missing_elemental_total_energy_symbols = [\n", - " symbol for symbol in elements if elemental_total_energy_holders[symbol] is None\n", - "]\n", - "if missing_elemental_total_energy_symbols:\n", - " raise RuntimeError(\n", - " \"Missing total_energy for elemental reference material(s) \"\n", - " f\"{', '.join(missing_elemental_total_energy_symbols)}. \"\n", - " \"Run total_energy.ipynb for the corresponding Standata elemental material(s) first.\"\n", - " )\n", + "missing = [s for s in elements if elemental_total_energy_holders[s] is None]\n", + "if missing:\n", + " raise RuntimeError(f\"Missing total_energy for {missing}. Run total_energy.ipynb first.\")\n", "\n", "for symbol in elements:\n", - " holder = elemental_total_energy_holders[symbol]\n", + " h = elemental_total_energy_holders[symbol]\n", " print(\n", - " f\"♻️ Found total energy for {symbol}: \"\n", - " f\"{holder.get('data', {}).get('value', 'N/A')} eV \"\n", - " f\"(precision: {holder.get('precision', {}).get('value', 'N/A')}, \"\n", - " f\"group: {COMPOUND_GROUP}, source: {ELEMENTAL_TE_SOURCE})\"\n", + " f\"♻️ {symbol}: {h['data']['value']} eV \"\n", + " f\"(precision: {h['precision']['value']}, group: {COMPOUND_GROUP})\"\n", " )\n" ] }, @@ -480,32 +414,23 @@ "from mat3ra.standata.workflows import WorkflowStandata\n", "from mat3ra.wode.context.providers import PointsGridDataProvider\n", "from mat3ra.wode.workflows import Workflow\n", - "from mat3ra.notebooks_utils.core.entity.workflow.api import get_or_create_workflow\n", "from mat3ra.notebooks_utils.ipython.entity.workflow.visualize import visualize_workflow\n", "\n", - "\n", - "def apply_workflow_kgrids(workflow: Workflow, scf_kgrid=None) -> Workflow:\n", - " if scf_kgrid is not None:\n", - " new_context_scf = PointsGridDataProvider(dimensions=scf_kgrid, isEdited=True).get_context_item_data()\n", - " for subworkflow in workflow.subworkflows:\n", - " unit_names = [unit.name for unit in subworkflow.units]\n", - " if \"pw_scf\" not in unit_names:\n", - " continue\n", - " unit_to_modify_scf = subworkflow.get_unit_by_name(name=\"pw_scf\")\n", - " unit_to_modify_scf.add_context(new_context_scf)\n", - " subworkflow.set_unit(unit_to_modify_scf)\n", - " break\n", - " return workflow\n", - "\n", - "\n", "formation_workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(\n", " WORKFLOW_SEARCH_TERM\n", ")\n", "formation_workflow = Workflow.create(formation_workflow_config)\n", "formation_workflow.name = MY_WORKFLOW_NAME\n", - "formation_workflow = apply_workflow_kgrids(formation_workflow, scf_kgrid=SCF_KGRID)\n", "\n", - "visualize_workflow(formation_workflow)" + "if SCF_KGRID is not None:\n", + " new_context = PointsGridDataProvider(dimensions=SCF_KGRID, isEdited=True).get_context_item_data()\n", + " for subworkflow in formation_workflow.subworkflows:\n", + " unit = subworkflow.get_unit_by_name(name=\"pw_scf\")\n", + " if unit:\n", + " unit.add_context(new_context)\n", + " subworkflow.set_unit(unit)\n", + "\n", + "visualize_workflow(formation_workflow)\n" ] }, { @@ -523,6 +448,8 @@ "metadata": {}, "outputs": [], "source": [ + "from mat3ra.notebooks_utils.core.entity.workflow.api import get_or_create_workflow\n", + "\n", "saved_formation_workflow_response = get_or_create_workflow(\n", " client, formation_workflow, ACCOUNT_ID\n", ")\n", @@ -661,14 +588,6 @@ "formation_energy_data = client.properties.get_for_job(formation_job_id)\n", "visualize_properties(formation_energy_data)\n" ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "41", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { From d2b681ebb2009a2063d4a2c631906c79955bff9e Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Tue, 7 Jul 2026 11:47:37 -0700 Subject: [PATCH 6/7] update: make work --- .../workflows/formation_energy.ipynb | 135 +++++++++++------- 1 file changed, 83 insertions(+), 52 deletions(-) diff --git a/other/materials_designer/workflows/formation_energy.ipynb b/other/materials_designer/workflows/formation_energy.ipynb index 6c1830b2..c5eade1f 100644 --- a/other/materials_designer/workflows/formation_energy.ipynb +++ b/other/materials_designer/workflows/formation_energy.ipynb @@ -83,7 +83,7 @@ "# 4. Workflow parameters\n", "WORKFLOW_SEARCH_TERM = \"formation_energy.json\" # Search term for Workflows Standata\n", "APPLICATION_NAME = \"espresso\" # Application for the QE formation-energy subworkflow\n", - "SCF_KGRID = None # e.g., [10, 10, 10]\n", + "SCF_KGRID = [6, 6, 6] # K-point grid; formation energy requires kgrid context for precision\n", "MY_WORKFLOW_NAME = \"Formation Energy\"\n", "\n", "# 5. Elemental total energy source\n", @@ -93,11 +93,6 @@ "# 'my_account' — curators' or your own properties\n", "ELEMENTAL_TE_SOURCE = \"public\"\n", "\n", - "# 5b. Property group filter for method consistency\n", - "# Ensures elemental total energies come from the same computational method.\n", - "# Format: {application_short_name}:{model_type}:{model_subtype}\n", - "COMPOUND_GROUP = \"qe:dft:gga:pbe\" # Quantum ESPRESSO + DFT/GGA/PBE\n", - "\n", "# 6. Compute parameters\n", "CLUSTER_NAME = None # specify full or partial name i.e. \"cluster-001\" to select\n", "QUEUE_NAME = QueueName.D\n", @@ -124,6 +119,21 @@ "id": "6", "metadata": {}, "outputs": [], + "source": [ + "# For local dev only. TODO: remove if committed\n", + "import os\n", + "os.environ[\"API_PORT\"] = \"3000\"\n", + "os.environ[\"API_SECURE\"] = \"false\"\n", + "os.environ[\"API_HOST\"] = \"localhost\"\n", + "# os.environ[\"OIDC_ACCESS_TOKEN\"] = \"Qswbf3EIxF2teMKByk6TYpJIzc66NJu1r5hZnijpW7Y\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], "source": [ "from mat3ra.notebooks_utils.auth import authenticate\n", "\n", @@ -132,7 +142,7 @@ }, { "cell_type": "markdown", - "id": "7", + "id": "8", "metadata": {}, "source": [ "### 2.2. Initialize API client\n" @@ -141,7 +151,7 @@ { "cell_type": "code", "execution_count": null, - "id": "8", + "id": "9", "metadata": {}, "outputs": [], "source": [ @@ -153,7 +163,7 @@ }, { "cell_type": "markdown", - "id": "9", + "id": "10", "metadata": {}, "source": [ "### 2.3. Select account to work under\n" @@ -162,7 +172,7 @@ { "cell_type": "code", "execution_count": null, - "id": "10", + "id": "11", "metadata": {}, "outputs": [], "source": [ @@ -172,7 +182,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -187,7 +197,7 @@ }, { "cell_type": "markdown", - "id": "12", + "id": "13", "metadata": {}, "source": [ "### 2.4. Select project\n" @@ -196,7 +206,7 @@ { "cell_type": "code", "execution_count": null, - "id": "13", + "id": "14", "metadata": {}, "outputs": [], "source": [ @@ -207,7 +217,7 @@ }, { "cell_type": "markdown", - "id": "14", + "id": "15", "metadata": {}, "source": [ "## 3. Load compound material\n", @@ -217,7 +227,7 @@ { "cell_type": "code", "execution_count": null, - "id": "15", + "id": "16", "metadata": {}, "outputs": [], "source": [ @@ -234,7 +244,7 @@ }, { "cell_type": "markdown", - "id": "16", + "id": "17", "metadata": {}, "source": [ "### 3.2. Resolve unique elements in the compound\n" @@ -243,7 +253,7 @@ { "cell_type": "code", "execution_count": null, - "id": "17", + "id": "18", "metadata": {}, "outputs": [], "source": [ @@ -253,7 +263,7 @@ }, { "cell_type": "markdown", - "id": "18", + "id": "19", "metadata": {}, "source": [ "## 4. Resolve elemental reference materials and check total energies\n", @@ -263,7 +273,7 @@ { "cell_type": "code", "execution_count": null, - "id": "19", + "id": "20", "metadata": {}, "outputs": [], "source": [ @@ -293,7 +303,7 @@ }, { "cell_type": "markdown", - "id": "20", + "id": "21", "metadata": {}, "source": [ "### 4.2. Check refined elemental total energies\n" @@ -302,20 +312,19 @@ { "cell_type": "code", "execution_count": null, - "id": "21", + "id": "22", "metadata": {}, "outputs": [], "source": [ "def get_elemental_total_energy_holders(\n", " api_client, symbols, element_material_ids_by_symbol,\n", - " owner_slug, compound_group, source=\"public\"\n", + " owner_slug, source=\"public\"\n", "):\n", " holders = {}\n", " for symbol in symbols:\n", " query = {\n", " \"exabyteId\": {\"$in\": element_material_ids_by_symbol[symbol]},\n", " \"slug\": \"total_energy\",\n", - " \"group\": compound_group,\n", " }\n", " if source == \"my_account\":\n", " query[\"owner.slug\"] = {\"$in\": [owner_slug, \"curators\"]}\n", @@ -333,7 +342,6 @@ "elemental_total_energy_holders = get_elemental_total_energy_holders(\n", " client, elements, element_material_ids_by_symbol,\n", " owner_slug=selected_account.entity_cache.get(\"slug\", selected_account.name),\n", - " compound_group=COMPOUND_GROUP,\n", " source=ELEMENTAL_TE_SOURCE,\n", ")\n", "\n", @@ -343,15 +351,12 @@ "\n", "for symbol in elements:\n", " h = elemental_total_energy_holders[symbol]\n", - " print(\n", - " f\"♻️ {symbol}: {h['data']['value']} eV \"\n", - " f\"(precision: {h['precision']['value']}, group: {COMPOUND_GROUP})\"\n", - " )\n" + " print(f\"♻️ {symbol}: {h['data']['value']} eV (precision: {h['precision']['value']})\")\n" ] }, { "cell_type": "markdown", - "id": "22", + "id": "23", "metadata": {}, "source": [ "### 4.3. Save compound material for the workflow\n" @@ -360,7 +365,7 @@ { "cell_type": "code", "execution_count": null, - "id": "23", + "id": "24", "metadata": {}, "outputs": [], "source": [ @@ -374,7 +379,7 @@ }, { "cell_type": "markdown", - "id": "24", + "id": "25", "metadata": {}, "source": [ "## 5. Configure the Formation Energy workflow\n", @@ -384,7 +389,7 @@ { "cell_type": "code", "execution_count": null, - "id": "25", + "id": "26", "metadata": {}, "outputs": [], "source": [ @@ -398,7 +403,7 @@ }, { "cell_type": "markdown", - "id": "26", + "id": "27", "metadata": {}, "source": [ "### 5.2. Load workflow from Standata and preview it\n" @@ -407,12 +412,11 @@ { "cell_type": "code", "execution_count": null, - "id": "27", + "id": "28", "metadata": {}, "outputs": [], "source": [ "from mat3ra.standata.workflows import WorkflowStandata\n", - "from mat3ra.wode.context.providers import PointsGridDataProvider\n", "from mat3ra.wode.workflows import Workflow\n", "from mat3ra.notebooks_utils.ipython.entity.workflow.visualize import visualize_workflow\n", "\n", @@ -422,20 +426,39 @@ "formation_workflow = Workflow.create(formation_workflow_config)\n", "formation_workflow.name = MY_WORKFLOW_NAME\n", "\n", + "visualize_workflow(formation_workflow)\n" + ] + }, + { + "cell_type": "markdown", + "id": "29", + "metadata": {}, + "source": [ + "### 5.3. Modify important settings\n", + "Set k-grid.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "30", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.wode.context.providers import PointsGridDataProvider\n", + "\n", "if SCF_KGRID is not None:\n", " new_context = PointsGridDataProvider(dimensions=SCF_KGRID, isEdited=True).get_context_item_data()\n", " for subworkflow in formation_workflow.subworkflows:\n", " unit = subworkflow.get_unit_by_name(name=\"pw_scf\")\n", " if unit:\n", " unit.add_context(new_context)\n", - " subworkflow.set_unit(unit)\n", - "\n", - "visualize_workflow(formation_workflow)\n" + " subworkflow.set_unit(unit)\n" ] }, { "cell_type": "markdown", - "id": "28", + "id": "31", "metadata": {}, "source": [ "### 5.3. Save workflow to collection\n" @@ -444,7 +467,7 @@ { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "32", "metadata": {}, "outputs": [], "source": [ @@ -459,7 +482,7 @@ }, { "cell_type": "markdown", - "id": "30", + "id": "33", "metadata": {}, "source": [ "## 6. Create the compute configuration\n", @@ -469,7 +492,7 @@ { "cell_type": "code", "execution_count": null, - "id": "31", + "id": "34", "metadata": {}, "outputs": [], "source": [ @@ -479,7 +502,7 @@ }, { "cell_type": "markdown", - "id": "32", + "id": "35", "metadata": {}, "source": [ "### 6.2. Create compute configuration\n" @@ -488,7 +511,7 @@ { "cell_type": "code", "execution_count": null, - "id": "33", + "id": "36", "metadata": {}, "outputs": [], "source": [ @@ -505,7 +528,7 @@ }, { "cell_type": "markdown", - "id": "34", + "id": "37", "metadata": {}, "source": [ "## 7. Create the Formation Energy job\n", @@ -515,7 +538,7 @@ { "cell_type": "code", "execution_count": null, - "id": "35", + "id": "38", "metadata": {}, "outputs": [], "source": [ @@ -538,7 +561,7 @@ }, { "cell_type": "markdown", - "id": "36", + "id": "39", "metadata": {}, "source": [ "### 7.2. Submit the Formation Energy job and monitor the status\n" @@ -547,7 +570,7 @@ { "cell_type": "code", "execution_count": null, - "id": "37", + "id": "40", "metadata": {}, "outputs": [], "source": [ @@ -558,18 +581,18 @@ { "cell_type": "code", "execution_count": null, - "id": "38", + "id": "41", "metadata": {}, "outputs": [], "source": [ "from mat3ra.notebooks_utils.api.job import wait_for_jobs_to_finish_async\n", "\n", - "await wait_for_jobs_to_finish_async(client.jobs, formation_job_id, poll_interval=POLL_INTERVAL)\n" + "await wait_for_jobs_to_finish_async(client.jobs, [formation_job_id], poll_interval=POLL_INTERVAL)\n" ] }, { "cell_type": "markdown", - "id": "39", + "id": "42", "metadata": {}, "source": [ "## 8. Retrieve results\n", @@ -579,7 +602,7 @@ { "cell_type": "code", "execution_count": null, - "id": "40", + "id": "43", "metadata": {}, "outputs": [], "source": [ @@ -588,6 +611,14 @@ "formation_energy_data = client.properties.get_for_job(formation_job_id)\n", "visualize_properties(formation_energy_data)\n" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "44", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { From 5fff6059dc2a02b0b6e8aa9808ab48a8b0522516 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Tue, 7 Jul 2026 13:22:07 -0700 Subject: [PATCH 7/7] update: cleanup --- .../workflows/formation_energy.ipynb | 182 +++++++++--------- 1 file changed, 92 insertions(+), 90 deletions(-) diff --git a/other/materials_designer/workflows/formation_energy.ipynb b/other/materials_designer/workflows/formation_energy.ipynb index c5eade1f..41667b9f 100644 --- a/other/materials_designer/workflows/formation_energy.ipynb +++ b/other/materials_designer/workflows/formation_energy.ipynb @@ -59,7 +59,7 @@ "id": "3", "metadata": {}, "source": [ - "### 1.2. Set parameters and configurations for the workflow and job\n" + "### 1.2. Set parameters\n" ] }, { @@ -73,7 +73,6 @@ "from mat3ra.ide.compute import QueueName\n", "\n", "# 2. Auth and organization parameters\n", - "# Set organization name to use it as the owner, otherwise your personal account is used\n", "ORGANIZATION_NAME = None\n", "\n", "# 3. Material parameters\n", @@ -81,24 +80,16 @@ "MATERIAL_NAME = \"Silicon\" # Name of the compound to load from uploads folder, Standata, or platform\n", "\n", "# 4. Workflow parameters\n", - "WORKFLOW_SEARCH_TERM = \"formation_energy.json\" # Search term for Workflows Standata\n", - "APPLICATION_NAME = \"espresso\" # Application for the QE formation-energy subworkflow\n", - "SCF_KGRID = [6, 6, 6] # K-point grid; formation energy requires kgrid context for precision\n", + "WORKFLOW_SEARCH_TERM = \"formation_energy.json\"\n", + "APPLICATION_NAME = \"espresso\"\n", "MY_WORKFLOW_NAME = \"Formation Energy\"\n", "\n", - "# 5. Elemental total energy source\n", - "# Controls whose total_energy properties are used for elemental references:\n", - "# 'public' — any owner (highest precision wins)\n", - "# 'curators' — only curators' properties\n", - "# 'my_account' — curators' or your own properties\n", - "ELEMENTAL_TE_SOURCE = \"public\"\n", - "\n", - "# 6. Compute parameters\n", + "# 5. Compute parameters\n", "CLUSTER_NAME = None # specify full or partial name i.e. \"cluster-001\" to select\n", "QUEUE_NAME = QueueName.D\n", "PPN = 1\n", "\n", - "# 7. Job parameters\n", + "# 6. Job parameters\n", "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", "POLL_INTERVAL = 30 # seconds\n" ] @@ -108,9 +99,7 @@ "id": "5", "metadata": {}, "source": [ - "## 2. Authenticate and initialize API client\n", - "### 2.1. Authenticate\n", - "Authenticate in the browser and have credentials stored in environment variable `OIDC_ACCESS_TOKEN`.\n" + "### 1.3. Set specific formation energy parameters\n" ] }, { @@ -120,18 +109,31 @@ "metadata": {}, "outputs": [], "source": [ - "# For local dev only. TODO: remove if committed\n", - "import os\n", - "os.environ[\"API_PORT\"] = \"3000\"\n", - "os.environ[\"API_SECURE\"] = \"false\"\n", - "os.environ[\"API_HOST\"] = \"localhost\"\n", - "# os.environ[\"OIDC_ACCESS_TOKEN\"] = \"Qswbf3EIxF2teMKByk6TYpJIzc66NJu1r5hZnijpW7Y\"\n" + "# K-grid for SCF (if not set, the workflow default is used)\n", + "SCF_KGRID = None # e.g. [6, 6, 6]\n", + "\n", + "# Elemental total energy source\n", + "# Controls whose total_energy properties are used for elemental references:\n", + "# 'public' — any owner (highest precision wins)\n", + "# 'curators' — only curators' properties\n", + "# 'my_account' — curators' or your own properties\n", + "ELEMENTAL_TE_SOURCE = \"public\"\n" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "## 2. Authenticate and initialize API client\n", + "### 2.1. Authenticate\n", + "Authenticate in the browser and have credentials stored in environment variable `OIDC_ACCESS_TOKEN`.\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "7", + "id": "8", "metadata": {}, "outputs": [], "source": [ @@ -142,7 +144,7 @@ }, { "cell_type": "markdown", - "id": "8", + "id": "9", "metadata": {}, "source": [ "### 2.2. Initialize API client\n" @@ -151,7 +153,7 @@ { "cell_type": "code", "execution_count": null, - "id": "9", + "id": "10", "metadata": {}, "outputs": [], "source": [ @@ -163,7 +165,7 @@ }, { "cell_type": "markdown", - "id": "10", + "id": "11", "metadata": {}, "source": [ "### 2.3. Select account to work under\n" @@ -172,7 +174,7 @@ { "cell_type": "code", "execution_count": null, - "id": "11", + "id": "12", "metadata": {}, "outputs": [], "source": [ @@ -182,7 +184,7 @@ { "cell_type": "code", "execution_count": null, - "id": "12", + "id": "13", "metadata": {}, "outputs": [], "source": [ @@ -197,7 +199,7 @@ }, { "cell_type": "markdown", - "id": "13", + "id": "14", "metadata": {}, "source": [ "### 2.4. Select project\n" @@ -206,7 +208,7 @@ { "cell_type": "code", "execution_count": null, - "id": "14", + "id": "15", "metadata": {}, "outputs": [], "source": [ @@ -217,7 +219,7 @@ }, { "cell_type": "markdown", - "id": "15", + "id": "16", "metadata": {}, "source": [ "## 3. Load compound material\n", @@ -227,7 +229,7 @@ { "cell_type": "code", "execution_count": null, - "id": "16", + "id": "17", "metadata": {}, "outputs": [], "source": [ @@ -244,7 +246,7 @@ }, { "cell_type": "markdown", - "id": "17", + "id": "18", "metadata": {}, "source": [ "### 3.2. Resolve unique elements in the compound\n" @@ -253,7 +255,7 @@ { "cell_type": "code", "execution_count": null, - "id": "18", + "id": "19", "metadata": {}, "outputs": [], "source": [ @@ -263,7 +265,7 @@ }, { "cell_type": "markdown", - "id": "19", + "id": "20", "metadata": {}, "source": [ "## 4. Resolve elemental reference materials and check total energies\n", @@ -273,7 +275,7 @@ { "cell_type": "code", "execution_count": null, - "id": "20", + "id": "21", "metadata": {}, "outputs": [], "source": [ @@ -303,7 +305,7 @@ }, { "cell_type": "markdown", - "id": "21", + "id": "22", "metadata": {}, "source": [ "### 4.2. Check refined elemental total energies\n" @@ -312,38 +314,38 @@ { "cell_type": "code", "execution_count": null, - "id": "22", + "id": "23", "metadata": {}, "outputs": [], "source": [ - "def get_elemental_total_energy_holders(\n", - " api_client, symbols, element_material_ids_by_symbol,\n", - " owner_slug, source=\"public\"\n", - "):\n", - " holders = {}\n", - " for symbol in symbols:\n", - " query = {\n", - " \"exabyteId\": {\"$in\": element_material_ids_by_symbol[symbol]},\n", - " \"slug\": \"total_energy\",\n", - " }\n", - " if source == \"my_account\":\n", - " query[\"owner.slug\"] = {\"$in\": [owner_slug, \"curators\"]}\n", - " elif source == \"curators\":\n", - " query[\"owner.slug\"] = \"curators\"\n", - "\n", - " props = api_client.properties.list(\n", - " query=query,\n", - " projection={\"sort\": {\"precision.value\": -1}, \"limit\": 1},\n", - " )\n", - " holders[symbol] = props[0] if props else None\n", - " return holders\n", - "\n", - "\n", - "elemental_total_energy_holders = get_elemental_total_energy_holders(\n", - " client, elements, element_material_ids_by_symbol,\n", - " owner_slug=selected_account.entity_cache.get(\"slug\", selected_account.name),\n", - " source=ELEMENTAL_TE_SOURCE,\n", + "# Extract the elemental TE query and projection from the workflow definition\n", + "# so they stay in sync with what the workflow actually uses at runtime.\n", + "from mat3ra.standata.workflows import WorkflowStandata\n", + "from mat3ra.wode.workflows import Workflow\n", + "\n", + "workflow_config = WorkflowStandata.filter_by_application(APPLICATION_NAME).get_by_name_first_match(\n", + " WORKFLOW_SEARCH_TERM\n", ")\n", + "formation_energy_subworkflow = Workflow.create(workflow_config).subworkflows[1]\n", + "io_unit = formation_energy_subworkflow.get_unit_by_name(name=\"io-elemental-energy\")\n", + "query_template = io_unit.input[0][\"endpoint_options\"][\"params\"][\"query\"]\n", + "projection_template = io_unit.input[0][\"endpoint_options\"][\"params\"][\"projection\"]\n", + "compound_group = eval(formation_energy_subworkflow.get_unit_by_name(name=\"assign-compound-group\").value)\n", + "\n", + "owner_slug = selected_account.entity_cache.get(\"slug\", selected_account.name)\n", + "\n", + "elemental_total_energy_holders = {}\n", + "for symbol in elements:\n", + " query = eval(query_template, {\"__builtins__\": {}}, {\n", + " \"ELEMENT_MATERIAL_IDS_BY_SYMBOL\": element_material_ids_by_symbol,\n", + " \"CURRENT_ELEMENT\": symbol,\n", + " \"COMPOUND_GROUP\": compound_group,\n", + " \"JOB_OWNER_SLUG\": owner_slug,\n", + " \"ELEMENTAL_TE_SOURCE\": ELEMENTAL_TE_SOURCE,\n", + " })\n", + " projection = eval(projection_template, {\"__builtins__\": {}})\n", + " props = client.properties.list(query=query, projection=projection)\n", + " elemental_total_energy_holders[symbol] = props[0] if props else None\n", "\n", "missing = [s for s in elements if elemental_total_energy_holders[s] is None]\n", "if missing:\n", @@ -351,12 +353,12 @@ "\n", "for symbol in elements:\n", " h = elemental_total_energy_holders[symbol]\n", - " print(f\"♻️ {symbol}: {h['data']['value']} eV (precision: {h['precision']['value']})\")\n" + " print(f\"\\u267b\\ufe0f {symbol}: {h['data']['value']} eV (precision: {h['precision']['value']})\")\n" ] }, { "cell_type": "markdown", - "id": "23", + "id": "24", "metadata": {}, "source": [ "### 4.3. Save compound material for the workflow\n" @@ -365,7 +367,7 @@ { "cell_type": "code", "execution_count": null, - "id": "24", + "id": "25", "metadata": {}, "outputs": [], "source": [ @@ -379,7 +381,7 @@ }, { "cell_type": "markdown", - "id": "25", + "id": "26", "metadata": {}, "source": [ "## 5. Configure the Formation Energy workflow\n", @@ -389,7 +391,7 @@ { "cell_type": "code", "execution_count": null, - "id": "26", + "id": "27", "metadata": {}, "outputs": [], "source": [ @@ -403,7 +405,7 @@ }, { "cell_type": "markdown", - "id": "27", + "id": "28", "metadata": {}, "source": [ "### 5.2. Load workflow from Standata and preview it\n" @@ -412,7 +414,7 @@ { "cell_type": "code", "execution_count": null, - "id": "28", + "id": "29", "metadata": {}, "outputs": [], "source": [ @@ -431,7 +433,7 @@ }, { "cell_type": "markdown", - "id": "29", + "id": "30", "metadata": {}, "source": [ "### 5.3. Modify important settings\n", @@ -441,7 +443,7 @@ { "cell_type": "code", "execution_count": null, - "id": "30", + "id": "31", "metadata": {}, "outputs": [], "source": [ @@ -458,16 +460,16 @@ }, { "cell_type": "markdown", - "id": "31", + "id": "32", "metadata": {}, "source": [ - "### 5.3. Save workflow to collection\n" + "### 5.4. Save workflow to collection\n" ] }, { "cell_type": "code", "execution_count": null, - "id": "32", + "id": "33", "metadata": {}, "outputs": [], "source": [ @@ -482,7 +484,7 @@ }, { "cell_type": "markdown", - "id": "33", + "id": "34", "metadata": {}, "source": [ "## 6. Create the compute configuration\n", @@ -492,7 +494,7 @@ { "cell_type": "code", "execution_count": null, - "id": "34", + "id": "35", "metadata": {}, "outputs": [], "source": [ @@ -502,7 +504,7 @@ }, { "cell_type": "markdown", - "id": "35", + "id": "36", "metadata": {}, "source": [ "### 6.2. Create compute configuration\n" @@ -511,7 +513,7 @@ { "cell_type": "code", "execution_count": null, - "id": "36", + "id": "37", "metadata": {}, "outputs": [], "source": [ @@ -528,7 +530,7 @@ }, { "cell_type": "markdown", - "id": "37", + "id": "38", "metadata": {}, "source": [ "## 7. Create the Formation Energy job\n", @@ -538,7 +540,7 @@ { "cell_type": "code", "execution_count": null, - "id": "38", + "id": "39", "metadata": {}, "outputs": [], "source": [ @@ -561,7 +563,7 @@ }, { "cell_type": "markdown", - "id": "39", + "id": "40", "metadata": {}, "source": [ "### 7.2. Submit the Formation Energy job and monitor the status\n" @@ -570,7 +572,7 @@ { "cell_type": "code", "execution_count": null, - "id": "40", + "id": "41", "metadata": {}, "outputs": [], "source": [ @@ -581,7 +583,7 @@ { "cell_type": "code", "execution_count": null, - "id": "41", + "id": "42", "metadata": {}, "outputs": [], "source": [ @@ -592,7 +594,7 @@ }, { "cell_type": "markdown", - "id": "42", + "id": "43", "metadata": {}, "source": [ "## 8. Retrieve results\n", @@ -602,7 +604,7 @@ { "cell_type": "code", "execution_count": null, - "id": "43", + "id": "44", "metadata": {}, "outputs": [], "source": [ @@ -615,7 +617,7 @@ { "cell_type": "code", "execution_count": null, - "id": "44", + "id": "45", "metadata": {}, "outputs": [], "source": []