From 80ebee4d1387c2ab8024d612621f3c41180cdcb9 Mon Sep 17 00:00:00 2001 From: Thijs van der Plas Date: Thu, 3 Sep 2026 13:58:44 +0200 Subject: [PATCH 1/3] Add additional aux data for testing --- .../11-TvdP-convert-aux-UK-27700-data.ipynb | 204 ++++++++++++++++++ 1 file changed, 204 insertions(+) create mode 100644 notebooks/11-TvdP-convert-aux-UK-27700-data.ipynb diff --git a/notebooks/11-TvdP-convert-aux-UK-27700-data.ipynb b/notebooks/11-TvdP-convert-aux-UK-27700-data.ipynb new file mode 100644 index 0000000..b4ccede --- /dev/null +++ b/notebooks/11-TvdP-convert-aux-UK-27700-data.ipynb @@ -0,0 +1,204 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import geopandas as gpd\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "from tqdm import tqdm\n", + "\n", + "os.chdir(\"..\")\n", + "os.getcwd()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "aef_size = 256\n", + "unlabelled = True\n", + "\n", + "DATA_DIR = os.environ.get(\"DATA_DIR\", \"data/\")\n", + "\n", + "aux_df = pd.read_csv(\n", + " f'{DATA_DIR}/s2bms/model_ready_s2bms{\"-unlabelled-merged\" if unlabelled else \"\"}.csv'\n", + ")\n", + "aef_df = pd.read_csv(\n", + " f'{DATA_DIR}/s2bms/eo/avr_aef_{aef_size}{\"_unlabelled\" if unlabelled else \"\"}.csv'\n", + ")\n", + "\n", + "import torch # only used here, to read the .pth split-index file; no tensors kept downstream\n", + "\n", + "# split_indices = torch.load(\n", + "# os.path.join(f'{DATA_DIR}/s2bms/splits/s2bms{\"_unlabelled\" if unlabelled else \"\"}_union_val_test.pth'),\n", + "# weights_only=False,\n", + "# )\n", + "split_indices = torch.load(\n", + " os.path.join(\n", + " f\"{DATA_DIR}/s2bms/splits/split_indices_s2bms+s2bms-unlabelled-20260529_2026-05-29-1438.pth\"\n", + " ),\n", + " weights_only=False,\n", + ")\n", + "aux_idx = aux_df.name_loc\n", + "aef_idx = aef_df.name_loc\n", + "\n", + "common_train_idx = pd.Series(\n", + " list(set(split_indices[\"train_indices\"]) & set(aef_idx) & set(aux_idx))\n", + ")\n", + "common_val_idx = pd.Series(list(set(split_indices[\"val_indices\"]) & set(aef_idx) & set(aux_idx)))\n", + "common_test_idx = pd.Series(list(set(split_indices[\"test_indices\"]) & set(aef_idx) & set(aux_idx)))\n", + "print(\n", + " \"train | val | test\\n\",\n", + " len(common_train_idx),\n", + " \"|\",\n", + " len(common_val_idx),\n", + " \"|\",\n", + " len(common_test_idx),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "df_coords = pd.read_csv(f\"{DATA_DIR}/s2bms/model_ready_s2bms-unlabelled-merged.csv\")\n", + "gdf_coords = gpd.GeoDataFrame(\n", + " df_coords, geometry=gpd.points_from_xy(df_coords[\"lon\"], df_coords[\"lat\"]), crs=\"EPSG:4326\"\n", + ")\n", + "gdf_coords = gdf_coords.to_crs(\"EPSG:27700\")\n", + "gdf_coords[\"x\"] = gdf_coords.geometry.x\n", + "gdf_coords[\"y\"] = gdf_coords.geometry.y\n", + "\n", + "gdf_coords.plot(\"aux_bioclim_01\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "data_env = pd.read_csv(\"/Users/tplas/Downloads/gridxdata.csv\")\n", + "data_pollution = pd.read_csv(\"/Users/tplas/Downloads/mappm252024g.csv\")\n", + "ind_row_new_header = 4\n", + "new_columns = list(data_pollution.iloc[ind_row_new_header].values)\n", + "data_pollution = data_pollution[ind_row_new_header + 1 :]\n", + "data_pollution.columns = new_columns\n", + "for i_c, c in enumerate(data_pollution.columns):\n", + " data_pollution[c] = data_pollution[c].replace(\"MISSING\", np.nan)\n", + " if c == \"pm252024g\":\n", + " data_pollution[c] = data_pollution[c].astype(float)\n", + " else:\n", + " data_pollution[c] = data_pollution[c].astype(int)\n", + "data_pollution" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "gdf_env = gpd.GeoDataFrame(\n", + " data_env, geometry=gpd.points_from_xy(data_env[\"x\"], data_env[\"y\"]), crs=\"EPSG:27700\"\n", + ")\n", + "\n", + "# gdf_env = gdf_env.to_crs(\"EPSG:4326\")\n", + "# gdf_env[\"lon\"] = gdf_env.geometry.x\n", + "# gdf_env[\"lat\"] = gdf_env.geometry.y\n", + "\n", + "gdf_pollution = gpd.GeoDataFrame(\n", + " data_pollution,\n", + " geometry=gpd.points_from_xy(data_pollution[\"x\"], data_pollution[\"y\"]),\n", + " crs=\"EPSG:27700\",\n", + ")\n", + "\n", + "# gdf_pollution = gdf_pollution.to_crs(\"EPSG:4326\")\n", + "# gdf_pollution[\"lon\"] = gdf_pollution.geometry.x\n", + "# gdf_pollution[\"lat\"] = gdf_pollution.geometry.y\n", + "\n", + "fig, ax = plt.subplots(1, 3, figsize=(10, 10))\n", + "\n", + "gdf_coords.plot(ax=ax[0], column=\"aux_bioclim_01\", markersize=1, cmap=\"terrain\", legend=True)\n", + "gdf_env.plot(ax=ax[1], column=\"temp_avg\", markersize=1, cmap=\"terrain\", legend=True)\n", + "gdf_pollution.plot(ax=ax[2], column=\"pm252024g\", markersize=1, cmap=\"terrain\", legend=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "from pyproj import Transformer\n", + "from scipy.spatial import cKDTree\n", + "\n", + "query_x, query_y = gdf_coords.geometry.x.to_numpy(), gdf_coords.geometry.y.to_numpy()\n", + "gdf_final = gdf_coords.copy()\n", + "for prefix, gdf_ref in zip([\"env\", \"pol\"], [gdf_env, gdf_pollution]): # reference grid\n", + "\n", + " ref_coords = gdf_ref[[\"x\", \"y\"]].to_numpy()\n", + " tree = cKDTree(ref_coords)\n", + "\n", + " # --- 3. query nearest reference point for every query point (single vectorized call) ---\n", + " query_coords = np.column_stack([query_x, query_y])\n", + " dist, idx = tree.query(query_coords, k=1)\n", + " n_not_within_1km = sum(dist >= np.sqrt(500**2 + 500**2))\n", + " print(f\"Number of {prefix} points not within 1km: {n_not_within_1km}\")\n", + " # --- 4. attach matched reference rows + distance (metres) back onto the query gdf ---\n", + " matched = gdf_ref.iloc[idx].reset_index(drop=True)\n", + " gdf_final = gdf_final.join(matched.add_prefix(prefix + \"_\"))\n", + " gdf_final[f\"{prefix}_dist_nn_m\"] = dist" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "gdf_final[gdf_final[\"env_dist_nn_m\"] != gdf_final[\"pol_dist_nn_m\"]]" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "aether", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 958f23a56ce71ba7452cc12f907e87c129827422 Mon Sep 17 00:00:00 2001 From: Thijs van der Plas Date: Thu, 3 Sep 2026 22:08:36 +0200 Subject: [PATCH 2/3] new concept captions --- data/s2bms/concept_captions/v5.json | 696 ++++++++++++++++++++++++++++ 1 file changed, 696 insertions(+) create mode 100644 data/s2bms/concept_captions/v5.json diff --git a/data/s2bms/concept_captions/v5.json b/data/s2bms/concept_captions/v5.json new file mode 100644 index 0000000..58baf50 --- /dev/null +++ b/data/s2bms/concept_captions/v5.json @@ -0,0 +1,696 @@ +[ + { + "concept_caption": [ + "A warm location with mild temperatures throughout the year", + "Area with a warm, sunny climate for much of the year", + "Location with a consistently warm annual climate" + ], + "is_max": true, + "col": "aux_bioclim_01" + }, + { + "concept_caption": [ + "Location where daytime warmth gives way to distinctly cooler nights", + "Area where nights turn noticeably cool after warm days", + "Climate with a pronounced swing between warm days and cool nights" + ], + "is_max": true, + "col": "aux_bioclim_02" + }, + { + "concept_caption": [ + "Location where temperatures stay fairly steady from day to day and season to season", + "Climate with a small difference between typical daily and typical seasonal temperature swings", + "Area with a very stable, even temperature pattern year-round" + ], + "is_max": true, + "col": "aux_bioclim_03" + }, + { + "concept_caption": [ + "Location with a strong contrast between summer heat and winter chill", + "Area where winters feel distinctly cold and summers feel distinctly warm", + "Climate marked by pronounced seasonal temperature swings" + ], + "is_max": true, + "col": "aux_bioclim_04" + }, + { + "concept_caption": [ + "Location with notably hot summer days", + "Area that regularly experiences intense summer heat", + "Location prone to intense heat during summer heatwaves" + ], + "is_max": true, + "col": "aux_bioclim_05" + }, + { + "concept_caption": [ + "Location with consistently mild winters", + "Area where the coldest month still feels mild", + "Location with a low risk of hard winter frost" + ], + "is_max": true, + "col": "aux_bioclim_06" + }, + { + "concept_caption": [ + "Location with a wide gap between summer warmth and winter cold", + "Area where summer and winter temperatures differ sharply", + "Climate with a wide annual temperature range" + ], + "is_max": true, + "col": "aux_bioclim_07" + }, + { + "concept_caption": [ + "Region where the wettest time of year is also mild", + "Location with mild, damp weather during its rainiest months", + "Area where rainfall coincides with mild temperatures" + ], + "is_max": true, + "col": "aux_bioclim_08" + }, + { + "concept_caption": [ + "Region where its driest months are also mild", + "Location with a mild, dry stretch of the year", + "Area with a dry season that stays mild" + ], + "is_max": true, + "col": "aux_bioclim_09" + }, + { + "concept_caption": [ + "Region with a long, warm summer season", + "Location that stays mild-to-warm for an extended summer period", + "Area with consistently warm temperatures across its three warmest months" + ], + "is_max": true, + "col": "aux_bioclim_10" + }, + { + "concept_caption": [ + "Region with a mild winter season overall", + "Location where the coldest months stay mild", + "Area with a gentle, low-frost winter season" + ], + "is_max": true, + "col": "aux_bioclim_11" + }, + { + "concept_caption": [ + "A very rainy area with high annual rainfall", + "Location that receives abundant rainfall throughout the year", + "Area with heavy total yearly rainfall" + ], + "is_max": true, + "col": "aux_bioclim_12" + }, + { + "concept_caption": [ + "Region with an intensely rainy peak month", + "Location that experiences a month of very heavy rainfall", + "Area with a sharp monthly rainfall peak" + ], + "is_max": true, + "col": "aux_bioclim_13" + }, + { + "concept_caption": [ + "Region where rain falls even in its driest month", + "Location with a mild, damp climate where rain falls year-round", + "Area with rain spread fairly evenly throughout the year" + ], + "is_max": true, + "col": "aux_bioclim_14" + }, + { + "concept_caption": [ + "Region with a strongly seasonal rainfall pattern", + "Location with a clear difference between its wetter and drier months", + "Area with a pronounced wet-and-dry rainfall pattern across the year" + ], + "is_max": true, + "col": "aux_bioclim_15" + }, + { + "concept_caption": [ + "Region with one markedly wet season", + "Location where a three-month stretch brings much of the year's rain", + "Area with a concentrated rainy season" + ], + "is_max": true, + "col": "aux_bioclim_16" + }, + { + "concept_caption": [ + "Region where even its driest quarter receives notable rainfall", + "Location with a mild, damp climate throughout its driest season", + "Area where even the driest season stays fairly damp" + ], + "is_max": true, + "col": "aux_bioclim_17" + }, + { + "concept_caption": [ + "Region with rainy summers", + "Location where the warm season is also wet", + "Area with frequent summer showers" + ], + "is_max": true, + "col": "aux_bioclim_18" + }, + { + "concept_caption": [ + "Region with wet, rainy winters", + "Location where the cold season brings heavy rain", + "Area with a rainy winter climate" + ], + "is_max": true, + "col": "aux_bioclim_19" + }, + { + "concept_caption": [ + "Heavily built-up, artificial landscape", + "Area dominated by man-made structures and surfaces", + "Urbanized and developed land with little natural cover" + ], + "is_max": true, + "col": "aux_corine_frac_1" + }, + { + "concept_caption": [ + "Densely populated area with many houses", + "Residential neighborhood made up of homes and apartment buildings", + "Built-up area primarily used for housing", + "Town or city district covered mostly by residential buildings" + ], + "is_max": true, + "col": "aux_corine_frac_11" + }, + { + "concept_caption": [ + "Densely built city center where buildings almost completely cover the ground", + "Continuous city blocks where buildings stand tightly packed together", + "Compact urban core with near-total building coverage" + ], + "is_max": true, + "col": "aux_corine_frac_111" + }, + { + "concept_caption": [ + "Suburban area with houses separated by gardens and green space", + "Low-density residential area where buildings are spaced apart", + "Neighborhood of houses with yards and open gaps between buildings" + ], + "is_max": true, + "col": "aux_corine_frac_112" + }, + { + "concept_caption": [ + "Area dominated by factories, warehouses, and commercial buildings", + "Industrial and commercial zone with transport infrastructure", + "Business park or industrial estate with large commercial buildings" + ], + "is_max": true, + "col": "aux_corine_frac_12" + }, + { + "concept_caption": [ + "Site occupied by factories or large commercial facilities", + "Industrial complex with warehouses and manufacturing buildings", + "Commercial zone with large stores and industrial units" + ], + "is_max": true, + "col": "aux_corine_frac_121" + }, + { + "concept_caption": [ + "Land disturbed by mining, dumping, or construction activity", + "Area of quarries, waste sites, or active building sites", + "Excavated or disturbed ground from extraction or construction work" + ], + "is_max": true, + "col": "aux_corine_frac_13" + }, + { + "concept_caption": [ + "Open-pit mine or quarry site", + "Area of active mineral or gravel extraction", + "Land scarred by quarrying and mining operations" + ], + "is_max": true, + "col": "aux_corine_frac_131" + }, + { + "concept_caption": [ + "Man-made green space within a built-up area", + "Urban park or landscaped green area", + "Vegetated recreational area within an artificial setting" + ], + "is_max": true, + "col": "aux_corine_frac_14" + }, + { + "concept_caption": [ + "City park with grass, trees, and paths", + "Urban green space used for public recreation", + "Landscaped park within a town or city" + ], + "is_max": true, + "col": "aux_corine_frac_141" + }, + { + "concept_caption": [ + "Sports complex with pitches or courts", + "Golf course, stadium, or leisure facility", + "Area dedicated to organized sports and recreation" + ], + "is_max": true, + "col": "aux_corine_frac_142" + }, + { + "concept_caption": [ + "Farmland used for crops or livestock", + "Rural agricultural landscape", + "Land dominated by farming activity" + ], + "is_max": true, + "col": "aux_corine_frac_2" + }, + { + "concept_caption": [ + "Cropland used for growing annual crops", + "Plowed farmland planted with rotating crops", + "Fields of cultivated arable crops" + ], + "is_max": true, + "col": "aux_corine_frac_21" + }, + { + "concept_caption": [ + "Rain-fed farmland watered solely by natural rainfall", + "Dryland crop fields relying only on natural rainfall", + "Fields of dryland cereals or other rotating crops" + ], + "is_max": true, + "col": "aux_corine_frac_211" + }, + { + "concept_caption": [ + "Grazing land used for livestock", + "Meadow used as pasture for grazing animals", + "Grassy field where cattle or sheep graze" + ], + "is_max": true, + "col": "aux_corine_frac_23" + }, + { + "concept_caption": [ + "Permanent grassland grazed by livestock", + "Pasture field with grazing cattle or sheep", + "Grassy meadow maintained for animal grazing" + ], + "is_max": true, + "col": "aux_corine_frac_231" + }, + { + "concept_caption": [ + "Patchwork landscape mixing different crops and farmland types", + "Farmland where crops, pastures, and natural vegetation intermix", + "Mosaic of small fields with mixed agricultural uses" + ], + "is_max": true, + "col": "aux_corine_frac_24" + }, + { + "concept_caption": [ + "Farmland interspersed with patches of natural vegetation", + "Agricultural area that still retains significant natural habitat", + "Farmed landscape mixed with pockets of semi-natural land" + ], + "is_max": true, + "col": "aux_corine_frac_243" + }, + { + "concept_caption": [ + "Forested or semi-natural landscape", + "Wild, largely unfarmed area covered by trees and natural vegetation", + "Woodland or natural terrain with minimal human development" + ], + "is_max": true, + "col": "aux_corine_frac_3" + }, + { + "concept_caption": [ + "Area covered by woodland and trees", + "Dense forest landscape", + "Land covered mostly by tree canopy" + ], + "is_max": true, + "col": "aux_corine_frac_31" + }, + { + "concept_caption": [ + "Deciduous forest of broad-leaved trees", + "Woodland dominated by broadleaf, leafy trees", + "Forest of broad-leaved tree species" + ], + "is_max": true, + "col": "aux_corine_frac_311" + }, + { + "concept_caption": [ + "Evergreen coniferous forest", + "Woodland dominated by needle-leaved conifer trees", + "Forest of conifer species" + ], + "is_max": true, + "col": "aux_corine_frac_312" + }, + { + "concept_caption": [ + "Forest mixing both broadleaf and coniferous trees", + "Woodland with a blend of deciduous and evergreen trees", + "Mixed-species forest of conifers and broadleaf trees together" + ], + "is_max": true, + "col": "aux_corine_frac_313" + }, + { + "concept_caption": [ + "Open landscape of shrubs and grassland vegetation", + "Natural terrain covered by low shrubs and herbaceous plants", + "Scrubland or rough grassland with sparse tree cover" + ], + "is_max": true, + "col": "aux_corine_frac_32" + }, + { + "concept_caption": [ + "Open natural grassland growing wild", + "Natural meadow of wild grasses", + "Wild, natural grassy plain" + ], + "is_max": true, + "col": "aux_corine_frac_321" + }, + { + "concept_caption": [ + "Open heather moorland with low shrubby vegetation", + "Heathland landscape of low-growing shrubby vegetation", + "Windswept moor with heather and scrub vegetation" + ], + "is_max": true, + "col": "aux_corine_frac_322" + }, + { + "concept_caption": [ + "Transitional landscape of young trees and regenerating shrub", + "Area of scattered saplings and shrubby regrowth between grassland and forest", + "Land in transition from shrubland to forest, with sparse young trees" + ], + "is_max": true, + "col": "aux_corine_frac_324" + }, + { + "concept_caption": [ + "Sparse, rocky terrain with minimal vegetation", + "Bare ground such as rock, shingle, or sand with minimal plant cover", + "Open, sparsely vegetated natural area" + ], + "is_max": true, + "col": "aux_corine_frac_33" + }, + { + "concept_caption": [ + "Wetland landscape of marshes or waterlogged ground", + "Area of flooded or saturated land with wetland vegetation", + "Boggy, water-saturated terrain" + ], + "is_max": true, + "col": "aux_corine_frac_4" + }, + { + "concept_caption": [ + "Freshwater marsh or bog away from the coast", + "Inland wetland of waterlogged, marshy ground", + "Peat bog or freshwater marsh in an inland setting" + ], + "is_max": true, + "col": "aux_corine_frac_41" + }, + { + "concept_caption": [ + "Freshwater marsh with reeds and standing water", + "Low-lying inland marsh prone to flooding", + "Wet grassy marshland fed by fresh water" + ], + "is_max": true, + "col": "aux_corine_frac_411" + }, + { + "concept_caption": [ + "Coastal wetland shaped by tides", + "Tidal marsh or coastal wetland near the sea", + "Saline wetland along the coastline" + ], + "is_max": true, + "col": "aux_corine_frac_42" + }, + { + "concept_caption": [ + "Salt marsh vegetated with salt-tolerant plants", + "Coastal salt meadow flooded by high tides", + "Tidal salt marsh along an estuary or coast" + ], + "is_max": true, + "col": "aux_corine_frac_421" + }, + { + "concept_caption": [ + "Muddy or sandy flats exposed at low tide", + "Bare tidal flat between high and low water marks", + "Coastal mudflat submerged and exposed with the tides" + ], + "is_max": true, + "col": "aux_corine_frac_423" + }, + { + "concept_caption": [ + "Area covered by open water", + "Location dominated by a body of water", + "Landscape largely covered by water, such as a lake, river, or the sea" + ], + "is_max": true, + "col": "aux_corine_frac_5" + }, + { + "concept_caption": [ + "Freshwater lake or river, away from the sea", + "Inland body of fresh water", + "Landscape dominated by a river or freshwater lake" + ], + "is_max": true, + "col": "aux_corine_frac_51" + }, + { + "concept_caption": [ + "Still freshwater lake or reservoir", + "Standing body of water such as a lake or reservoir", + "Area adjacent to a large inland lake or reservoir" + ], + "is_max": true, + "col": "aux_corine_frac_512" + }, + { + "concept_caption": [ + "Coastal or open sea waters", + "Area covered by marine, saltwater seas", + "Location dominated by ocean or sea water" + ], + "is_max": true, + "col": "aux_corine_frac_52" + }, + { + "concept_caption": [ + "Estuary where a river meets the sea", + "Tidal river mouth blending fresh and salt water", + "Coastal estuary at a river's outlet to the sea" + ], + "is_max": true, + "col": "aux_corine_frac_522" + }, + { + "concept_caption": [ + "Open sea or ocean waters", + "Location out at sea, far from land", + "Vast expanse of ocean water" + ], + "is_max": true, + "col": "aux_corine_frac_523" + }, + { + "concept_caption": [ + "Remote area with a point far from any road", + "Location containing an inaccessible spot distant from roads", + "Area reaching far beyond the nearest roadway" + ], + "is_max": true, + "col": "aux_maxdist_road" + }, + { + "concept_caption": [ + "Sparsely connected area, generally far from roads", + "Region with a thin, spread-out road network", + "Location generally distant from roads across its whole area" + ], + "is_max": true, + "col": "aux_meandist_road" + }, + { + "concept_caption": [ + "Densely populated area with many people per square kilometer", + "Crowded urban area with high population density", + "Location with a high concentration of residents" + ], + "is_max": true, + "col": "aux_pop_density" + }, + { + "concept_caption": [ + "Populous area home to a large number of people", + "Large town or city with a big total population", + "Location with a substantial resident population" + ], + "is_max": true, + "col": "aux_total_population" + }, + { + "concept_caption": [ + "Area with a highly diverse mix of different land-cover types", + "Location where many different land-cover types occur together in similar proportions", + "Landscape combining several land-cover types in a varied mosaic" + ], + "is_max": true, + "col": "shdi_lc" + }, + { + "concept_caption": [ + "Area covered by broadleaf woodland", + "Deciduous woodland landscape of broad-leaved trees", + "Location with extensive broadleaf tree cover" + ], + "is_max": true, + "col": "p_broadleaf" + }, + { + "concept_caption": [ + "Area covered by coniferous woodland", + "Evergreen conifer forest landscape", + "Location with extensive coniferous tree cover" + ], + "is_max": true, + "col": "p_conifer" + }, + { + "concept_caption": [ + "Area of arable farmland used for growing crops", + "Cropland landscape under cultivation", + "Location dominated by ploughed, cultivated fields" + ], + "is_max": true, + "col": "p_arable" + }, + { + "concept_caption": [ + "Intensively managed grassland, reseeded and fertilised for farming", + "Grassland kept productive through regular management and fertiliser use", + "Improved pasture maintained for productive grazing or silage" + ], + "is_max": true, + "col": "p_imprv_grsl" + }, + { + "concept_caption": [ + "Semi-natural, unimproved grassland with a diverse mix of wild plants", + "Species-rich rough grassland that has developed naturally over time", + "Species-rich meadow or rough grazing land retaining natural vegetation" + ], + "is_max": true, + "col": "p_smnat_grsl" + }, + { + "concept_caption": [ + "Upland terrain of mountain ground or peat bog", + "High, rugged terrain or waterlogged boggy ground", + "Area of montane or bog habitat with sparse vegetation" + ], + "is_max": true, + "col": "p_mountain" + }, + { + "concept_caption": [ + "Area of coastal terrain bordering the shoreline", + "Land classified as coastal habitat, close to the sea's edge", + "Shoreline landscape directly adjoining the coast" + ], + "is_max": true, + "col": "p_coast" + }, + { + "concept_caption": [ + "Built-up urban terrain with extensive development", + "Area classified as urban land, dense with buildings and infrastructure", + "Town or city landscape with heavy urban development" + ], + "is_max": true, + "col": "p_urban" + }, + { + "concept_caption": [ + "Location far inland, a long distance from the sea", + "Area well away from any coastline", + "Inland location distant from the nearest coast" + ], + "is_max": true, + "col": "dist_to_sea" + }, + { + "concept_caption": [ + "High-altitude location, well above sea level", + "Upland area at a high elevation", + "Elevated terrain, situated high above sea level" + ], + "is_max": true, + "col": "elevation" + }, + { + "concept_caption": [ + "Steeply sloping terrain", + "Area with a pronounced incline in the land surface", + "Hilly terrain with a steep gradient" + ], + "is_max": true, + "col": "slope" + }, + { + "concept_caption": [ + "Landscape with a dense network of rivers and streams", + "Area richly veined with watercourses", + "Location where rivers and streams are closely spaced across the terrain" + ], + "is_max": true, + "col": "river_dens" + }, + { + "concept_caption": [ + "Area with elevated background PM2.5 air pollution", + "Location with elevated fine particulate air pollution", + "Polluted air with high concentrations of fine particulate matter" + ], + "is_max": true, + "col": "pm252024g" + } +] From 27a6a20094d76cccef54e4db7868ea12e8a2570d Mon Sep 17 00:00:00 2001 From: Thijs van der Plas Date: Thu, 3 Sep 2026 22:32:57 +0200 Subject: [PATCH 3/3] Update captions with val_av All aux- concepts with R2 > 0.5 on merged data --- data/s2bms/concept_captions/v6.json | 773 ++++++++++++++++++++++++++++ notebooks/10-TvdP-probe_aef.ipynb | 408 +++++++++++++++ 2 files changed, 1181 insertions(+) create mode 100644 data/s2bms/concept_captions/v6.json create mode 100644 notebooks/10-TvdP-probe_aef.ipynb diff --git a/data/s2bms/concept_captions/v6.json b/data/s2bms/concept_captions/v6.json new file mode 100644 index 0000000..8b4c75a --- /dev/null +++ b/data/s2bms/concept_captions/v6.json @@ -0,0 +1,773 @@ +[ + { + "concept_caption": [ + "A warm location with mild temperatures throughout the year", + "Area with a warm, sunny climate for much of the year", + "Location with a consistently warm annual climate" + ], + "is_max": true, + "col": "aux_bioclim_01", + "val_av": true + }, + { + "concept_caption": [ + "Location where daytime warmth gives way to distinctly cooler nights", + "Area where nights turn noticeably cool after warm days", + "Climate with a pronounced swing between warm days and cool nights" + ], + "is_max": true, + "col": "aux_bioclim_02", + "val_av": true + }, + { + "concept_caption": [ + "Location where temperatures stay fairly steady from day to day and season to season", + "Climate with a small difference between typical daily and typical seasonal temperature swings", + "Area with a very stable, even temperature pattern year-round" + ], + "is_max": true, + "col": "aux_bioclim_03", + "val_av": false + }, + { + "concept_caption": [ + "Location with a strong contrast between summer heat and winter chill", + "Area where winters feel distinctly cold and summers feel distinctly warm", + "Climate marked by pronounced seasonal temperature swings" + ], + "is_max": true, + "col": "aux_bioclim_04", + "val_av": true + }, + { + "concept_caption": [ + "Location with notably hot summer days", + "Area that regularly experiences intense summer heat", + "Location prone to intense heat during summer heatwaves" + ], + "is_max": true, + "col": "aux_bioclim_05", + "val_av": true + }, + { + "concept_caption": [ + "Location with consistently mild winters", + "Area where the coldest month still feels mild", + "Location with a low risk of hard winter frost" + ], + "is_max": true, + "col": "aux_bioclim_06", + "val_av": true + }, + { + "concept_caption": [ + "Location with a wide gap between summer warmth and winter cold", + "Area where summer and winter temperatures differ sharply", + "Climate with a wide annual temperature range" + ], + "is_max": true, + "col": "aux_bioclim_07", + "val_av": true + }, + { + "concept_caption": [ + "Region where the wettest time of year is also mild", + "Location with mild, damp weather during its rainiest months", + "Area where rainfall coincides with mild temperatures" + ], + "is_max": true, + "col": "aux_bioclim_08", + "val_av": false + }, + { + "concept_caption": [ + "Region where its driest months are also mild", + "Location with a mild, dry stretch of the year", + "Area with a dry season that stays mild" + ], + "is_max": true, + "col": "aux_bioclim_09", + "val_av": false + }, + { + "concept_caption": [ + "Region with a long, warm summer season", + "Location that stays mild-to-warm for an extended summer period", + "Area with consistently warm temperatures across its three warmest months" + ], + "is_max": true, + "col": "aux_bioclim_10", + "val_av": true + }, + { + "concept_caption": [ + "Region with a mild winter season overall", + "Location where the coldest months stay mild", + "Area with a gentle, low-frost winter season" + ], + "is_max": true, + "col": "aux_bioclim_11", + "val_av": true + }, + { + "concept_caption": [ + "A very rainy area with high annual rainfall", + "Location that receives abundant rainfall throughout the year", + "Area with heavy total yearly rainfall" + ], + "is_max": true, + "col": "aux_bioclim_12", + "val_av": true + }, + { + "concept_caption": [ + "Region with an intensely rainy peak month", + "Location that experiences a month of very heavy rainfall", + "Area with a sharp monthly rainfall peak" + ], + "is_max": true, + "col": "aux_bioclim_13", + "val_av": true + }, + { + "concept_caption": [ + "Region where rain falls even in its driest month", + "Location with a mild, damp climate where rain falls year-round", + "Area with rain spread fairly evenly throughout the year" + ], + "is_max": true, + "col": "aux_bioclim_14", + "val_av": true + }, + { + "concept_caption": [ + "Region with a strongly seasonal rainfall pattern", + "Location with a clear difference between its wetter and drier months", + "Area with a pronounced wet-and-dry rainfall pattern across the year" + ], + "is_max": true, + "col": "aux_bioclim_15", + "val_av": true + }, + { + "concept_caption": [ + "Region with one markedly wet season", + "Location where a three-month stretch brings much of the year's rain", + "Area with a concentrated rainy season" + ], + "is_max": true, + "col": "aux_bioclim_16", + "val_av": true + }, + { + "concept_caption": [ + "Region where even its driest quarter receives notable rainfall", + "Location with a mild, damp climate throughout its driest season", + "Area where even the driest season stays fairly damp" + ], + "is_max": true, + "col": "aux_bioclim_17", + "val_av": true + }, + { + "concept_caption": [ + "Region with rainy summers", + "Location where the warm season is also wet", + "Area with frequent summer showers" + ], + "is_max": true, + "col": "aux_bioclim_18", + "val_av": true + }, + { + "concept_caption": [ + "Region with wet, rainy winters", + "Location where the cold season brings heavy rain", + "Area with a rainy winter climate" + ], + "is_max": true, + "col": "aux_bioclim_19", + "val_av": true + }, + { + "concept_caption": [ + "Heavily built-up, artificial landscape", + "Area dominated by man-made structures and surfaces", + "Urbanized and developed land with little natural cover" + ], + "is_max": true, + "col": "aux_corine_frac_1", + "val_av": true + }, + { + "concept_caption": [ + "Densely populated area with many houses", + "Residential neighborhood made up of homes and apartment buildings", + "Built-up area primarily used for housing", + "Town or city district covered mostly by residential buildings" + ], + "is_max": true, + "col": "aux_corine_frac_11", + "val_av": true + }, + { + "concept_caption": [ + "Densely built city center where buildings almost completely cover the ground", + "Continuous city blocks where buildings stand tightly packed together", + "Compact urban core with near-total building coverage" + ], + "is_max": true, + "col": "aux_corine_frac_111", + "val_av": false + }, + { + "concept_caption": [ + "Suburban area with houses separated by gardens and green space", + "Low-density residential area where buildings are spaced apart", + "Neighborhood of houses with yards and open gaps between buildings" + ], + "is_max": true, + "col": "aux_corine_frac_112", + "val_av": true + }, + { + "concept_caption": [ + "Area dominated by factories, warehouses, and commercial buildings", + "Industrial and commercial zone with transport infrastructure", + "Business park or industrial estate with large commercial buildings" + ], + "is_max": true, + "col": "aux_corine_frac_12", + "val_av": false + }, + { + "concept_caption": [ + "Site occupied by factories or large commercial facilities", + "Industrial complex with warehouses and manufacturing buildings", + "Commercial zone with large stores and industrial units" + ], + "is_max": true, + "col": "aux_corine_frac_121", + "val_av": false + }, + { + "concept_caption": [ + "Land disturbed by mining, dumping, or construction activity", + "Area of quarries, waste sites, or active building sites", + "Excavated or disturbed ground from extraction or construction work" + ], + "is_max": true, + "col": "aux_corine_frac_13", + "val_av": false + }, + { + "concept_caption": [ + "Open-pit mine or quarry site", + "Area of active mineral or gravel extraction", + "Land scarred by quarrying and mining operations" + ], + "is_max": true, + "col": "aux_corine_frac_131", + "val_av": false + }, + { + "concept_caption": [ + "Man-made green space within a built-up area", + "Urban park or landscaped green area", + "Vegetated recreational area within an artificial setting" + ], + "is_max": true, + "col": "aux_corine_frac_14", + "val_av": false + }, + { + "concept_caption": [ + "City park with grass, trees, and paths", + "Urban green space used for public recreation", + "Landscaped park within a town or city" + ], + "is_max": true, + "col": "aux_corine_frac_141", + "val_av": false + }, + { + "concept_caption": [ + "Sports complex with pitches or courts", + "Golf course, stadium, or leisure facility", + "Area dedicated to organized sports and recreation" + ], + "is_max": true, + "col": "aux_corine_frac_142", + "val_av": false + }, + { + "concept_caption": [ + "Farmland used for crops or livestock", + "Rural agricultural landscape", + "Land dominated by farming activity" + ], + "is_max": true, + "col": "aux_corine_frac_2", + "val_av": true + }, + { + "concept_caption": [ + "Cropland used for growing annual crops", + "Plowed farmland planted with rotating crops", + "Fields of cultivated arable crops" + ], + "is_max": true, + "col": "aux_corine_frac_21", + "val_av": true + }, + { + "concept_caption": [ + "Rain-fed farmland watered solely by natural rainfall", + "Dryland crop fields relying only on natural rainfall", + "Fields of dryland cereals or other rotating crops" + ], + "is_max": true, + "col": "aux_corine_frac_211", + "val_av": true + }, + { + "concept_caption": [ + "Grazing land used for livestock", + "Meadow used as pasture for grazing animals", + "Grassy field where cattle or sheep graze" + ], + "is_max": true, + "col": "aux_corine_frac_23", + "val_av": true + }, + { + "concept_caption": [ + "Permanent grassland grazed by livestock", + "Pasture field with grazing cattle or sheep", + "Grassy meadow maintained for animal grazing" + ], + "is_max": true, + "col": "aux_corine_frac_231", + "val_av": true + }, + { + "concept_caption": [ + "Patchwork landscape mixing different crops and farmland types", + "Farmland where crops, pastures, and natural vegetation intermix", + "Mosaic of small fields with mixed agricultural uses" + ], + "is_max": true, + "col": "aux_corine_frac_24", + "val_av": false + }, + { + "concept_caption": [ + "Farmland interspersed with patches of natural vegetation", + "Agricultural area that still retains significant natural habitat", + "Farmed landscape mixed with pockets of semi-natural land" + ], + "is_max": true, + "col": "aux_corine_frac_243", + "val_av": false + }, + { + "concept_caption": [ + "Forested or semi-natural landscape", + "Wild, largely unfarmed area covered by trees and natural vegetation", + "Woodland or natural terrain with minimal human development" + ], + "is_max": true, + "col": "aux_corine_frac_3", + "val_av": true + }, + { + "concept_caption": [ + "Area covered by woodland and trees", + "Dense forest landscape", + "Land covered mostly by tree canopy" + ], + "is_max": true, + "col": "aux_corine_frac_31", + "val_av": true + }, + { + "concept_caption": [ + "Deciduous forest of broad-leaved trees", + "Woodland dominated by broadleaf, leafy trees", + "Forest of broad-leaved tree species" + ], + "is_max": true, + "col": "aux_corine_frac_311", + "val_av": true + }, + { + "concept_caption": [ + "Evergreen coniferous forest", + "Woodland dominated by needle-leaved conifer trees", + "Forest of conifer species" + ], + "is_max": true, + "col": "aux_corine_frac_312", + "val_av": true + }, + { + "concept_caption": [ + "Forest mixing both broadleaf and coniferous trees", + "Woodland with a blend of deciduous and evergreen trees", + "Mixed-species forest of conifers and broadleaf trees together" + ], + "is_max": true, + "col": "aux_corine_frac_313", + "val_av": false + }, + { + "concept_caption": [ + "Open landscape of shrubs and grassland vegetation", + "Natural terrain covered by low shrubs and herbaceous plants", + "Scrubland or rough grassland with sparse tree cover" + ], + "is_max": true, + "col": "aux_corine_frac_32", + "val_av": true + }, + { + "concept_caption": [ + "Open natural grassland growing wild", + "Natural meadow of wild grasses", + "Wild, natural grassy plain" + ], + "is_max": true, + "col": "aux_corine_frac_321", + "val_av": true + }, + { + "concept_caption": [ + "Open heather moorland with low shrubby vegetation", + "Heathland landscape of low-growing shrubby vegetation", + "Windswept moor with heather and scrub vegetation" + ], + "is_max": true, + "col": "aux_corine_frac_322", + "val_av": true + }, + { + "concept_caption": [ + "Transitional landscape of young trees and regenerating shrub", + "Area of scattered saplings and shrubby regrowth between grassland and forest", + "Land in transition from shrubland to forest, with sparse young trees" + ], + "is_max": true, + "col": "aux_corine_frac_324", + "val_av": false + }, + { + "concept_caption": [ + "Sparse, rocky terrain with minimal vegetation", + "Bare ground such as rock, shingle, or sand with minimal plant cover", + "Open, sparsely vegetated natural area" + ], + "is_max": true, + "col": "aux_corine_frac_33", + "val_av": true + }, + { + "concept_caption": [ + "Wetland landscape of marshes or waterlogged ground", + "Area of flooded or saturated land with wetland vegetation", + "Boggy, water-saturated terrain" + ], + "is_max": true, + "col": "aux_corine_frac_4", + "val_av": true + }, + { + "concept_caption": [ + "Freshwater marsh or bog away from the coast", + "Inland wetland of waterlogged, marshy ground", + "Peat bog or freshwater marsh in an inland setting" + ], + "is_max": true, + "col": "aux_corine_frac_41", + "val_av": true + }, + { + "concept_caption": [ + "Freshwater marsh with reeds and standing water", + "Low-lying inland marsh prone to flooding", + "Wet grassy marshland fed by fresh water" + ], + "is_max": true, + "col": "aux_corine_frac_411", + "val_av": false + }, + { + "concept_caption": [ + "Coastal wetland shaped by tides", + "Tidal marsh or coastal wetland near the sea", + "Saline wetland along the coastline" + ], + "is_max": true, + "col": "aux_corine_frac_42", + "val_av": false + }, + { + "concept_caption": [ + "Salt marsh vegetated with salt-tolerant plants", + "Coastal salt meadow flooded by high tides", + "Tidal salt marsh along an estuary or coast" + ], + "is_max": true, + "col": "aux_corine_frac_421", + "val_av": false + }, + { + "concept_caption": [ + "Muddy or sandy flats exposed at low tide", + "Bare tidal flat between high and low water marks", + "Coastal mudflat submerged and exposed with the tides" + ], + "is_max": true, + "col": "aux_corine_frac_423", + "val_av": false + }, + { + "concept_caption": [ + "Area covered by open water", + "Location dominated by a body of water", + "Landscape largely covered by water, such as a lake, river, or the sea" + ], + "is_max": true, + "col": "aux_corine_frac_5", + "val_av": true + }, + { + "concept_caption": [ + "Freshwater lake or river, away from the sea", + "Inland body of fresh water", + "Landscape dominated by a river or freshwater lake" + ], + "is_max": true, + "col": "aux_corine_frac_51", + "val_av": false + }, + { + "concept_caption": [ + "Still freshwater lake or reservoir", + "Standing body of water such as a lake or reservoir", + "Area adjacent to a large inland lake or reservoir" + ], + "is_max": true, + "col": "aux_corine_frac_512", + "val_av": false + }, + { + "concept_caption": [ + "Coastal or open sea waters", + "Area covered by marine, saltwater seas", + "Location dominated by ocean or sea water" + ], + "is_max": true, + "col": "aux_corine_frac_52", + "val_av": true + }, + { + "concept_caption": [ + "Estuary where a river meets the sea", + "Tidal river mouth blending fresh and salt water", + "Coastal estuary at a river's outlet to the sea" + ], + "is_max": true, + "col": "aux_corine_frac_522", + "val_av": false + }, + { + "concept_caption": [ + "Open sea or ocean waters", + "Location out at sea, far from land", + "Vast expanse of ocean water" + ], + "is_max": true, + "col": "aux_corine_frac_523", + "val_av": true + }, + { + "concept_caption": [ + "Remote area with a point far from any road", + "Location containing an inaccessible spot distant from roads", + "Area reaching far beyond the nearest roadway" + ], + "is_max": true, + "col": "aux_maxdist_road", + "val_av": true + }, + { + "concept_caption": [ + "Sparsely connected area, generally far from roads", + "Region with a thin, spread-out road network", + "Location generally distant from roads across its whole area" + ], + "is_max": true, + "col": "aux_meandist_road", + "val_av": true + }, + { + "concept_caption": [ + "Densely populated area with many people per square kilometer", + "Crowded urban area with high population density", + "Location with a high concentration of residents" + ], + "is_max": true, + "col": "aux_pop_density", + "val_av": true + }, + { + "concept_caption": [ + "Populous area home to a large number of people", + "Large town or city with a big total population", + "Location with a substantial resident population" + ], + "is_max": true, + "col": "aux_total_population", + "val_av": true + }, + { + "concept_caption": [ + "Area with a highly diverse mix of different land-cover types", + "Location where many different land-cover types occur together in similar proportions", + "Landscape combining several land-cover types in a varied mosaic" + ], + "is_max": true, + "col": "shdi_lc", + "val_av": false + }, + { + "concept_caption": [ + "Area covered by broadleaf woodland", + "Deciduous woodland landscape of broad-leaved trees", + "Location with extensive broadleaf tree cover" + ], + "is_max": true, + "col": "p_broadleaf", + "val_av": false + }, + { + "concept_caption": [ + "Area covered by coniferous woodland", + "Evergreen conifer forest landscape", + "Location with extensive coniferous tree cover" + ], + "is_max": true, + "col": "p_conifer", + "val_av": false + }, + { + "concept_caption": [ + "Area of arable farmland used for growing crops", + "Cropland landscape under cultivation", + "Location dominated by ploughed, cultivated fields" + ], + "is_max": true, + "col": "p_arable", + "val_av": false + }, + { + "concept_caption": [ + "Intensively managed grassland, reseeded and fertilised for farming", + "Grassland kept productive through regular management and fertiliser use", + "Improved pasture maintained for productive grazing or silage" + ], + "is_max": true, + "col": "p_imprv_grsl", + "val_av": false + }, + { + "concept_caption": [ + "Semi-natural, unimproved grassland with a diverse mix of wild plants", + "Species-rich rough grassland that has developed naturally over time", + "Species-rich meadow or rough grazing land retaining natural vegetation" + ], + "is_max": true, + "col": "p_smnat_grsl", + "val_av": false + }, + { + "concept_caption": [ + "Upland terrain of mountain ground or peat bog", + "High, rugged terrain or waterlogged boggy ground", + "Area of montane or bog habitat with sparse vegetation" + ], + "is_max": true, + "col": "p_mountain", + "val_av": false + }, + { + "concept_caption": [ + "Area of coastal terrain bordering the shoreline", + "Land classified as coastal habitat, close to the sea's edge", + "Shoreline landscape directly adjoining the coast" + ], + "is_max": true, + "col": "p_coast", + "val_av": false + }, + { + "concept_caption": [ + "Built-up urban terrain with extensive development", + "Area classified as urban land, dense with buildings and infrastructure", + "Town or city landscape with heavy urban development" + ], + "is_max": true, + "col": "p_urban", + "val_av": false + }, + { + "concept_caption": [ + "Location far inland, a long distance from the sea", + "Area well away from any coastline", + "Inland location distant from the nearest coast" + ], + "is_max": true, + "col": "dist_to_sea", + "val_av": false + }, + { + "concept_caption": [ + "High-altitude location, well above sea level", + "Upland area at a high elevation", + "Elevated terrain, situated high above sea level" + ], + "is_max": true, + "col": "elevation", + "val_av": false + }, + { + "concept_caption": [ + "Steeply sloping terrain", + "Area with a pronounced incline in the land surface", + "Hilly terrain with a steep gradient" + ], + "is_max": true, + "col": "slope", + "val_av": false + }, + { + "concept_caption": [ + "Landscape with a dense network of rivers and streams", + "Area richly veined with watercourses", + "Location where rivers and streams are closely spaced across the terrain" + ], + "is_max": true, + "col": "river_dens", + "val_av": false + }, + { + "concept_caption": [ + "Area with elevated background PM2.5 air pollution", + "Location with elevated fine particulate air pollution", + "Polluted air with high concentrations of fine particulate matter" + ], + "is_max": true, + "col": "pm252024g", + "val_av": false + } +] diff --git a/notebooks/10-TvdP-probe_aef.ipynb b/notebooks/10-TvdP-probe_aef.ipynb new file mode 100644 index 0000000..a613460 --- /dev/null +++ b/notebooks/10-TvdP-probe_aef.ipynb @@ -0,0 +1,408 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "import matplotlib.pyplot as plt\n", + "import numpy as np\n", + "import pandas as pd\n", + "from scipy.stats import pearsonr\n", + "from sklearn.linear_model import Ridge\n", + "from sklearn.metrics import r2_score\n", + "from sklearn.preprocessing import StandardScaler\n", + "from tqdm import tqdm\n", + "\n", + "os.chdir(\"..\")\n", + "os.getcwd()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1", + "metadata": {}, + "outputs": [], + "source": [ + "aef_size = 256\n", + "unlabelled = True\n", + "\n", + "DATA_DIR = os.environ.get(\"DATA_DIR\", \"data/\")\n", + "\n", + "# aux_df = pd.read_csv(f'{DATA_DIR}/s2bms/model_ready_s2bms{\"-unlabelled-merged\" if unlabelled else \"\"}.csv')\n", + "aux_df = pd.read_csv(\n", + " f'{DATA_DIR}/s2bms/model_ready_s2bms{\"-unlabelled-merged_incl-test-aux\" if unlabelled else \"\"}.csv'\n", + ")\n", + "aef_df = pd.read_csv(\n", + " f'{DATA_DIR}/s2bms/eo/avr_aef_{aef_size}{\"_unlabelled\" if unlabelled else \"\"}.csv'\n", + ")\n", + "\n", + "import torch # only used here, to read the .pth split-index file; no tensors kept downstream\n", + "\n", + "# split_indices = torch.load(\n", + "# os.path.join(f'{DATA_DIR}/s2bms/splits/s2bms{\"_unlabelled\" if unlabelled else \"\"}_union_val_test.pth'),\n", + "# weights_only=False,\n", + "# )\n", + "split_indices = torch.load(\n", + " os.path.join(\n", + " f\"{DATA_DIR}/s2bms/splits/split_indices_s2bms+s2bms-unlabelled-20260529_2026-05-29-1438.pth\"\n", + " ),\n", + " weights_only=False,\n", + ")\n", + "aux_idx = aux_df.name_loc\n", + "aef_idx = aef_df.name_loc\n", + "\n", + "common_train_idx = pd.Series(\n", + " list(set(split_indices[\"train_indices\"]) & set(aef_idx) & set(aux_idx))\n", + ")\n", + "common_val_idx = pd.Series(list(set(split_indices[\"val_indices\"]) & set(aef_idx) & set(aux_idx)))\n", + "common_test_idx = pd.Series(list(set(split_indices[\"test_indices\"]) & set(aef_idx) & set(aux_idx)))\n", + "print(\n", + " \"train | val | test\\n\",\n", + " len(common_train_idx),\n", + " \"|\",\n", + " len(common_val_idx),\n", + " \"|\",\n", + " len(common_test_idx),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "def get_split(name_loc_set):\n", + " aef_sub = aef_df[aef_df[\"name_loc\"].isin(map(str, name_loc_set))].sort_values(\"name_loc\")\n", + " aux_sub = aux_df[aux_df[\"name_loc\"].isin(map(str, name_loc_set))].sort_values(\"name_loc\")\n", + " return aef_sub, aux_sub\n", + "\n", + "\n", + "emb_cols = [f\"emb_{i}\" for i in range(64)] # must match aef_size\n", + "aux_cols = [c for c in aux_df.columns if c.startswith(\"aux_\") and \"top\" not in c]\n", + "\n", + "aef_train_df, aux_train_df = get_split(common_train_idx)\n", + "aef_val_df, aux_val_df = get_split(common_val_idx)\n", + "aef_test_df, aux_test_df = get_split(common_test_idx)\n", + "\n", + "aef_train = aef_train_df[emb_cols].to_numpy()\n", + "aef_val = aef_val_df[emb_cols].to_numpy()\n", + "aef_test = aef_test_df[emb_cols].to_numpy()\n", + "\n", + "aux_train = aux_train_df[aux_cols].to_numpy()\n", + "aux_val = aux_val_df[aux_cols].to_numpy()\n", + "aux_test = aux_test_df[aux_cols].to_numpy()\n", + "\n", + "print(aef_train.shape, aef_val.shape, aef_test.shape)\n", + "print(aux_train.shape, aux_val.shape, aux_test.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3", + "metadata": {}, + "outputs": [], + "source": [ + "# Compute on RAW (pre-filter, 87-column) arrays -- before any keep_mask slicing\n", + "aux_std_raw = aux_train.std(axis=0)\n", + "train_nonzero_frac = (aux_train != 0).mean(axis=0)\n", + "val_nonzero_frac = (aux_val != 0).mean(axis=0)\n", + "test_nonzero_frac = (aux_test != 0).mean(axis=0)\n", + "\n", + "min_std = 1e-2\n", + "min_nonzero_frac = 0.01\n", + "\n", + "keep_mask = (\n", + " (aux_std_raw > min_std)\n", + " & (train_nonzero_frac > min_nonzero_frac)\n", + " & (val_nonzero_frac > min_nonzero_frac)\n", + " & (test_nonzero_frac > min_nonzero_frac)\n", + ")\n", + "dropped_names = [name for name, keep in zip(aux_cols, keep_mask) if not keep]\n", + "print(f\"Dropping {len(dropped_names)} low-variance/sparse aux columns: {dropped_names}\")\n", + "\n", + "kept_names = [name for name, keep in zip(aux_cols, keep_mask) if keep]\n", + "\n", + "coverage_df = pd.DataFrame(\n", + " {\n", + " \"AUX_Variable\": kept_names,\n", + " \"train_std\": aux_std_raw[keep_mask],\n", + " \"pct_nonzero_train\": train_nonzero_frac[keep_mask] * 100,\n", + " \"pct_nonzero_val\": val_nonzero_frac[keep_mask] * 100,\n", + " \"pct_nonzero_test\": test_nonzero_frac[keep_mask] * 100,\n", + " }\n", + ").sort_values(\"pct_nonzero_val\")\n", + "# print(coverage_df.head(15))\n", + "\n", + "# NOW subset, once, after keep_mask is finalized\n", + "aux_train, aux_val, aux_test = (\n", + " aux_train[:, keep_mask],\n", + " aux_val[:, keep_mask],\n", + " aux_test[:, keep_mask],\n", + ")\n", + "\n", + "# kept_names" + ] + }, + { + "cell_type": "markdown", + "id": "4", + "metadata": {}, + "source": [ + "## Linear probing " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "5", + "metadata": {}, + "outputs": [], + "source": [ + "# Standardize using TRAIN stats only\n", + "aef_scaler = StandardScaler().fit(aef_train)\n", + "X_train = aef_scaler.transform(aef_train)\n", + "X_val = aef_scaler.transform(aef_val)\n", + "X_test = aef_scaler.transform(aef_test)\n", + "\n", + "aux_scaler = StandardScaler().fit(aux_train)\n", + "y_train = aux_scaler.transform(aux_train)\n", + "y_val = aux_scaler.transform(aux_val)\n", + "y_test = aux_scaler.transform(aux_test)\n", + "\n", + "print(\"Shape of X_train and y_train:\", X_train.shape, y_train.shape)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "# Ridge regression\n", + "# alphas = [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]\n", + "alphas = [1]\n", + "\n", + "r2_scores = []\n", + "best_alphas = []\n", + "\n", + "for i in range(y_train.shape[1]):\n", + " y_target_train = y_train[:, i]\n", + " y_target_val = y_val[:, i]\n", + " y_target_test = y_test[:, i]\n", + "\n", + " # pick alpha using the spatial val split, not sklearn's internal random CV\n", + " val_r2_per_alpha = []\n", + " for a in alphas:\n", + " m = Ridge(alpha=a)\n", + " m.fit(X_train, y_target_train)\n", + " val_r2_per_alpha.append(r2_score(y_target_val, m.predict(X_val)))\n", + "\n", + " best_alpha = alphas[int(np.argmax(val_r2_per_alpha))]\n", + " best_alphas.append(best_alpha)\n", + "\n", + " final_model = Ridge(alpha=best_alpha)\n", + " final_model.fit(X_train, y_target_train)\n", + " y_pred = final_model.predict(X_test)\n", + "\n", + " r2_scores.append(r2_score(y_target_test, y_pred))\n", + "\n", + "\n", + "results_df = pd.DataFrame(\n", + " {\n", + " \"AUX_Variable\": kept_names,\n", + " \"Test_R2\": r2_scores,\n", + " \"Best_Alpha\": best_alphas,\n", + " }\n", + ")\n", + "\n", + "plot_var = \"Test_R2\"\n", + "df_sorted = results_df.sort_values(plot_var, ascending=False)\n", + "# print(df_sorted)\n", + "# df_sorted.to_csv(f'../ridge_reg_avr_aef_{aef_size}_to_aux_unlabelled.csv', index=False)\n", + "\n", + "df_sorted[\"lc_sum\"] = np.nan\n", + "df_sorted[\"lc_n_above_threshold\"] = np.nan\n", + "\n", + "dict_sum_lc = {name: sum_val for name, sum_val in zip(kept_names, aux_train.sum(0))}\n", + "\n", + "threshold_cover = 0.2\n", + "\n", + "for k, v in dict_sum_lc.items():\n", + " if \"corine\" in k:\n", + " df_sorted.loc[df_sorted[\"AUX_Variable\"] == k, \"lc_sum\"] = v\n", + " ## get number of locations with k values above threshold_cover\n", + " df_sorted.loc[df_sorted[\"AUX_Variable\"] == k, \"lc_n_above_threshold\"] = (\n", + " aux_train[:, kept_names.index(k)] > threshold_cover\n", + " ).sum()\n", + "\n", + "\n", + "fig, ax = plt.subplots(figsize=(18, 6))\n", + "ax.bar(df_sorted.AUX_Variable, df_sorted[plot_var], color=\"#4C72B0\")\n", + "ax.set_ylabel(\"Test R²\")\n", + "ax.set_ylim(-1, 1.0)\n", + "ax.set_title(\"Ridge Probe Performance per Auxiliary Variable\")\n", + "plt.xticks(rotation=90)\n", + "ax.axhline(0.5, color=\"red\", linestyle=\"--\", label=\"R² = 0.5\")\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7", + "metadata": {}, + "outputs": [], + "source": [ + "## For all columns with R2 > 0.5, val_av=True, else False. Add that to JSON and write back.\n", + "\n", + "import json\n", + "\n", + "fp_cc = f\"{DATA_DIR}/s2bms/concept_captions/v5.json\"\n", + "assert os.path.exists(fp_cc), f\"Concept captions file not found: {fp_cc}\"\n", + "with open(fp_cc, \"r\") as f:\n", + " concept_captions = json.load(f)\n", + "\n", + "for i, dict_c in enumerate(concept_captions):\n", + " aux_name = dict_c[\"col\"]\n", + " if aux_name in df_sorted[\"AUX_Variable\"].values:\n", + " r2_val = df_sorted.loc[df_sorted[\"AUX_Variable\"] == aux_name, \"Test_R2\"].values[0]\n", + " concept_captions[i][\"val_av\"] = bool(r2_val > 0.5)\n", + " else:\n", + " concept_captions[i][\"val_av\"] = False\n", + "\n", + "new_fp_cc = f\"{DATA_DIR}/s2bms/concept_captions/v6.json\"\n", + "# with open(new_fp_cc, 'w') as f:\n", + "# json.dump(concept_captions, f, indent=2)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "## Plot both test R2 and lc sum as a bar plot\n", + "\n", + "# Butterfly barplot: Test R2 (up) vs lc_sum (down, log scale) per land-cover variable\n", + "lc_df = df_sorted[df_sorted[\"lc_sum\"].notna()].sort_values(\"Test_R2\", ascending=False)\n", + "\n", + "fig, (ax_top, ax_bot) = plt.subplots(\n", + " 2,\n", + " 1,\n", + " figsize=(14, 8),\n", + " sharex=True,\n", + " gridspec_kw={\"height_ratios\": [1, 1], \"hspace\": 0.05},\n", + ")\n", + "\n", + "ax_top.bar(lc_df.AUX_Variable, lc_df.Test_R2, color=\"#4C72B0\")\n", + "ax_top.set_ylabel(\"Test R²\")\n", + "ax_top.axhline(0.5, color=\"black\", linewidth=0.8)\n", + "ax_top.axhline(0, color=\"black\", linewidth=0.8)\n", + "\n", + "ax_bot.bar(lc_df.AUX_Variable, lc_df.lc_sum, color=\"#DD8452\")\n", + "ax_bot.set_yscale(\"log\")\n", + "ax_bot.invert_yaxis()\n", + "ax_bot.set_ylabel(\"LC sum (log)\")\n", + "ax_bot.axhline(100, color=\"black\", linewidth=0.8)\n", + "\n", + "plt.xticks(rotation=90)\n", + "fig.suptitle(\"Ridge Probe Test R² vs. Land-Cover Sum (Train)\")\n", + "plt.tight_layout()\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "9", + "metadata": {}, + "outputs": [], + "source": [ + "# df_sorted[:30]\n", + "\n", + "import seaborn as sns\n", + "\n", + "col_1, col_2 = \"Test_R2\", \"rse\"\n", + "inds_nonnan = df_sorted[col_1].notna() & df_sorted[col_2].notna()\n", + "r, p = pearsonr(df_sorted.loc[inds_nonnan, col_1], df_sorted.loc[inds_nonnan, col_2])\n", + "print(f\"Pearson correlation between {col_1} and {col_2}: r={r:.4f}, p={p:.4e}\")\n", + "df_sorted[\"lc_sum_bin\"] = pd.cut(df_sorted[\"lc_sum\"], bins=10, labels=False, include_lowest=True)\n", + "sns.scatterplot(\n", + " data=df_sorted, x=col_1, y=col_2, hue=\"lc_sum_bin\", palette=\"viridis\", hue_norm=(0, 9)\n", + ")\n", + "# plt.ylim(-1.3, 1.0)\n", + "\n", + "# plt.xlim(0, 500)" + ] + }, + { + "cell_type": "markdown", + "id": "10", + "metadata": {}, + "source": [ + "## RSE" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "print(\"Shape of aux_train and aef_train:\", aux_train.shape, aef_train.shape)\n", + "\n", + "from scipy.spatial.distance import cdist, pdist\n", + "\n", + "dist_aef = cdist(X_train, X_train, metric=\"euclidean\")\n", + "upper_tri_indices = np.triu_indices(dist_aef.shape[0], k=1)\n", + "dist_aef = dist_aef[upper_tri_indices]\n", + "\n", + "dict_rse = {}\n", + "for i_col, aux_name in tqdm(enumerate(kept_names)):\n", + " dist_aux = cdist(\n", + " y_train[:, i_col].reshape(-1, 1), y_train[:, i_col].reshape(-1, 1), metric=\"euclidean\"\n", + " )\n", + " dist_aux = dist_aux[upper_tri_indices]\n", + " corr, _ = pearsonr(dist_aef, dist_aux)\n", + " dict_rse[aux_name] = corr\n", + " # print(f\"Pearson correlation between AEF and {kept_names[i_col]} distance matrices: {corr:.4f}\")\n", + "\n", + " df_sorted[\"rse\"] = df_sorted[\"AUX_Variable\"].map(dict_rse)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "aether", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.12" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}