Skip to content

Commit

Permalink
Merge pull request awst-austria#86 from daberer/feature/intra-annual-…
Browse files Browse the repository at this point in the history 
…metrics/stability-metrics

Add Stability Metrics Plots
  • Loading branch information
@daberer
daberer authored Oct 31, 2024
2 parents 650946f + 1413b4f commit d4fe535
Show file tree
Hide file tree
Showing 15 changed files with 471 additions and 150 deletions.
2 changes: 1 addition & 1 deletion src/qa4sm_reader/comparing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -141,7 +141,7 @@ def common_metrics(self) -> dict:
img_metrics = {}
for metric in img.metrics:
# hardcoded because n_obs cannot be compared. todo: exclude empty metrics (problem: the values are not loaded here)
if metric in glob.metric_groups[0] or metric in [
if metric in glob.metric_groups['common'] or metric in [
"tau", "p_tau"
]:
continue
Expand Down
122 changes: 87 additions & 35 deletions src/qa4sm_reader/globals.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@
os.path.abspath(__file__)), 'static', 'images', 'logo',
'QA4SM_logo_long.png')


# === filename template ===
ds_fn_templ = "{i}-{ds}.{var}"
ds_fn_sep = "_with_"
Expand Down Expand Up @@ -133,6 +134,16 @@ def get_status_colors():
'YlGn'], # sequential: increasing value good (n_obs, STDerr)
'qua_neutr':
get_status_colors(), # qualitative category with 2 forced colors
# Added colormaps for slope metrics
'div_slopeBIAS': matplotlib.colormaps[
'RdBu_r'
], # diverging colormap for slopeBIAS
'div_slopeR': matplotlib.colormaps[
'PiYG'
], # diverging colormap for slopeR
'div_slopeURMSD': matplotlib.colormaps[
'PuOr'
] # diverging colormap for slopeURMSD
}

_colormaps = { # from /qa4sm/validator/validation/graphics.py
Expand All @@ -155,6 +166,9 @@ def get_status_colors():
'err_std': _cclasses['seq_worse'],
'beta': _cclasses['div_neutr'],
'status': _cclasses['qua_neutr'],
'slopeR': _cclasses['div_slopeR'],
'slopeURMSD': _cclasses['div_slopeURMSD'],
'slopeBIAS': _cclasses['div_slopeBIAS'],
}

# Colorbars for difference plots
Expand All @@ -180,34 +194,42 @@ def get_status_colors():

# METRICS AND VARIABLES DEFINITIONS
# =====================================================
# 0=common metrics, 2=paired metrics (2 datasets), 3=triple metrics (TC, 3 datasets)
# common metrics, pairwise metrics (2 datasets), triple metrics (TC, 3 datasets), pairwise stability metrics(2 datasets)
metric_groups = {
0: ['n_obs'],
2: [
'common': ['n_obs'],
'pairwise': [
'R', 'p_R', 'rho', 'p_rho', 'RMSD', 'BIAS', 'urmsd', 'mse', 'mse_corr',
'mse_bias', 'mse_var', 'RSS', 'tau', 'p_tau', 'status'
'mse_bias', 'mse_var', 'RSS', 'tau', 'p_tau', 'status'
],
3: ['snr', 'err_std', 'beta', 'status']
'triple': ['snr', 'err_std', 'beta', 'status'],
'pairwise_stability': ['slopeURMSD', 'slopeR', 'slopeBIAS']
}

def get_metric_format(group, metric_dict):
# metric groups 'pairwise and 'pairwise_stability should be handled the same
if group == "pairwise_stability":
group = "pairwise"
return metric_dict.get(group)

# === variable template ===
# how the metric is separated from the rest
var_name_metric_sep = {
0: "{metric}",
2: "{metric}_between_",
3: "{metric}_{mds_id:d}-{mds}_between_"
'common': "{metric}",
'pairwise': "{metric}_between_",
'triple': "{metric}_{mds_id:d}-{mds}_between_"
}

var_name_CI = {
0: "{metric}_ci_{bound}_between_",
2: "{metric}_ci_{bound}_between_",
3: "{metric}_ci_{bound}_{mds_id:d}-{mds}_between_"
'common': "{metric}_ci_{bound}_between_",
'pairwise': "{metric}_ci_{bound}_between_",
'triple': "{metric}_ci_{bound}_{mds_id:d}-{mds}_between_"
}
# how two datasets are separated, ids must be marked as numbers with :d!
var_name_ds_sep = {
0: None,
2: "{ref_id:d}-{ref_ds}_and_{sat_id0:d}-{sat_ds0}",
3:
"{ref_id:d}-{ref_ds}_and_{sat_id0:d}-{sat_ds0}_and_{sat_id1:d}-{sat_ds1}"
'common': None,
'pairwise': "{ref_id:d}-{ref_ds}_and_{sat_id0:d}-{sat_ds0}",
'triple':
"{ref_id:d}-{ref_ds}_and_{sat_id0:d}-{sat_ds0}_and_{sat_id1:d}-{sat_ds1}",
}

# === metadata templates ===
Expand All @@ -221,14 +243,11 @@ def get_status_colors():

# format should have (metric, ds, ref, other ds)
_variable_pretty_name = {
0: "{}",
2: "{}\nof {}\nwith {} as reference",
3: "{} of {} \n against {}, {}"
'common': "{}",
'pairwise': "{}\nof {}\nwith {} as reference",
'triple': "{} of {} \n against {}, {}"
}

# check if every metric has a colormap
for group in metric_groups.keys():
assert all([m in _colormaps.keys() for m in metric_groups[group]])

# Value ranges of metrics, either absolute values, or a quantile between 0 and 1
_metric_value_ranges = { # from /qa4sm/validator/validation/graphics.py
Expand All @@ -251,6 +270,9 @@ def get_status_colors():
'err_std': [None, None],
'beta': [None, None],
'status': [-1, len(status)-2],
'slopeR': [None, None],
'slopeURMSD': [None, None],
'slopeBIAS': [None, None],
}
# mask values out of range
_metric_mask_range = {
Expand Down Expand Up @@ -282,11 +304,14 @@ def get_status_colors():
'err_std': ' in {}',
'beta': ' in {}',
'status': '',
'slopeR': '',
'slopeURMSD': ' in {}',
'slopeBIAS': ' in {}',
}


# units for all datasets
def get_metric_units(dataset, raise_error=False):
def get_metric_units(dataset, metric=None, raise_error=False):
# function to get m.u. with possibility to raise error
_metric_units = { # from /qa4sm/validator/validation/graphics.py
'ISMN': 'm³/m³',
Expand All @@ -311,22 +336,24 @@ def get_metric_units(dataset, raise_error=False):
'SMOS_SBPCA': 'm³/m³',
}

try:
return _metric_units[dataset]
unit = _metric_units.get(dataset)

except KeyError:
if unit is None:
if raise_error:
raise KeyError(
f"The dataset {dataset} has not been specified in {__name__}")

raise KeyError(f"The dataset '{dataset}' has not been specified in {__name__}.")
else:
warnings.warn(
f"The dataset {dataset} has not been specified in {__name__}. "
f"Set 'raise_error' to True to raise an exception for this case."
f"The dataset '{dataset}' has not been specified in {__name__}. "
"Set 'raise_error' to True to raise an exception for this case.",
UserWarning
)

return "n.a."

if metric in STABILITY_METRICS:
unit += ' per decade'

return unit

COMMON_METRICS = {
'R': 'Pearson\'s r',
'p_R': 'Pearson\'s r p-value',
Expand All @@ -352,7 +379,7 @@ def get_metric_units(dataset, raise_error=False):
'p_rho': 'Spearman\'s ρ p-value',
'tau': 'Kendall rank correlation',
'p_tau': 'Kendall tau p-value',
'status': 'Validation errors'
'status': 'Validation errors',
}

QA4SM_EXCLUSIVE_METRICS = {
Expand All @@ -363,7 +390,13 @@ def get_metric_units(dataset, raise_error=False):
'status': '# status',
}

_metric_name = {**COMMON_METRICS, **READER_EXCLUSIVE_METRICS, **TC_METRICS}
STABILITY_METRICS = {
'slopeR' : 'Theil-Sen slope of R',
'slopeURMSD' : 'Theil-Sen slope of urmsd',
'slopeBIAS' : 'Theil-Sen slope of BIAS'
}

_metric_name = {**COMMON_METRICS, **READER_EXCLUSIVE_METRICS, **TC_METRICS, **STABILITY_METRICS}

METRICS = {**COMMON_METRICS, **QA4SM_EXCLUSIVE_METRICS}

Expand Down Expand Up @@ -426,6 +459,7 @@ def get_metric_units(dataset, raise_error=False):
}
}


# BACKUPS
# =====================================================
# to fallback to in case the dataset attributes in the .nc file are
Expand Down Expand Up @@ -741,6 +775,9 @@ def get_resolution_info(dataset, raise_error=False):
'tau',
'p_tau',
'status',
'slopeR',
'slopeURMSD',
'slopeBIAS',
]

METRIC_TEMPLATE = '_between_{ds1}_and_{ds2}'
Expand Down Expand Up @@ -783,9 +820,26 @@ def get_resolution_info(dataset, raise_error=False):
"Oct": [[10, 1], [10, 31]],
"Nov": [[11, 1], [11, 30]],
"Dec": [[12, 1], [12, 31]],
},
#Fix as TemporalSubWindowsCreator checks pre-defined tsw on init
"stability":{
}
}

def add_annual_subwindows(years):
# Loops through each year and create a subwindow for the entire year
annual_subwindows = {}
for year in years:
annual_subwindows[str(year)] = [
[year, 1, 1],
[year, 12, 31]
]
TEMPORAL_SUB_WINDOWS["custom"] = annual_subwindows


#years = [2008, 2009, 2010]
#add_annual_subwindows(years)

CLUSTERED_BOX_PLOT_STYLE = {
'fig_params': {
'title_fontsize': 20,
Expand All @@ -806,8 +860,6 @@ def get_resolution_info(dataset, raise_error=False):

CLUSTERED_BOX_PLOT_OUTDIR = 'comparison_boxplots'



# netCDF transcription related settings
# =====================================================
OLD_NCFILE_SUFFIX = '.old'
Expand Down
Loading

0 comments on commit d4fe535

Please sign in to comment.