@article{Sheikholeslami-Razavi-2019-What-should,
title = "What should we do when a model crashes? Recommendations for global sensitivity analysis of Earth and environmental systems models",
author = "Sheikholeslami, Razi and
Razavi, Saman and
Haghnegahdar, Amin",
journal = "Geoscientific Model Development, Volume 12, Issue 10",
volume = "12",
number = "10",
year = "2019",
publisher = "Copernicus GmbH",
url = "https://gwf-uwaterloo.github.io/gwf-publications/G19-159001",
doi = "10.5194/gmd-12-4275-2019",
pages = "4275--4296",
abstract = "Abstract. Complex, software-intensive, technically advanced, and computationally demanding models, presumably with ever-growing realism and fidelity, have been widely used to simulate and predict the dynamics of the Earth and environmental systems. The parameter-induced simulation crash (failure) problem is typical across most of these models despite considerable efforts that modellers have directed at model development and implementation over the last few decades. A simulation failure mainly occurs due to the violation of numerical stability conditions, non-robust numerical implementations, or errors in programming. However, the existing sampling-based analysis techniques such as global sensitivity analysis (GSA) methods, which require running these models under many configurations of parameter values, are ill equipped to effectively deal with model failures. To tackle this problem, we propose a new approach that allows users to cope with failed designs (samples) when performing GSA without rerunning the entire experiment. This approach deems model crashes as missing data and uses strategies such as median substitution, single nearest-neighbor, or response surface modeling to fill in for model crashes. We test the proposed approach on a 10-parameter HBV-SASK (Hydrologiska Byr{\aa}ns Vattenbalansavdelning modified by the second author for educational purposes) rainfall{--}runoff model and a 111-parameter Mod{\'e}lisation Environmentale{--}Surface et Hydrologie (MESH) land surface{--}hydrology model. Our results show that response surface modeling is a superior strategy, out of the data-filling strategies tested, and can comply with the dimensionality of the model, sample size, and the ratio of the number of failures to the sample size. Further, we conduct a {``}failure analysis{''} and discuss some possible causes of the MESH model failure that can be used for future model improvement.",
}
<?xml version="1.0" encoding="UTF-8"?>
<modsCollection xmlns="http://www.loc.gov/mods/v3">
<mods ID="Sheikholeslami-Razavi-2019-What-should">
<titleInfo>
<title>What should we do when a model crashes? Recommendations for global sensitivity analysis of Earth and environmental systems models</title>
</titleInfo>
<name type="personal">
<namePart type="given">Razi</namePart>
<namePart type="family">Sheikholeslami</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Saman</namePart>
<namePart type="family">Razavi</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<name type="personal">
<namePart type="given">Amin</namePart>
<namePart type="family">Haghnegahdar</namePart>
<role>
<roleTerm authority="marcrelator" type="text">author</roleTerm>
</role>
</name>
<originInfo>
<dateIssued>2019</dateIssued>
</originInfo>
<typeOfResource>text</typeOfResource>
<genre authority="bibutilsgt">journal article</genre>
<relatedItem type="host">
<titleInfo>
<title>Geoscientific Model Development, Volume 12, Issue 10</title>
</titleInfo>
<originInfo>
<issuance>continuing</issuance>
<publisher>Copernicus GmbH</publisher>
</originInfo>
<genre authority="marcgt">periodical</genre>
<genre authority="bibutilsgt">academic journal</genre>
</relatedItem>
<abstract>Abstract. Complex, software-intensive, technically advanced, and computationally demanding models, presumably with ever-growing realism and fidelity, have been widely used to simulate and predict the dynamics of the Earth and environmental systems. The parameter-induced simulation crash (failure) problem is typical across most of these models despite considerable efforts that modellers have directed at model development and implementation over the last few decades. A simulation failure mainly occurs due to the violation of numerical stability conditions, non-robust numerical implementations, or errors in programming. However, the existing sampling-based analysis techniques such as global sensitivity analysis (GSA) methods, which require running these models under many configurations of parameter values, are ill equipped to effectively deal with model failures. To tackle this problem, we propose a new approach that allows users to cope with failed designs (samples) when performing GSA without rerunning the entire experiment. This approach deems model crashes as missing data and uses strategies such as median substitution, single nearest-neighbor, or response surface modeling to fill in for model crashes. We test the proposed approach on a 10-parameter HBV-SASK (Hydrologiska Byråns Vattenbalansavdelning modified by the second author for educational purposes) rainfall–runoff model and a 111-parameter Modélisation Environmentale–Surface et Hydrologie (MESH) land surface–hydrology model. Our results show that response surface modeling is a superior strategy, out of the data-filling strategies tested, and can comply with the dimensionality of the model, sample size, and the ratio of the number of failures to the sample size. Further, we conduct a “failure analysis” and discuss some possible causes of the MESH model failure that can be used for future model improvement.</abstract>
<identifier type="citekey">Sheikholeslami-Razavi-2019-What-should</identifier>
<identifier type="doi">10.5194/gmd-12-4275-2019</identifier>
<location>
<url>https://gwf-uwaterloo.github.io/gwf-publications/G19-159001</url>
</location>
<part>
<date>2019</date>
<detail type="volume"><number>12</number></detail>
<detail type="issue"><number>10</number></detail>
<extent unit="page">
<start>4275</start>
<end>4296</end>
</extent>
</part>
</mods>
</modsCollection>
%0 Journal Article
%T What should we do when a model crashes? Recommendations for global sensitivity analysis of Earth and environmental systems models
%A Sheikholeslami, Razi
%A Razavi, Saman
%A Haghnegahdar, Amin
%J Geoscientific Model Development, Volume 12, Issue 10
%D 2019
%V 12
%N 10
%I Copernicus GmbH
%F Sheikholeslami-Razavi-2019-What-should
%X Abstract. Complex, software-intensive, technically advanced, and computationally demanding models, presumably with ever-growing realism and fidelity, have been widely used to simulate and predict the dynamics of the Earth and environmental systems. The parameter-induced simulation crash (failure) problem is typical across most of these models despite considerable efforts that modellers have directed at model development and implementation over the last few decades. A simulation failure mainly occurs due to the violation of numerical stability conditions, non-robust numerical implementations, or errors in programming. However, the existing sampling-based analysis techniques such as global sensitivity analysis (GSA) methods, which require running these models under many configurations of parameter values, are ill equipped to effectively deal with model failures. To tackle this problem, we propose a new approach that allows users to cope with failed designs (samples) when performing GSA without rerunning the entire experiment. This approach deems model crashes as missing data and uses strategies such as median substitution, single nearest-neighbor, or response surface modeling to fill in for model crashes. We test the proposed approach on a 10-parameter HBV-SASK (Hydrologiska Byråns Vattenbalansavdelning modified by the second author for educational purposes) rainfall–runoff model and a 111-parameter Modélisation Environmentale–Surface et Hydrologie (MESH) land surface–hydrology model. Our results show that response surface modeling is a superior strategy, out of the data-filling strategies tested, and can comply with the dimensionality of the model, sample size, and the ratio of the number of failures to the sample size. Further, we conduct a “failure analysis” and discuss some possible causes of the MESH model failure that can be used for future model improvement.
%R 10.5194/gmd-12-4275-2019
%U https://gwf-uwaterloo.github.io/gwf-publications/G19-159001
%U https://doi.org/10.5194/gmd-12-4275-2019
%P 4275-4296
Markdown (Informal)
[What should we do when a model crashes? Recommendations for global sensitivity analysis of Earth and environmental systems models](https://gwf-uwaterloo.github.io/gwf-publications/G19-159001) (Sheikholeslami et al., GWF 2019)
ACL
- Razi Sheikholeslami, Saman Razavi, and Amin Haghnegahdar. 2019. What should we do when a model crashes? Recommendations for global sensitivity analysis of Earth and environmental systems models. Geoscientific Model Development, Volume 12, Issue 10, 12(10):4275–4296.
