From 4cfeb4ff5fae97fb9546d03983668de793ff4da7 Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Fri, 5 Mar 2021 13:17:38 +0100 Subject: [PATCH 01/15] Add simple plotting tool --- setup.py | 1 + src/flownet/utils/plot_results.py | 226 ++++++++++++++++++++++++++++++ 2 files changed, 227 insertions(+) create mode 100644 src/flownet/utils/plot_results.py diff --git a/setup.py b/setup.py index cb6f414e8..4115b0317 100644 --- a/setup.py +++ b/setup.py @@ -66,6 +66,7 @@ "flownet_run_flow=flownet.ert._flow_job:run_flow", "flownet_save_iteration_parameters=flownet.ahm:save_iteration_parameters", "flownet_save_iteration_analytics=flownet.ahm:save_iteration_analytics", + "flownet_plot_results=flownet.utils.plot_results:main", ], }, zip_safe=False, diff --git a/src/flownet/utils/plot_results.py b/src/flownet/utils/plot_results.py new file mode 100644 index 000000000..208f38ad4 --- /dev/null +++ b/src/flownet/utils/plot_results.py @@ -0,0 +1,226 @@ +import argparse +import pathlib +import re +from typing import Optional, List, Tuple + +import matplotlib +import matplotlib.pyplot as plt +import pandas as pd +from fmu import ensemble +from ecl.summary import EclSum + +matplotlib.use("Agg") + + +def plot_ensembles( + vector: str, + ensembles_data: List[pd.DataFrame], + color: Tuple[float, float, float, float], +): + """Function to plot a list of ensembles. + + Args: + vector: Name of the vector to plot + ensembles_data: List of dataframes with ensemble data + color: Color to use when plotting the ensemble + """ + for ensemble_data in ensembles_data: + ensemble_data = ( + remove_duplicates(ensemble_data[["DATE", "REAL", vector]]) + .pivot(index="DATE", columns="REAL", values=vector) + .dropna() + ) + + plt.plot( + ensemble_data.index, + ensemble_data.values, + color=color, + ) + + +def plot( + vector: str, + prior_data: list, + posterior_data: list, + reference_simulation: Optional[EclSum], + plot_settings: dict, +): + """Main plotting function that generate builds up a single plot build up + from potentially multiple ensembles and other data. + + Args: + vector: Name of the vector to plot. + prior_data: List of prior ensemble data DataFrames. + posterior_data: List of posterior ensemble data DataFrames. + reference_simulation: EclSum object for the reference simulation. + plot_settings: Settings dictionary for the plots. + + """ + plt.figure() # (figsize=[16, 8]) + + if len(prior_data): + plot_ensembles(vector, prior_data, (0.5, 0.5, 0.5, 0.1)) + + if len(posterior_data): + plot_ensembles(vector, posterior_data, (0.0, 0.1, 0.6, 0.1)) + + if reference_simulation: + plt.plot( + reference_simulation.dates, + reference_simulation.numpy_vector(vector), + color=(1.0, 0.0, 0.0, 1.0), + ) + + plt.ylim([plot_settings["ymin"], plot_settings["ymax"]]) + plt.xlabel("date") + plt.ylabel(vector + " [" + plot_settings["units"] + "]") + plt.savefig(re.sub(r"[^\w\-_\. ]", "_", vector), dpi=300) + + +def remove_duplicates(df: pd.DataFrame) -> pd.DataFrame: + """Remove duplicates for the combination or DATE and REAL. + + Args: + df: Input pandas DataFrame with columns: [DATE, REAL, VECTOR1, VECTOR2, ..., VECTOR_N] + + Returns: + A cleaned dataframe + + """ + return df[~df[["DATE", "REAL"]].apply(frozenset, axis=1).duplicated()] + + +def check_args(args): + """Helper function to verify input arguments. + + Returns: + Nothing + + Raises: + ValueError in case the input arguments are inconsistent. + + """ + if not (len(args.ymin) == 1 or len(args.ymin) == len(args.vector)): + raise ValueError( + f"You should either supply a single minimum y-value or as many as you have vectors ({len(args.vectors)}." + ) + + if not (len(args.ymax) == 1 or len(args.ymax) == len(args.vector)): + raise ValueError( + f"You should either supply a single maximum y-value or as many as you have vectors ({len(args.vectors)}." + ) + + if not (len(args.units) == 1 or len(args.units) == len(args.vector)): + raise ValueError( + f"You should either supply a units label or as many as you have vectors ({len(args.vectors)}." + ) + + if len(args.prior) and len(args.posterior) and not args.reference_simulation: + raise ValueError( + "There is no prior, posterior or reference simulation to plot. Supply at least something for me to plot." + ) + + +def main(): + """Main function for the plotting of simulations results from FlowNet. + + Return: + Nothing + """ + + parser = argparse.ArgumentParser( + prog=("Simple tool to plot FlowNet ensembles simulation results.") + ) + parser.add_argument( + "vectors", + type=str, + nargs="+", + help="One or more vectors to plot separated by spaces. Example: WOPR:WELL1 FOPR", + ) + parser.add_argument( + "-prior", + type=str, + nargs="+", + help="One or more paths to prior ensembles separated by a space." + "The path should include a '%d' which indicates the realization number.", + ) + parser.add_argument( + "-posterior", + type=str, + nargs="+", + help="One or more paths to posterior ensembles separated by a space." + "The path should include a '%d' which indicates the realization number.", + ) + parser.add_argument( + "-reference_simulation", + "-r", + type=pathlib.Path, + help="Path to the reference simulation case.", + ) + parser.add_argument( + "-ymin", + type=float, + default=0, + nargs="+", + help="One or #vectors minimum y values.", + ) + parser.add_argument( + "-ymax", + type=float, + default=1000, + nargs="+", + help="One or #vectors maximum y values.", + ) + parser.add_argument( + "-units", + type=str, + default="Cows/Lightyear", + nargs="+", + help="One or #vectors unit labels.", + ) + args = parser.parse_args() + + check_args(args) + + prior_data: list = [] + for prior in args.prior: + prior_data.append( + ensemble.ScratchEnsemble( + "flownet_ensemble", + paths=prior.replace("%d", "*"), + ).get_smry(column_keys=args.vectors) + ) + + posterior_data: list = [] + for posterior in args.posterior: + posterior_data.append( + ensemble.ScratchEnsemble( + "flownet_ensemble", paths=posterior.replace("%d", "*") + ).get_smry(column_keys=args.vectors) + ) + + reference_eclsum = EclSum(str(args.reference_simulation.with_suffix(".UNSMRY"))) + + for i, vector in enumerate(args.vectors): + + plot_settings = { + "ymin": args.ymin[0] if len(args.ymin) == 1 else args.ymin[i], + "ymax": args.ymax[0] if len(args.ymax) == 1 else args.ymax[i], + "units": args.units[0] if len(args.units) == 1 else args.units[i], + } + + print(f"Plotting {vector}...", end=" ", flush=True) + + plot( + vector, + prior_data, + posterior_data, + reference_eclsum, + plot_settings, + ) + + print("[Done]", flush=True) + + +if __name__ == "__main__": + main() From a8b437eafd15566a4a968352c13c1f04ed5827f1 Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Fri, 5 Mar 2021 14:31:26 +0100 Subject: [PATCH 02/15] Further updates --- src/flownet/utils/plot_results.py | 80 ++++++++++++++++++++++++++----- 1 file changed, 69 insertions(+), 11 deletions(-) diff --git a/src/flownet/utils/plot_results.py b/src/flownet/utils/plot_results.py index 208f38ad4..2df8021ad 100644 --- a/src/flownet/utils/plot_results.py +++ b/src/flownet/utils/plot_results.py @@ -1,7 +1,7 @@ import argparse import pathlib import re -from typing import Optional, List, Tuple +from typing import Optional, List import matplotlib import matplotlib.pyplot as plt @@ -13,28 +13,47 @@ def plot_ensembles( + plot_type: str, vector: str, ensembles_data: List[pd.DataFrame], - color: Tuple[float, float, float, float], + plot_settings: dict, ): """Function to plot a list of ensembles. Args: + plot_type: prior or posterior vector: Name of the vector to plot ensembles_data: List of dataframes with ensemble data - color: Color to use when plotting the ensemble + plot_settings: Settings dictionary for the plots. + + Returns: + Nothing + + Raises: + Value error if incorrect plot type. + """ - for ensemble_data in ensembles_data: + if not plot_type in ("prior", "posterior"): + raise ValueError("Plot type should be either prior or posterior.") + + for i, ensemble_data in enumerate(ensembles_data): ensemble_data = ( remove_duplicates(ensemble_data[["DATE", "REAL", vector]]) .pivot(index="DATE", columns="REAL", values=vector) .dropna() ) + color = ( + plot_settings[f"{plot_type}_colors"][0] + if len(plot_settings[f"{plot_type}_colors"][0]) == 1 + else plot_settings[f"{plot_type}_colors"][i] + ) + plt.plot( ensemble_data.index, ensemble_data.values, color=color, + alpha=0.1, ) @@ -59,16 +78,17 @@ def plot( plt.figure() # (figsize=[16, 8]) if len(prior_data): - plot_ensembles(vector, prior_data, (0.5, 0.5, 0.5, 0.1)) + plot_ensembles("prior", vector, prior_data, plot_settings) if len(posterior_data): - plot_ensembles(vector, posterior_data, (0.0, 0.1, 0.6, 0.1)) + plot_ensembles("posterior", vector, posterior_data, plot_settings) if reference_simulation: plt.plot( reference_simulation.dates, reference_simulation.numpy_vector(vector), - color=(1.0, 0.0, 0.0, 1.0), + color=plot_settings["reference_simulation_color"], + alpha=1, ) plt.ylim([plot_settings["ymin"], plot_settings["ymax"]]) @@ -100,19 +120,19 @@ def check_args(args): ValueError in case the input arguments are inconsistent. """ - if not (len(args.ymin) == 1 or len(args.ymin) == len(args.vector)): + if not (len(args.ymin) == 1 or len(args.ymin) == len(args.vectors)): raise ValueError( f"You should either supply a single minimum y-value or as many as you have vectors ({len(args.vectors)}." ) - if not (len(args.ymax) == 1 or len(args.ymax) == len(args.vector)): + if not (len(args.ymax) == 1 or len(args.ymax) == len(args.vectors)): raise ValueError( f"You should either supply a single maximum y-value or as many as you have vectors ({len(args.vectors)}." ) - if not (len(args.units) == 1 or len(args.units) == len(args.vector)): + if not (len(args.units) == 1 or len(args.units) == len(args.vectors)): raise ValueError( - f"You should either supply a units label or as many as you have vectors ({len(args.vectors)}." + f"You should either supply a single units label or as many as you have vectors ({len(args.vectors)}." ) if len(args.prior) and len(args.posterior) and not args.reference_simulation: @@ -120,6 +140,21 @@ def check_args(args): "There is no prior, posterior or reference simulation to plot. Supply at least something for me to plot." ) + if not (len(args.prior_colors) == 1 or len(args.prior_colors) == len(args.prior)): + raise ValueError( + "You should either supply a single prior color or as " + f"many as you have prior distributions ({len(args.prior)}." + ) + + if not ( + len(args.posterior_colors) == 1 + or len(args.posterior_colors) == len(args.posterior) + ): + raise ValueError( + "You should either supply a single posterior color or as " + f"many as you have posterior distributions ({len(args.posterior)}." + ) + def main(): """Main function for the plotting of simulations results from FlowNet. @@ -178,6 +213,26 @@ def main(): nargs="+", help="One or #vectors unit labels.", ) + parser.add_argument( + "-prior_colors", + type=str, + default=["gray"], + nargs="+", + help="One or #prior ensembles colors.", + ) + parser.add_argument( + "-posterior_colors", + type=str, + default=["blue"], + nargs="+", + help="One or #posterior ensembles colors.", + ) + parser.add_argument( + "-reference_simulation_color", + type=str, + default="red", + help="The reference simulation color.", + ) args = parser.parse_args() check_args(args) @@ -207,6 +262,9 @@ def main(): "ymin": args.ymin[0] if len(args.ymin) == 1 else args.ymin[i], "ymax": args.ymax[0] if len(args.ymax) == 1 else args.ymax[i], "units": args.units[0] if len(args.units) == 1 else args.units[i], + "prior_colors": args.prior_colors, + "posterior_colors": args.posterior_colors, + "reference_simulation_color": args.reference_simulation_color, } print(f"Plotting {vector}...", end=" ", flush=True) From 429419258de4cdd1bb2fcea11d480f24e718185e Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Fri, 5 Mar 2021 19:27:19 +0100 Subject: [PATCH 03/15] Further updates --- src/flownet/utils/plot_results.py | 83 ++++++++++++++++++++++++------- 1 file changed, 65 insertions(+), 18 deletions(-) diff --git a/src/flownet/utils/plot_results.py b/src/flownet/utils/plot_results.py index 2df8021ad..6e0359146 100644 --- a/src/flownet/utils/plot_results.py +++ b/src/flownet/utils/plot_results.py @@ -1,6 +1,7 @@ import argparse import pathlib import re +from datetime import datetime from typing import Optional, List import matplotlib @@ -13,7 +14,7 @@ def plot_ensembles( - plot_type: str, + ensemble_type: str, vector: str, ensembles_data: List[pd.DataFrame], plot_settings: dict, @@ -21,7 +22,7 @@ def plot_ensembles( """Function to plot a list of ensembles. Args: - plot_type: prior or posterior + ensemble_type: prior or posterior vector: Name of the vector to plot ensembles_data: List of dataframes with ensemble data plot_settings: Settings dictionary for the plots. @@ -33,10 +34,11 @@ def plot_ensembles( Value error if incorrect plot type. """ - if not plot_type in ("prior", "posterior"): + if not ensemble_type in ("prior", "posterior"): raise ValueError("Plot type should be either prior or posterior.") for i, ensemble_data in enumerate(ensembles_data): + ensemble_data = ( remove_duplicates(ensemble_data[["DATE", "REAL", vector]]) .pivot(index="DATE", columns="REAL", values=vector) @@ -44,16 +46,22 @@ def plot_ensembles( ) color = ( - plot_settings[f"{plot_type}_colors"][0] - if len(plot_settings[f"{plot_type}_colors"][0]) == 1 - else plot_settings[f"{plot_type}_colors"][i] + plot_settings[f"{ensemble_type}_colors"][0] + if len(plot_settings[f"{ensemble_type}_colors"]) == 1 + else plot_settings[f"{ensemble_type}_colors"][i] + ) + alpha = ( + plot_settings[f"{ensemble_type}_alphas"][0] + if len(plot_settings[f"{ensemble_type}_alphas"]) == 1 + else plot_settings[f"{ensemble_type}_alphas"][i] ) plt.plot( ensemble_data.index, ensemble_data.values, color=color, - alpha=0.1, + alpha=alpha, + linestyle="solid", ) @@ -91,6 +99,9 @@ def plot( alpha=1, ) + if plot_settings["vertical_line"]: + plt.axvline(x=plot_settings["vertical_line"], color="k", linestyle="--") + plt.ylim([plot_settings["ymin"], plot_settings["ymax"]]) plt.xlabel("date") plt.ylabel(vector + " [" + plot_settings["units"] + "]") @@ -213,6 +224,20 @@ def main(): nargs="+", help="One or #vectors unit labels.", ) + parser.add_argument( + "-prior_alphas", + type=float, + default=[0.1], + nargs="+", + help="One or #prior ensembles alpha (transparency) values.", + ) + parser.add_argument( + "-posterior_alphas", + type=float, + default=[0.1], + nargs="+", + help="One or #posterior ensembles alpha (transparency) values.", + ) parser.add_argument( "-prior_colors", type=str, @@ -233,26 +258,45 @@ def main(): default="red", help="The reference simulation color.", ) + parser.add_argument( + "-vertical_line", + type=lambda s: datetime.strptime(s, "%Y-%m-%d"), + default=None, + help="The reference simulation color.", + ) args = parser.parse_args() check_args(args) prior_data: list = [] for prior in args.prior: - prior_data.append( - ensemble.ScratchEnsemble( - "flownet_ensemble", - paths=prior.replace("%d", "*"), - ).get_smry(column_keys=args.vectors) - ) + + df_data = ensemble.ScratchEnsemble( + "flownet_ensemble", + paths=prior.replace("%d", "*"), + ).get_smry(column_keys=args.vectors) + + df_data_sorted = df_data.sort_values("DATE") + df_realizations = df_data_sorted[ + df_data_sorted["DATE"] == df_data_sorted.values[-1][0] + ]["REAL"] + + prior_data.append(df_data.merge(df_realizations, how="inner")) posterior_data: list = [] for posterior in args.posterior: - posterior_data.append( - ensemble.ScratchEnsemble( - "flownet_ensemble", paths=posterior.replace("%d", "*") - ).get_smry(column_keys=args.vectors) - ) + + df_data = ensemble.ScratchEnsemble( + "flownet_ensemble", + paths=posterior.replace("%d", "*"), + ).get_smry(column_keys=args.vectors) + + df_data_sorted = df_data.sort_values("DATE") + df_realizations = df_data_sorted[ + df_data_sorted["DATE"] == df_data_sorted.values[-1][0] + ]["REAL"] + + posterior_data.append(df_data.merge(df_realizations, how="inner")) reference_eclsum = EclSum(str(args.reference_simulation.with_suffix(".UNSMRY"))) @@ -262,9 +306,12 @@ def main(): "ymin": args.ymin[0] if len(args.ymin) == 1 else args.ymin[i], "ymax": args.ymax[0] if len(args.ymax) == 1 else args.ymax[i], "units": args.units[0] if len(args.units) == 1 else args.units[i], + "prior_alphas": args.prior_alphas, + "posterior_alphas": args.posterior_alphas, "prior_colors": args.prior_colors, "posterior_colors": args.posterior_colors, "reference_simulation_color": args.reference_simulation_color, + "vertical_line": args.vertical_line, } print(f"Plotting {vector}...", end=" ", flush=True) From 3a421e1e9793bca7c02139798845805114161ec1 Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Fri, 5 Mar 2021 19:41:41 +0100 Subject: [PATCH 04/15] Put build_ensemble_df_list --- src/flownet/utils/plot_results.py | 52 +++++++++++++------------------ 1 file changed, 22 insertions(+), 30 deletions(-) diff --git a/src/flownet/utils/plot_results.py b/src/flownet/utils/plot_results.py index 6e0359146..50397ad06 100644 --- a/src/flownet/utils/plot_results.py +++ b/src/flownet/utils/plot_results.py @@ -167,6 +167,26 @@ def check_args(args): ) +def build_ensemble_df_list(ensemble_data, vectors): + data: list = [] + + for prior in ensemble_data: + + df_data = ensemble.ScratchEnsemble( + "flownet_ensemble", + paths=prior.replace("%d", "*"), + ).get_smry(column_keys=vectors) + + df_data_sorted = df_data.sort_values("DATE") + df_realizations = df_data_sorted[ + df_data_sorted["DATE"] == df_data_sorted.values[-1][0] + ]["REAL"] + + data.append(df_data.merge(df_realizations, how="inner")) + + return data + + def main(): """Main function for the plotting of simulations results from FlowNet. @@ -268,36 +288,8 @@ def main(): check_args(args) - prior_data: list = [] - for prior in args.prior: - - df_data = ensemble.ScratchEnsemble( - "flownet_ensemble", - paths=prior.replace("%d", "*"), - ).get_smry(column_keys=args.vectors) - - df_data_sorted = df_data.sort_values("DATE") - df_realizations = df_data_sorted[ - df_data_sorted["DATE"] == df_data_sorted.values[-1][0] - ]["REAL"] - - prior_data.append(df_data.merge(df_realizations, how="inner")) - - posterior_data: list = [] - for posterior in args.posterior: - - df_data = ensemble.ScratchEnsemble( - "flownet_ensemble", - paths=posterior.replace("%d", "*"), - ).get_smry(column_keys=args.vectors) - - df_data_sorted = df_data.sort_values("DATE") - df_realizations = df_data_sorted[ - df_data_sorted["DATE"] == df_data_sorted.values[-1][0] - ]["REAL"] - - posterior_data.append(df_data.merge(df_realizations, how="inner")) - + prior_data = build_ensemble_df_list(args.prior, args.vectors) + posterior_data = build_ensemble_df_list(args.posterior, args.vectors) reference_eclsum = EclSum(str(args.reference_simulation.with_suffix(".UNSMRY"))) for i, vector in enumerate(args.vectors): From 3fe64aa710538603030afa9eef9120ad2e9c906d Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Fri, 5 Mar 2021 19:45:40 +0100 Subject: [PATCH 05/15] Add docstring --- src/flownet/utils/plot_results.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/src/flownet/utils/plot_results.py b/src/flownet/utils/plot_results.py index 50397ad06..51083c345 100644 --- a/src/flownet/utils/plot_results.py +++ b/src/flownet/utils/plot_results.py @@ -167,10 +167,22 @@ def check_args(args): ) -def build_ensemble_df_list(ensemble_data, vectors): +def build_ensemble_df_list( + ensemble_paths: List[str], vectors: List[str] +) -> List[pd.DataFrame]: + """Helper function to read and prepare ensemble data. + + Args: + ensemble_paths: The ensemble paths to retrieve data from + vectors: List of vector to extract + + Returns: + List of ensemble dataframe with required data to create plots. + + """ data: list = [] - for prior in ensemble_data: + for prior in ensemble_paths: df_data = ensemble.ScratchEnsemble( "flownet_ensemble", From 1c3ba63318bfa88046cc3d6e29aef36889878bf8 Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Fri, 5 Mar 2021 20:37:44 +0100 Subject: [PATCH 06/15] Fix lists --- src/flownet/utils/plot_results.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/flownet/utils/plot_results.py b/src/flownet/utils/plot_results.py index 51083c345..66a279d09 100644 --- a/src/flownet/utils/plot_results.py +++ b/src/flownet/utils/plot_results.py @@ -238,21 +238,21 @@ def main(): parser.add_argument( "-ymin", type=float, - default=0, + default=[0], nargs="+", help="One or #vectors minimum y values.", ) parser.add_argument( "-ymax", type=float, - default=1000, + default=[1000], nargs="+", help="One or #vectors maximum y values.", ) parser.add_argument( "-units", type=str, - default="Cows/Lightyear", + default=["Cows/Lightyear"], nargs="+", help="One or #vectors unit labels.", ) From f6ceab80d0281e2a853d1e536fe9c4d3e89dc464 Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Fri, 5 Mar 2021 20:39:40 +0100 Subject: [PATCH 07/15] Fix no info error --- src/flownet/utils/plot_results.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/flownet/utils/plot_results.py b/src/flownet/utils/plot_results.py index 66a279d09..3bab54b1b 100644 --- a/src/flownet/utils/plot_results.py +++ b/src/flownet/utils/plot_results.py @@ -146,7 +146,11 @@ def check_args(args): f"You should either supply a single units label or as many as you have vectors ({len(args.vectors)}." ) - if len(args.prior) and len(args.posterior) and not args.reference_simulation: + if ( + not len(args.prior) > 0 + and not len(args.posterior) > 0 + and not args.reference_simulation + ): raise ValueError( "There is no prior, posterior or reference simulation to plot. Supply at least something for me to plot." ) From d2708b961a693fb3e26dc2e7c9975c8dfe22e9de Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Fri, 5 Mar 2021 20:43:18 +0100 Subject: [PATCH 08/15] Check None --- src/flownet/utils/plot_results.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/flownet/utils/plot_results.py b/src/flownet/utils/plot_results.py index 3bab54b1b..f265905cb 100644 --- a/src/flownet/utils/plot_results.py +++ b/src/flownet/utils/plot_results.py @@ -306,7 +306,11 @@ def main(): prior_data = build_ensemble_df_list(args.prior, args.vectors) posterior_data = build_ensemble_df_list(args.posterior, args.vectors) - reference_eclsum = EclSum(str(args.reference_simulation.with_suffix(".UNSMRY"))) + + if args.reference_simulation is not None: + reference_eclsum = EclSum(str(args.reference_simulation.with_suffix(".UNSMRY"))) + else: + reference_eclsum = None for i, vector in enumerate(args.vectors): From be494c4dc472ad934f200196ddec47df6756a35c Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Wed, 10 Mar 2021 11:04:48 +0100 Subject: [PATCH 09/15] Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98a496959..78399d850 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## Unreleased ### Added +- [#351](https://github.com/equinor/flownet/pull/351) Added simple plotting tool that allows for plotting of FlowNet ensembles and observations. ### Fixes From 6c8487d712bbc14ed8969af827b6cb13a823489d Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Wed, 10 Mar 2021 11:42:30 +0100 Subject: [PATCH 10/15] Add observations to plotting --- src/flownet/utils/observations.py | 52 +++++++++++++++++++++++++ src/flownet/utils/plot_results.py | 27 +++++++++++++ tests/test_check_obsfiles_ert_yaml.py | 56 +-------------------------- 3 files changed, 81 insertions(+), 54 deletions(-) create mode 100644 src/flownet/utils/observations.py diff --git a/src/flownet/utils/observations.py b/src/flownet/utils/observations.py new file mode 100644 index 000000000..777b42843 --- /dev/null +++ b/src/flownet/utils/observations.py @@ -0,0 +1,52 @@ +import os +import pathlib +from datetime import datetime +import yaml + + +def _read_ert_obs(ert_obs_file_name: pathlib.Path) -> dict: + """This function reads the content of a ERT observation file and returns the information in a dictionary. + + Args: + ert_obs_file_name: path to the ERT observation file + + Returns: + ert_obs: dictionary that contains the information in a ERT observation file + """ + assert os.path.exists(ert_obs_file_name) == 1 + ert_obs: dict = {} + text = "" + with open(ert_obs_file_name, "r") as a_ert_file: + for line in a_ert_file: + text = text + line + + for item in text.replace(" ", "").split("};"): + if "SUMMARY_OBSERVATION" in item: + tmp = item.split("{")[1].split(";") + dic = {} + for var in tmp: + tmp2 = var.split("=") + if len(tmp2) > 1: + dic[tmp2[0]] = tmp2[1] + if not dic["KEY"] in ert_obs: + ert_obs[dic["KEY"]] = [[], [], []] + ert_obs[dic["KEY"]][0].append(datetime.strptime(dic["DATE"], "%d/%m/%Y")) + ert_obs[dic["KEY"]][1].append(float(dic["VALUE"])) + ert_obs[dic["KEY"]][2].append(float(dic["ERROR"])) + + return ert_obs + + +def _read_yaml_obs(yaml_obs_file_name: pathlib.Path) -> dict: + """This function reads the content of a YAML observation file and returns the information in a dictionary. + + Args: + yaml_obs_file_name: path to the YAML observation file + + Returns: + dictionary that contains the information in a YAML observation file + """ + assert os.path.exists(yaml_obs_file_name) == 1 + a_yaml_file = open(yaml_obs_file_name, "r") + + return yaml.load(a_yaml_file, Loader=yaml.FullLoader) diff --git a/src/flownet/utils/plot_results.py b/src/flownet/utils/plot_results.py index f265905cb..f50e77ec2 100644 --- a/src/flownet/utils/plot_results.py +++ b/src/flownet/utils/plot_results.py @@ -10,6 +10,8 @@ from fmu import ensemble from ecl.summary import EclSum +from .observations import _read_ert_obs + matplotlib.use("Agg") @@ -102,6 +104,19 @@ def plot( if plot_settings["vertical_line"]: plt.axvline(x=plot_settings["vertical_line"], color="k", linestyle="--") + if plot_settings["errors"] is not None: + if vector in plot_settings["errors"]: + plt.errorbar( + plot_settings["errors"][vector][0], + plot_settings["errors"][vector][1], + yerr=plot_settings["errors"][vector][2], + fmt="o", + color="k", + ecolor="k", + capsize=5, + elinewidth=2, + ) + plt.ylim([plot_settings["ymin"], plot_settings["ymax"]]) plt.xlabel("date") plt.ylabel(vector + " [" + plot_settings["units"] + "]") @@ -300,6 +315,12 @@ def main(): default=None, help="The reference simulation color.", ) + parser.add_argument( + "-ertobs", + type=pathlib.Path, + default=None, + help="Path to an ERT observation file.", + ) args = parser.parse_args() check_args(args) @@ -307,6 +328,11 @@ def main(): prior_data = build_ensemble_df_list(args.prior, args.vectors) posterior_data = build_ensemble_df_list(args.posterior, args.vectors) + if args.ertobs is not None: + ertobs = _read_ert_obs(args.ertobs) + else: + ertobs = None + if args.reference_simulation is not None: reference_eclsum = EclSum(str(args.reference_simulation.with_suffix(".UNSMRY"))) else: @@ -324,6 +350,7 @@ def main(): "posterior_colors": args.posterior_colors, "reference_simulation_color": args.reference_simulation_color, "vertical_line": args.vertical_line, + "errors": ertobs, } print(f"Plotting {vector}...", end=" ", flush=True) diff --git a/tests/test_check_obsfiles_ert_yaml.py b/tests/test_check_obsfiles_ert_yaml.py index f0128c5a2..77665db6c 100644 --- a/tests/test_check_obsfiles_ert_yaml.py +++ b/tests/test_check_obsfiles_ert_yaml.py @@ -1,16 +1,15 @@ import pathlib -import os.path from datetime import datetime, date import collections import numpy as np import pandas as pd -import yaml import jinja2 from flownet.realization import Schedule from flownet.ert import create_observation_file, resample_schedule_dates from flownet.realization._simulation_keywords import WCONHIST, WCONINJH +from flownet.utils.observations import _read_ert_obs, _read_yaml_obs _OBSERVATION_FILES = pathlib.Path("./tests/observation_files") _PRODUCTION_DATA_FILE_NAME = pathlib.Path(_OBSERVATION_FILES / "ProductionData.csv") @@ -28,57 +27,6 @@ _TEMPLATE_ENVIRONMENT.globals["isnan"] = np.isnan -def _read_ert_obs(ert_obs_file_name: pathlib.Path) -> dict: - """This function reads the content of a ERT observation file and returns the information in a dictionary. - - Args: - ert_obs_file_name: path to the ERT observation file - Returns: - ert_obs: dictionary that contains the information in a ERT observation file - """ - assert os.path.exists(ert_obs_file_name) == 1 - ert_obs = {} - text = "" - with open(ert_obs_file_name, "r") as a_ert_file: - for line in a_ert_file: - text = text + line - - text = text.replace(" ", "") - text = text.split("};") - for item in text: - if "SUMMARY_OBSERVATION" in item: - tmp = item.split("{")[1].split(";") - dic = {} - for var in tmp: - tmp2 = var.split("=") - if len(tmp2) > 1: - dic[tmp2[0]] = tmp2[1] - if not dic["KEY"] in ert_obs: - ert_obs[dic["KEY"]] = [[], [], []] - ert_obs[dic["KEY"]][0].append( - datetime.strptime(dic["DATE"], "%d/%m/%Y").toordinal() - ) - ert_obs[dic["KEY"]][1].append(float(dic["VALUE"])) - ert_obs[dic["KEY"]][2].append(float(dic["ERROR"])) - - return ert_obs - - -def _read_yaml_obs(yaml_obs_file_name: pathlib.Path) -> dict: - """This function reads the content of a YAML observation file and returns the information in a dictionary. - - Args: - yaml_obs_file_name: path to the YAML observation file - Returns: - dictionary that contains the information in a YAML observation file - """ - assert os.path.exists(yaml_obs_file_name) == 1 - a_yaml_file = open(yaml_obs_file_name, "r") - yaml.allow_duplicate_keys = True - - return yaml.load(a_yaml_file, Loader=yaml.FullLoader) - - def compare(ert_obs_dict: dict, yaml_obs_dict: dict) -> None: """This function compares if the given dictionaries: ert_obs_dict and yaml_obs_dict contain the same information. @@ -88,7 +36,7 @@ def compare(ert_obs_dict: dict, yaml_obs_dict: dict) -> None: Returns: None: the function stops by assert functions if both dictionaries have different information. """ - yaml_obs = {} + yaml_obs: dict = {} for item in yaml_obs_dict: for list_item in yaml_obs_dict[item]: for lost_item in list_item["observations"]: From 54e5d213c9f47ec2d3d5e0f2a1f47901ef0ba3a1 Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Wed, 10 Mar 2021 12:47:21 +0100 Subject: [PATCH 11/15] Fix observation tests --- src/flownet/utils/observations.py | 4 +++- tests/test_check_obsfiles_ert_yaml.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/flownet/utils/observations.py b/src/flownet/utils/observations.py index 777b42843..33d88a172 100644 --- a/src/flownet/utils/observations.py +++ b/src/flownet/utils/observations.py @@ -30,7 +30,9 @@ def _read_ert_obs(ert_obs_file_name: pathlib.Path) -> dict: dic[tmp2[0]] = tmp2[1] if not dic["KEY"] in ert_obs: ert_obs[dic["KEY"]] = [[], [], []] - ert_obs[dic["KEY"]][0].append(datetime.strptime(dic["DATE"], "%d/%m/%Y")) + ert_obs[dic["KEY"]][0].append( + datetime.strptime(dic["DATE"], "%d/%m/%Y").date() + ) ert_obs[dic["KEY"]][1].append(float(dic["VALUE"])) ert_obs[dic["KEY"]][2].append(float(dic["ERROR"])) diff --git a/tests/test_check_obsfiles_ert_yaml.py b/tests/test_check_obsfiles_ert_yaml.py index 77665db6c..9f2e9befc 100644 --- a/tests/test_check_obsfiles_ert_yaml.py +++ b/tests/test_check_obsfiles_ert_yaml.py @@ -42,7 +42,7 @@ def compare(ert_obs_dict: dict, yaml_obs_dict: dict) -> None: for lost_item in list_item["observations"]: if not list_item["key"] in yaml_obs: yaml_obs[list_item["key"]] = [[], [], []] - yaml_obs[list_item["key"]][0].append(lost_item["date"].toordinal()) + yaml_obs[list_item["key"]][0].append(lost_item["date"]) yaml_obs[list_item["key"]][1].append(float(lost_item["value"])) yaml_obs[list_item["key"]][2].append(float(lost_item["error"])) assert yaml_obs[list_item["key"]][0] == ert_obs_dict[list_item["key"]][0] From f747f1ae9a17bf950bec2d0caa30baadefe5639c Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Wed, 10 Mar 2021 13:24:07 +0100 Subject: [PATCH 12/15] Add support for multiple lines --- src/flownet/utils/plot_results.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/flownet/utils/plot_results.py b/src/flownet/utils/plot_results.py index f50e77ec2..3b9806ebd 100644 --- a/src/flownet/utils/plot_results.py +++ b/src/flownet/utils/plot_results.py @@ -101,8 +101,9 @@ def plot( alpha=1, ) - if plot_settings["vertical_line"]: - plt.axvline(x=plot_settings["vertical_line"], color="k", linestyle="--") + if plot_settings["vertical_lines"] is not None: + for vertical_line_date in plot_settings["vertical_lines"]: + plt.axvline(x=vertical_line_date, color="k", linestyle="--") if plot_settings["errors"] is not None: if vector in plot_settings["errors"]: @@ -310,10 +311,11 @@ def main(): help="The reference simulation color.", ) parser.add_argument( - "-vertical_line", + "-vertical_lines", type=lambda s: datetime.strptime(s, "%Y-%m-%d"), default=None, - help="The reference simulation color.", + nargs="+", + help="One or more date (YYYY-MM-DD) to add vertical lines in the plot.", ) parser.add_argument( "-ertobs", @@ -349,7 +351,7 @@ def main(): "prior_colors": args.prior_colors, "posterior_colors": args.posterior_colors, "reference_simulation_color": args.reference_simulation_color, - "vertical_line": args.vertical_line, + "vertical_lines": args.vertical_lines, "errors": ertobs, } From aafe59e36666f4626ee32ac375fb6986e56fdebf Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Wed, 10 Mar 2021 13:29:32 +0100 Subject: [PATCH 13/15] People don't like Cows/lightyear, I thought it was funny. --- src/flownet/utils/plot_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/flownet/utils/plot_results.py b/src/flownet/utils/plot_results.py index 3b9806ebd..55d3d3689 100644 --- a/src/flownet/utils/plot_results.py +++ b/src/flownet/utils/plot_results.py @@ -272,7 +272,7 @@ def main(): parser.add_argument( "-units", type=str, - default=["Cows/Lightyear"], + default=[""], nargs="+", help="One or #vectors unit labels.", ) From 02b0e5fd2931bc5d8ec94b9232c6cf31ef375328 Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Wed, 10 Mar 2021 14:17:53 +0100 Subject: [PATCH 14/15] Updated argparse descriptions --- src/flownet/utils/plot_results.py | 52 +++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/src/flownet/utils/plot_results.py b/src/flownet/utils/plot_results.py index 55d3d3689..d96a623f5 100644 --- a/src/flownet/utils/plot_results.py +++ b/src/flownet/utils/plot_results.py @@ -168,7 +168,7 @@ def check_args(args): and not args.reference_simulation ): raise ValueError( - "There is no prior, posterior or reference simulation to plot. Supply at least something for me to plot." + "There is no prior, posterior or reference simulation to plot. Supply at least at one of the three to plot." ) if not (len(args.prior_colors) == 1 or len(args.prior_colors) == len(args.prior)): @@ -233,89 +233,109 @@ def main(): "vectors", type=str, nargs="+", - help="One or more vectors to plot separated by spaces. Example: WOPR:WELL1 FOPR", + help="One or more vectors to plot separated by spaces. " + "Example: WOPR:WELL1 FOPR", ) parser.add_argument( "-prior", type=str, nargs="+", - help="One or more paths to prior ensembles separated by a space." - "The path should include a '%d' which indicates the realization number.", + help="One or more paths to prior ensembles separated by a space. " + "The path should include a '%d' which indicates the realization number. " + "Example: runpath/realization-%d/iter-0/", ) parser.add_argument( "-posterior", type=str, nargs="+", - help="One or more paths to posterior ensembles separated by a space." - "The path should include a '%d' which indicates the realization number.", + help="One or more paths to posterior ensembles separated by a space. " + "The path should include a '%d' which indicates the realization number. " + "Example: runpath/realization-%d/iter-4/", ) parser.add_argument( "-reference_simulation", "-r", type=pathlib.Path, - help="Path to the reference simulation case.", + help="Path to the reference simulation case. " + "Example: path/to/SIMULATION.DATA", ) parser.add_argument( "-ymin", type=float, default=[0], nargs="+", - help="One or #vectors minimum y values.", + help="Lower cut-off of the y-axis. Can be one number or multiple values, " + "depending on the number of vectors you are plotting. In the latter case, " + "the number of y-min values should be equal to the number of vectors.", ) parser.add_argument( "-ymax", type=float, default=[1000], nargs="+", - help="One or #vectors maximum y values.", + help="Upper cut-off of the y-axis. Can be one number or multiple values, " + "depending on the number of vectors you are plotting. In the latter case, " + "the number of y-max values should be equal to the number of vectors.", ) parser.add_argument( "-units", type=str, default=[""], nargs="+", - help="One or #vectors unit labels.", + help="Unit label for the y-axis. Can be one number or multiple units, " + "depending on the number of vectors you are plotting. In the latter case, " + "the number of units should be equal to the number of vectors.", ) parser.add_argument( "-prior_alphas", type=float, default=[0.1], nargs="+", - help="One or #prior ensembles alpha (transparency) values.", + help="Transparency of prior, value between 0 (transparent) and 1 (opaque). " + "Can be one number or multiple values, depending on the number of priors you " + "are plotting. In the latter case, the number of alpha values should be equal " + "to the number of priors.", ) parser.add_argument( "-posterior_alphas", type=float, default=[0.1], nargs="+", - help="One or #posterior ensembles alpha (transparency) values.", + help="Transparency of posterior, value between 0 (transparent) and 1(opaque). " + "Can be one number or multiple values, depending on the number of posteriors " + "you are plotting. In the latter case, the number of alpha values should be equal " + "to the number of posteriors.", ) parser.add_argument( "-prior_colors", type=str, default=["gray"], nargs="+", - help="One or #prior ensembles colors.", + help="Color of prior lines. Can be one number or multiple colors, depending on " + "the number of priors you are plotting. In the latter case, the number of colors " + "should be equal to the number of priors.", ) parser.add_argument( "-posterior_colors", type=str, default=["blue"], nargs="+", - help="One or #posterior ensembles colors.", + help="Color of posterior. Can be one number or colors, depending on the number " + "of posteriors you are plotting. In the latter case, the number of colors should " + "be equal to the number of posteriors.", ) parser.add_argument( "-reference_simulation_color", type=str, default="red", - help="The reference simulation color.", + help="The reference simulation color. Examples: 'red', 'blue', 'green'.", ) parser.add_argument( "-vertical_lines", type=lambda s: datetime.strptime(s, "%Y-%m-%d"), default=None, nargs="+", - help="One or more date (YYYY-MM-DD) to add vertical lines in the plot.", + help="One or more dates (YYYY-MM-DD) to add vertical lines in the plot.", ) parser.add_argument( "-ertobs", From 05a5e3d48c283aa176a8710acb3c837b57327900 Mon Sep 17 00:00:00 2001 From: "Wouter J. de Bruin" Date: Thu, 11 Mar 2021 09:15:59 +0100 Subject: [PATCH 15/15] Updates after review --- src/flownet/utils/plot_results.py | 41 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/src/flownet/utils/plot_results.py b/src/flownet/utils/plot_results.py index d96a623f5..7220937bb 100644 --- a/src/flownet/utils/plot_results.py +++ b/src/flownet/utils/plot_results.py @@ -74,7 +74,7 @@ def plot( reference_simulation: Optional[EclSum], plot_settings: dict, ): - """Main plotting function that generate builds up a single plot build up + """Main plotting function that generates a single plot from potentially multiple ensembles and other data. Args: @@ -87,10 +87,10 @@ def plot( """ plt.figure() # (figsize=[16, 8]) - if len(prior_data): + if prior_data: plot_ensembles("prior", vector, prior_data, plot_settings) - if len(posterior_data): + if posterior_data: plot_ensembles("posterior", vector, posterior_data, plot_settings) if reference_simulation: @@ -163,12 +163,13 @@ def check_args(args): ) if ( - not len(args.prior) > 0 - and not len(args.posterior) > 0 - and not args.reference_simulation + args.prior is None + and args.posterior is None + and args.reference_simulation is None ): raise ValueError( - "There is no prior, posterior or reference simulation to plot. Supply at least at one of the three to plot." + "There is no prior, no posterior and no reference simulation to plot. Supply at least at one " + "of the three to plot." ) if not (len(args.prior_colors) == 1 or len(args.prior_colors) == len(args.prior)): @@ -188,7 +189,7 @@ def check_args(args): def build_ensemble_df_list( - ensemble_paths: List[str], vectors: List[str] + ensemble_paths: Optional[List[str]], vectors: List[str] ) -> List[pd.DataFrame]: """Helper function to read and prepare ensemble data. @@ -202,19 +203,20 @@ def build_ensemble_df_list( """ data: list = [] - for prior in ensemble_paths: + if ensemble_paths is not None: + for prior in ensemble_paths: - df_data = ensemble.ScratchEnsemble( - "flownet_ensemble", - paths=prior.replace("%d", "*"), - ).get_smry(column_keys=vectors) + df_data = ensemble.ScratchEnsemble( + "flownet_ensemble", + paths=prior.replace("%d", "*"), + ).get_smry(column_keys=vectors) - df_data_sorted = df_data.sort_values("DATE") - df_realizations = df_data_sorted[ - df_data_sorted["DATE"] == df_data_sorted.values[-1][0] - ]["REAL"] + df_data_sorted = df_data.sort_values("DATE") + df_realizations = df_data_sorted[ + df_data_sorted["DATE"] == df_data_sorted.values[-1][0] + ]["REAL"] - data.append(df_data.merge(df_realizations, how="inner")) + data.append(df_data.merge(df_realizations, how="inner")) return data @@ -240,6 +242,7 @@ def main(): "-prior", type=str, nargs="+", + default=None, help="One or more paths to prior ensembles separated by a space. " "The path should include a '%d' which indicates the realization number. " "Example: runpath/realization-%d/iter-0/", @@ -248,6 +251,7 @@ def main(): "-posterior", type=str, nargs="+", + default=None, help="One or more paths to posterior ensembles separated by a space. " "The path should include a '%d' which indicates the realization number. " "Example: runpath/realization-%d/iter-4/", @@ -256,6 +260,7 @@ def main(): "-reference_simulation", "-r", type=pathlib.Path, + default=None, help="Path to the reference simulation case. " "Example: path/to/SIMULATION.DATA", )