metomi#137: add rose edit, rose macro functionality

demo/rose-config-edit/demo_meta/app/16-meta-import/rose-app.conf

@@ -0,0 +1,6 @@
+meta=rose-demo-baked-alaska/vn1.0
+
+[env]
+ICECREAM_TEMPERATURE=-1
+MERINGUE_NICENESS=0
+SPONGE_DENSITY=0.1

etc/rose-meta/rose-demo-baked-alaska-icecream/HEAD/rose-meta.conf

@@ -0,0 +1,4 @@
+[env=ICECREAM_TEMPERATURE]
+range=-20:0
+title=Icecream Temperature (celsius) (version HEAD)
+type=integer

etc/rose-meta/rose-demo-baked-alaska-icecream/vn1.0/rose-meta.conf

@@ -0,0 +1,4 @@
+[env=ICECREAM_TEMPERATURE]
+range=-20:0
+title=Icecream Temperature (celsius) (version 1)
+type=integer

etc/rose-meta/rose-demo-baked-alaska-icecream/vn2.0/rose-meta.conf

@@ -0,0 +1,4 @@
+[env=ICECREAM_TEMPERATURE]
+range=-20:0
+title=Icecream Temperature (celsius) (version 2)
+type=integer

etc/rose-meta/rose-demo-baked-alaska-sponge/HEAD/rose-meta.conf

@@ -0,0 +1,4 @@
+[env=SPONGE_DENSITY]
+range=0:1
+title=Sponge Density (g cm^-3) (version HEAD)
+type=real

etc/rose-meta/rose-demo-baked-alaska-sponge/vn1.0/lib/python/macros/desoggy.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+#-----------------------------------------------------------------------------
+# (C) British Crown Copyright 2012-4 Met Office.
+#
+# This file is part of Rose, a framework for meteorological suites.
+#
+# Rose is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Rose is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Rose. If not, see <http://www.gnu.org/licenses/>.
+#-----------------------------------------------------------------------------
+"""This module contains:
+
+SpongeDeSoggifier, a rose transform macro.
+
+"""
+
+import re
+import subprocess
+
+import rose.macro
+
+
+class SpongeDeSoggifier(rose.macro.MacroBase):
+
+    """De-soggifies the sponge."""
+
+    SOGGY_FIX_TEXT = "de-soggified"
+
+    def transform(self, config, meta_config=None):
+        """Reduce the density of the sponge."""
+        sponge_density = config.get_value(["env", "SPONGE_DENSITY"])
+        if sponge_density is not None and float(sponge_density) > 0.5:
+            # 1 g cm^-3 is pure water, so this is pretty soggy.
+            config.set(["env", "SPONGE_DENSITY"], "0.3")
+            self.add_report(
+                "env", "SPONGE_DENSITY", "0.3", self.SOGGY_FIX_TEXT)
+        return config, self.reports

etc/rose-meta/rose-demo-baked-alaska-sponge/vn1.0/rose-meta.conf

@@ -0,0 +1,4 @@
+[env=SPONGE_DENSITY]
+range=0:1
+title=Sponge Density (g cm^-3) (version 1)
+type=real

etc/rose-meta/rose-demo-baked-alaska-sponge/vn2.0/rose-meta.conf

@@ -0,0 +1,4 @@
+[env=SPONGE_DENSITY]
+range=0:1
+title=Sponge Density (g cm^-3) (version 2)
+type=real

etc/rose-meta/rose-demo-baked-alaska/HEAD/rose-meta.conf

@@ -0,0 +1,6 @@
+import=rose-demo-baked-alaska-icecream/HEAD rose-demo-baked-alaska-sponge/
+
+[env=MERINGUE_NICENESS]
+title=Meringue Niceness (version HEAD)
+range=-20:20
+type=integer

etc/rose-meta/rose-demo-baked-alaska/vn1.0/rose-meta.conf

@@ -0,0 +1,6 @@
+import=rose-demo-baked-alaska-icecream/vn1.0 rose-demo-baked-alaska-sponge/vn1.0
+
+[env=MERINGUE_NICENESS]
+title=Meringue Niceness (version 1)
+range=-20:20
+type=integer

etc/rose-meta/rose-demo-baked-alaska/vn2.0/rose-meta.conf

@@ -0,0 +1,6 @@
+import=rose-demo-baked-alaska-icecream/vn2.0 rose-demo-baked-alaska-sponge/vn2.0
+
+[env=MERINGUE_NICENESS]
+title=Meringue Niceness (version 2)
+range=-20:20
+type=integer

lib/python/rose/config_editor/data.py

@@ -39,6 +39,7 @@
 
 import rose.config
 import rose.config_editor.data_helper
+import rose.config_tree
 import rose.gtk.dialog
 import rose.macro
 import rose.metadata_check
@@ -334,20 +335,21 @@ def load_config(self, config_directory=None,
 
 
         if config_directory != self.top_level_directory and preview:
-            meta_config = rose.config.ConfigNode()
-            meta_files = []
+            meta_config_tree = rose.config_tree.ConfigTree()
         elif metadata_off:
-            meta_config = self.load_meta_config(config_type=config_type)
-            meta_files = []
+            meta_config_tree = self.load_meta_config_tree(
+                config_type=config_type)
         else:
-            meta_config = self.load_meta_config(config, config_directory,
-                                                config_type=config_type)
-            meta_files = self.load_meta_files(config, config_directory)
-
+            meta_config_tree = self.load_meta_config_tree(
+                config, config_directory,
+                config_type=config_type
+            )
+        meta_config = meta_config_tree.node
         opt_conf_lookup = self.load_optional_configs(config_directory)
 
 
         macro_module_prefix = self.helper.get_macro_module_prefix(name)
+        meta_files = self.load_meta_files(meta_config_tree)
         macros = rose.macro.load_meta_macro_modules(
                       meta_files, module_prefix=macro_module_prefix)
         meta_id = self.helper.get_config_meta_flag(
@@ -814,33 +816,22 @@ def clear_meta_lookups(self, config_name):
         if config_name in self._config_section_namespace_lookup:
             self._config_section_namespace_lookup.pop(config_name)
 
-    def load_meta_config(self, config=None, directory=None, config_type=None):
+    def load_meta_config_tree(self, config=None, directory=None,
+                              config_type=None):
         """Load the main metadata, and any specified in 'config'."""
         if config is None:
             config = rose.config.ConfigNode()
         error_handler = rose.config_editor.util.launch_error_dialog
-        return rose.macro.load_meta_config(config, directory,
-                                           config_type=config_type,
-                                           error_handler=error_handler)
+        return rose.macro.load_meta_config_tree(config, directory,
+                                                config_type=config_type,
+                                                error_handler=error_handler)
 
-    def load_meta_files(self, config=None, directory=None):
+    def load_meta_files(self, config_tree):
         """Load the file paths of files within the metadata directory."""
-        if config is None:
-            config = rose.config.ConfigNode()
-        meta_filepaths = []
-        meta_path, warning = self.load_meta_path(config, directory)
-        if meta_path is None:
-            return []
-        try:
-            file_tuples = os.walk(meta_path)
-        except OSError:
-            return []
-        for dirpath, dirnames, filenames in file_tuples:
-            if '/.' in dirpath:
-                continue
-            for fname in filenames:
-                meta_filepaths.append(os.path.join(dirpath, fname))
-        return meta_filepaths
+        meta_files = []
+        for rel_path, conf_dir in config_tree.files.items():
+            meta_files.append(os.path.join(conf_dir, rel_path))
+        return meta_files
 
     def filter_meta_config(self, config_name):
         """Filter out invalid metadata."""

lib/python/rose/config_editor/main.py

@@ -1442,14 +1442,16 @@ def refresh_metadata(self, metadata_off=False, only_this_config=None):
             del config_data.macros
             meta_config = config_data.meta
             if metadata_off:
-                meta_config = self.data.load_meta_config(
+                meta_config_tree = self.data.load_meta_config_tree(
                     config_type=config_data.config_type)
-                meta_files = []
+                meta_config = meta_config_tree.node
+                meta_files = self.data.load_meta_files(meta_config_tree)
                 macros = []
             else:
-                meta_config = self.data.load_meta_config(config, directory,
-                    config_type=config_data.config_type)
-                meta_files = self.data.load_meta_files(config, directory)
+                meta_config_tree = self.data.load_meta_config_tree(
+                    config, directory, config_type=config_data.config_type)
+                meta_config = meta_config_tree.node
+                meta_files = self.data.load_meta_files(meta_config_tree)
                 macro_module_prefix = (
                     self.data.helper.get_macro_module_prefix(config_name))
                 macros = rose.macro.load_meta_macro_modules(

lib/python/rose/config_editor/plugin/um/widget/stash.py

@@ -282,16 +282,10 @@ def get_stashmaster_meta_lookup_dict(self):
         """
         if self.STASHMASTER_META_PATH is None:
             return {}
-        conf = rose.resource.ResourceLocator.default().get_conf()
-        meta_path_list = []
-        meta_path_str = conf.get_value(["meta-path"])
-        if meta_path_str:
-            meta_path_list = meta_path_str.split(":")
         try:
             config = rose.config_tree.ConfigTreeLoader().load(
                 self.STASHMASTER_META_PATH,
-                self.STASHMASTER_META_FILENAME,
-                meta_path_list).node
+                self.STASHMASTER_META_FILENAME).node
         except (rose.config.ConfigSyntaxError, IOError, OSError) as e:
             rose.reporter.Reporter()(
                 "Error loading STASHmaster metadata resource: " +

lib/python/rose/config_tree.py

@@ -81,7 +81,7 @@ def __init__(self, *args, **kwargs):
         self.node_loader = ConfigLoader(*args, **kwargs)
 
     def load(self, conf_dir, conf_name, conf_dir_paths=None, opt_keys=None,
-             no_ignore=False):
+             conf_node=None, no_ignore=False):
         """Load a (runtime) configuration directory with inheritance.
 
         Return a ConfigTree object that represents the result.
@@ -92,6 +92,9 @@ def load(self, conf_dir, conf_name, conf_dir_paths=None, opt_keys=None,
         conf_dir_paths -- A list of directories to locate relative paths to
                           configurations.
         opt_keys -- Optional configuration keys.
+        conf_node -- A rose.config.ConfigNode to extend, or None to use a
+                     fresh one.
+        no_ignore -- If True, skip loading ignored config settings.
 
         """
 
@@ -117,7 +120,10 @@ def load(self, conf_dir, conf_name, conf_dir_paths=None, opt_keys=None,
             if bad_keys:
                 raise BadOptionalConfigurationKeysError(bad_keys)
 
-        conf_tree.node = ConfigNode()
+        if conf_node is None:
+            conf_tree.node = ConfigNode()
+        else:
+            conf_tree.node = conf_node
         for t_conf_dir in conf_tree.conf_dirs:
             node = nodes[t_conf_dir]
             for keys, sub_node in node.walk(no_ignore=no_ignore):

lib/python/rose/macro.py

@@ -34,6 +34,7 @@
 import traceback
 
 import rose.config
+import rose.config_tree
 import rose.formats.namelist
 from rose.opt_parse import RoseOptionParser
 import rose.reporter
@@ -472,40 +473,67 @@ def load_meta_path(config=None, directory=None, is_upgrade=False,
     return None, warning
 
 
-def load_meta_config(config, directory=None, config_type=None,
-                     error_handler=None, ignore_meta_error=False):
-    """Return the metadata config for a configuration."""
+def load_meta_config_tree(config, directory=None, config_type=None,
+                          error_handler=None, ignore_meta_error=False):
+    """Return the metadata config tree for a configuration."""
     if error_handler is None:
         error_handler = _report_error
-    meta_config = rose.config.ConfigNode()
     meta_list = ["rose-all/" + rose.META_CONFIG_NAME]
     if config_type is not None:
         default_meta_dir = config_type.replace(".", "-")
         meta_list.append(default_meta_dir + "/" + rose.META_CONFIG_NAME)
     config_meta_path, warning = load_meta_path(config, directory)
     if warning is not None and not ignore_meta_error:
         error_handler(text=warning)
-    if config_meta_path is not None:
-        path = os.path.join(config_meta_path, rose.META_CONFIG_NAME)
-        if path not in meta_list:
-            meta_list.append(path)
     locator = rose.resource.ResourceLocator(paths=sys.path)
     opt_node = config.get([rose.CONFIG_SECT_TOP,
                            rose.CONFIG_OPT_META_TYPE], no_ignore=True)
     ignore_meta_error = ignore_meta_error or opt_node is None
-    config_loader = rose.config.ConfigLoader()
+    meta_config_tree = None
+    meta_config = rose.config.ConfigNode()
     for meta_key in meta_list:
         try:
             meta_path = locator.locate(meta_key)
-        except rose.resource.ResourceError as e:
+        except rose.resource.ResourceError:
             if not ignore_meta_error:
                 error_handler(text=ERROR_LOAD_META_PATH.format(meta_key))
+            continue
+        try:
+            meta_config_tree = rose.config_tree.ConfigTreeLoader().load(
+                os.path.dirname(meta_path),
+                rose.META_CONFIG_NAME,
+                conf_dir_paths=list(sys.path),
+                conf_node=meta_config
+            )
+        except rose.config.ConfigSyntaxError as exc:
+            error_handler(text=str(exc))
         else:
-            try:
-                config_loader.load_with_opts(meta_path, meta_config)
-            except rose.config.ConfigSyntaxError as e:
-                error_handler(text=str(e))
-    return meta_config
+            meta_config = meta_config_tree.node
+    if config_meta_path is None:
+        return meta_config_tree
+    # Try and get a proper non-default meta config tree.
+    try:
+        meta_config_tree = rose.config_tree.ConfigTreeLoader().load(
+            config_meta_path,
+            rose.META_CONFIG_NAME,
+            conf_dir_paths=list(sys.path),
+            conf_node=meta_config
+        )
+    except rose.resource.ResourceError:
+        if not ignore_meta_error:
+            error_handler(text=ERROR_LOAD_META_PATH.format(meta_key))
+    except rose.config.ConfigSyntaxError as exc:
+        error_handler(text=str(exc))
+    return meta_config_tree
+
+
+def load_meta_config(config, directory=None, config_type=None,
+                     error_handler=None, ignore_meta_error=False):
+    """Return the metadata config for a configuration."""
+    config_tree = load_meta_config_tree(
+        config, directory=directory, config_type=config_type,
+        error_handler=error_handler, ignore_meta_error=ignore_meta_error)
+    return config_tree.node
 
 
 def load_meta_macro_modules(meta_files, module_prefix=None):
@@ -558,15 +586,12 @@ def get_macros_for_config(app_config=None,
     """Driver function to return macro names for a config object."""
     if app_config is None:
         app_config = rose.config.ConfigNode()
-    meta_path, warning = load_meta_path(app_config, config_directory)
-    if meta_path is None:
-        sys.exit(ERROR_LOAD_METADATA.format(""))
-    meta_filepaths = []
-    for dirpath, dirnames, filenames in os.walk(meta_path):
-        if "/.svn" in dirpath:
-            continue
-        for fname in filenames:
-            meta_filepaths.append(os.path.join(dirpath, fname))
+    meta_config_tree = load_meta_config_tree(
+        app_config, directory=config_directory)
+    if meta_config_tree is None:
+        return []
+    meta_filepaths = [
+        os.path.join(v, k) for k, v in meta_config_tree.files.items()]
     modules = load_meta_macro_modules(meta_filepaths)
     if include_system:
         import rose.macros  # Done to avoid cyclic top-level imports.

lib/python/rose/macros/duplicate.py

@@ -32,7 +32,6 @@ class DuplicateChecker(rose.macro.MacroBase):
 
     def validate(self, config, meta_config=None):
         """Return a list of errors, if any."""
-        meta_config = self._load_meta_config(config, meta_config)
         self.reports = []
         sect_error_no_dupl = {}
         sect_keys = config.value.keys()