Merge pull request #3129 from Flamefire/fix_exts_filter

Fix or avoid warnings that commonly arise in build log

Author: boegel

easybuild/framework/easyblock.py

@@ -61,6 +61,7 @@
 from easybuild.framework.easyconfig.style import MAX_LINE_LENGTH
 from easybuild.framework.easyconfig.tools import get_paths_for
 from easybuild.framework.easyconfig.templates import TEMPLATE_NAMES_EASYBLOCK_RUN_STEP, template_constant_dict
+from easybuild.framework.extension import resolve_exts_filter_template
 from easybuild.tools import config, filetools
 from easybuild.tools.build_details import get_build_stats
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, dry_run_warning, dry_run_set_dirs
@@ -1446,43 +1447,18 @@ def skip_extensions(self):
 
         if not exts_filter or len(exts_filter) == 0:
             raise EasyBuildError("Skipping of extensions, but no exts_filter set in easyconfig")
-        elif isinstance(exts_filter, string_type) or len(exts_filter) != 2:
-            raise EasyBuildError('exts_filter should be a list or tuple of ("command","input")')
-        cmdtmpl = exts_filter[0]
-        cmdinputtmpl = exts_filter[1]
 
         res = []
         for ext in self.exts:
-            name = ext['name']
-            if 'options' in ext and 'modulename' in ext['options']:
-                modname = ext['options']['modulename']
-            else:
-                modname = name
-            tmpldict = {
-                'ext_name': modname,
-                'ext_version': ext.get('version'),
-                'src': ext.get('source'),
-            }
-
-            try:
-                cmd = cmdtmpl % tmpldict
-            except KeyError as err:
-                msg = "KeyError occurred on completing extension filter template: %s; "
-                msg += "'name'/'version' keys are no longer supported, should use 'ext_name'/'ext_version' instead"
-                self.log.nosupport(msg % err, '2.0')
-
-            if cmdinputtmpl:
-                stdin = cmdinputtmpl % tmpldict
-                (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False)
-            else:
-                (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, regexp=False)
+            cmd, stdin = resolve_exts_filter_template(exts_filter, ext)
+            (cmdstdouterr, ec) = run_cmd(cmd, log_all=False, log_ok=False, simple=False, inp=stdin, regexp=False)
             self.log.info("exts_filter result %s %s", cmdstdouterr, ec)
             if ec:
-                self.log.info("Not skipping %s" % name)
+                self.log.info("Not skipping %s" % ext['name'])
                 self.log.debug("exit code: %s, stdout/err: %s" % (ec, cmdstdouterr))
                 res.append(ext)
             else:
-                self.log.info("Skipping %s" % name)
+                self.log.info("Skipping %s" % ext['name'])
         self.exts = res
 
     #
@@ -1600,8 +1576,9 @@ def post_iter_step(self):
 
     def det_iter_cnt(self):
         """Determine iteration count based on configure/build/install options that may be lists."""
-        iter_opt_counts = [len(self.cfg[opt]) for opt in ITERATE_OPTIONS
-                           if opt not in ['builddependencies'] and isinstance(self.cfg[opt], (list, tuple))]
+        # Using get_ref to avoid resolving templates as their required attributes may not be available yet
+        iter_opt_counts = [len(self.cfg.get_ref(opt)) for opt in ITERATE_OPTIONS
+                           if opt not in ['builddependencies'] and isinstance(self.cfg.get_ref(opt), (list, tuple))]
 
         # we need to take into account that builddependencies is always a list
         # we're only iterating over it if it's a list of lists

easybuild/framework/easyconfig/default.py

@@ -167,7 +167,7 @@
     'exts_defaultclass': [None, "List of module for and name of the default extension class", EXTENSIONS],
     'exts_default_options': [{}, "List of default options for extensions", EXTENSIONS],
     'exts_filter': [None, ("Extension filter details: template for cmd and input to cmd "
-                           "(templates for name, version and src)."), EXTENSIONS],
+                           "(templates for ext_name, ext_version and src)."), EXTENSIONS],
     'exts_list': [[], 'List with extensions added to the base installation', EXTENSIONS],
 
     # MODULES easyconfig parameters

easybuild/framework/easyconfig/easyconfig.py

@@ -816,16 +816,17 @@ def validate_iterate_opts_lists(self):
         for opt in ITERATE_OPTIONS:
 
             # only when builddependencies is a list of lists are we iterating over them
-            if opt == 'builddependencies' and not all(isinstance(e, list) for e in self[opt]):
+            if opt == 'builddependencies' and not all(isinstance(e, list) for e in self.get_ref(opt)):
                 continue
 
+            opt_value = self.get(opt, None, resolve=False)
             # anticipate changes in available easyconfig parameters (e.g. makeopts -> buildopts?)
-            if self.get(opt, None) is None:
+            if opt_value is None:
                 raise EasyBuildError("%s not available in self.cfg (anymore)?!", opt)
 
             # keep track of list, supply first element as first option to handle
-            if isinstance(self[opt], (list, tuple)):
-                opt_counts.append((opt, len(self[opt])))
+            if isinstance(opt_value, (list, tuple)):
+                opt_counts.append((opt, len(opt_value)))
 
         # make sure that options that specify lists have the same length
         list_opt_lengths = [length for (opt, length) in opt_counts if length > 1]
@@ -1523,12 +1524,13 @@ def __setitem__(self, key, value):
                                  key, value)
 
     @handle_deprecated_or_replaced_easyconfig_parameters
-    def get(self, key, default=None):
+    def get(self, key, default=None, resolve=True):
         """
         Gets the value of a key in the config, with 'default' as fallback.
+        :param resolve: if False, disables templating via calling get_ref, else resolves template values
         """
         if key in self:
-            return self[key]
+            return self[key] if resolve else self.get_ref(key)
         else:
             return default
 

easybuild/framework/extension.py

@@ -38,9 +38,47 @@
 
 from easybuild.framework.easyconfig.easyconfig import resolve_template
 from easybuild.framework.easyconfig.templates import template_constant_dict
-from easybuild.tools.build_log import EasyBuildError
+from easybuild.tools.build_log import EasyBuildError, raise_nosupport
 from easybuild.tools.filetools import change_dir
 from easybuild.tools.run import run_cmd
+from easybuild.tools.py2vs3 import string_type
+
+
+def resolve_exts_filter_template(exts_filter, ext):
+    """
+    Resolve the exts_filter tuple by replacing the template values using the extension
+    :param exts_filter: Tuple of (command, input) using template values (ext_name, ext_version, src)
+    :param ext: Instance of Extension or dictionary like with 'name' and optionally 'options', 'version', 'source' keys
+    :return (cmd, input) as a tuple of strings
+    """
+
+    if isinstance(exts_filter, string_type) or len(exts_filter) != 2:
+        raise EasyBuildError('exts_filter should be a list or tuple of ("command","input")')
+
+    cmd, cmdinput = exts_filter
+
+    if not isinstance(ext, dict):
+        ext = {'name': ext.name, 'version': ext.version, 'src': ext.src, 'options': ext.options}
+
+    name = ext['name']
+    if 'options' in ext and 'modulename' in ext['options']:
+        modname = ext['options']['modulename']
+    else:
+        modname = name
+    tmpldict = {
+        'ext_name': modname,
+        'ext_version': ext.get('version'),
+        'src': ext.get('src'),
+    }
+
+    try:
+        cmd = cmd % tmpldict
+        cmdinput = cmdinput % tmpldict if cmdinput else None
+    except KeyError as err:
+        msg = "KeyError occurred on completing extension filter template: %s; "
+        msg += "'name'/'version' keys are no longer supported, should use 'ext_name'/'ext_version' instead"
+        raise_nosupport(msg % err, '2.0')
+    return cmd, cmdinput
 
 
 class Extension(object):
@@ -144,16 +182,11 @@ def sanity_check_step(self):
         if os.path.isdir(self.installdir):
             change_dir(self.installdir)
 
-        # disabling templating is required here to support legacy string templates like name/version
-        self.cfg.enable_templating = False
-        exts_filter = self.cfg['exts_filter']
-        self.cfg.enable_templating = True
+        # Get raw value to translate ext_name, ext_version, src
+        exts_filter = self.cfg.get_ref('exts_filter')
 
-        if exts_filter is not None:
-            cmd, inp = exts_filter
-        else:
+        if exts_filter is None:
             self.log.debug("no exts_filter setting found, skipping sanitycheck")
-            cmd = None
 
         if 'modulename' in self.options:
             modname = self.options['modulename']
@@ -165,22 +198,8 @@ def sanity_check_step(self):
         # allow skipping of sanity check by setting module name to False
         if modname is False:
             self.log.info("modulename set to False for '%s' extension, so skipping sanity check", self.name)
-        elif cmd:
-            template = {
-                        'ext_name': modname,
-                        'ext_version': self.version,
-                        'src': self.src,
-                        # the ones below are only there for legacy purposes
-                        # TODO deprecated, remove in v2.0
-                        # TODO same dict is used in easyblock.py skip_extensions, resolve this
-                        'name': modname,
-                        'version': self.version,
-                       }
-            cmd = cmd % template
-
-            stdin = None
-            if inp:
-                stdin = inp % template
+        elif exts_filter:
+            cmd, stdin = resolve_exts_filter_template(exts_filter, self)
             # set log_ok to False so we can catch the error instead of run_cmd
             (output, ec) = run_cmd(cmd, log_ok=False, simple=False, regexp=False, inp=stdin)
 

easybuild/framework/extensioneasyblock.py

@@ -116,9 +116,9 @@ def sanity_check_step(self, exts_filter=None, custom_paths=None, custom_commands
         """
         Custom sanity check for extensions, whether installed as stand-alone module or not
         """
-        if not self.cfg['exts_filter']:
+        if not self.cfg.get_ref('exts_filter'):
             self.cfg['exts_filter'] = exts_filter
-        self.log.debug("starting sanity check for extension with filter %s", self.cfg['exts_filter'])
+        self.log.debug("starting sanity check for extension with filter %s", self.cfg.get_ref('exts_filter'))
 
         # for stand-alone installations that were done for multiple dependency versions (via multi_deps),
         # we need to perform the extension sanity check for each of them, by loading the corresponding modules first

easybuild/tools/build_log.py

@@ -90,6 +90,12 @@ def raise_easybuilderror(msg, *args):
     raise EasyBuildError(msg, *args)
 
 
+def raise_nosupport(msg, ver):
+    """Construct error message for no longer supported behaviour, and raise an EasyBuildError."""
+    nosupport_msg = "NO LONGER SUPPORTED since v%s: %s; see %s for more information"
+    raise_easybuilderror(nosupport_msg, ver, msg, DEPRECATED_DOC_URL)
+
+
 class EasyBuildLog(fancylogger.FancyLogger):
     """
     The EasyBuild logger, with its own error and exception functions.
@@ -154,9 +160,8 @@ def log_callback_warning_and_print(msg):
             fancylogger.FancyLogger.deprecated(self, msg, ver, max_ver, *args, **kwargs)
 
     def nosupport(self, msg, ver):
-        """Print error message for no longer supported behaviour, and raise an EasyBuildError."""
-        nosupport_msg = "NO LONGER SUPPORTED since v%s: %s; see %s for more information"
-        raise EasyBuildError(nosupport_msg, ver, msg, DEPRECATED_DOC_URL)
+        """Raise error message for no longer supported behaviour, and raise an EasyBuildError."""
+        raise_nosupport(msg, ver)
 
     def error(self, msg, *args, **kwargs):
         """Print error message and raise an EasyBuildError."""

easybuild/tools/filetools.py

@@ -384,12 +384,14 @@ def extract_file(fn, dest, cmd=None, extra_options=None, overwrite=False, forced
     return find_base_dir()
 
 
-def which(cmd, retain_all=False, check_perms=True):
+def which(cmd, retain_all=False, check_perms=True, log_ok=True, log_error=True):
     """
     Return (first) path in $PATH for specified command, or None if command is not found
 
     :param retain_all: returns *all* locations to the specified command in $PATH, not just the first one
     :param check_perms: check whether candidate path has read/exec permissions before accepting it as a match
+    :param log_ok: Log an info message where the command has been found (if any)
+    :param log_error: Log a warning message when command hasn't been found
     """
     if retain_all:
         res = []
@@ -401,7 +403,8 @@ def which(cmd, retain_all=False, check_perms=True):
         cmd_path = os.path.join(path, cmd)
         # only accept path if command is there
         if os.path.isfile(cmd_path):
-            _log.info("Command %s found at %s", cmd, cmd_path)
+            if log_ok:
+                _log.info("Command %s found at %s", cmd, cmd_path)
             if check_perms:
                 # check if read/executable permissions are available
                 if not os.access(cmd_path, os.R_OK | os.X_OK):
@@ -413,7 +416,7 @@ def which(cmd, retain_all=False, check_perms=True):
                 res = cmd_path
                 break
 
-    if not res:
+    if not res and log_error:
         _log.warning("Could not find command '%s' (with permissions to read/execute it) in $PATH (%s)" % (cmd, paths))
     return res
 

easybuild/tools/modules.py

@@ -180,16 +180,17 @@ def __init__(self, mod_paths=None, testing=False):
         if mod_paths is not None:
             self.set_mod_paths(mod_paths)
 
-        # only use command path in environment variable if command in not available in $PATH
-        if which(self.cmd) is None and env_cmd_path is not None:
-            self.log.debug("Set %s command via environment variable %s: %s",
-                           self.NAME, self.COMMAND_ENVIRONMENT, self.cmd)
-            self.cmd = env_cmd_path
-
-        # check whether paths obtained via $PATH and $LMOD_CMD are different
-        elif which(self.cmd) != env_cmd_path:
-            self.log.debug("Different paths found for %s command '%s' via which/$PATH and $%s: %s vs %s",
-                           self.NAME, self.COMMAND, self.COMMAND_ENVIRONMENT, self.cmd, env_cmd_path)
+        if env_cmd_path:
+            cmd_path = which(self.cmd, log_ok=False, log_error=False)
+            # only use command path in environment variable if command in not available in $PATH
+            if cmd_path is None:
+                self.cmd = env_cmd_path
+                self.log.debug("Set %s command via environment variable %s: %s",
+                               self.NAME, self.COMMAND_ENVIRONMENT, self.cmd)
+            # check whether paths obtained via $PATH and $LMOD_CMD are different
+            elif cmd_path != env_cmd_path:
+                self.log.debug("Different paths found for %s command '%s' via which/$PATH and $%s: %s vs %s",
+                               self.NAME, self.COMMAND, self.COMMAND_ENVIRONMENT, cmd_path, env_cmd_path)
 
         # make sure the module command was found
         if self.cmd is None:
@@ -284,7 +285,7 @@ def set_and_check_version(self):
 
     def check_cmd_avail(self):
         """Check whether modules tool command is available."""
-        cmd_path = which(self.cmd)
+        cmd_path = which(self.cmd, log_ok=False)
         if cmd_path is not None:
             self.cmd = cmd_path
             self.log.info("Full path for %s command is %s, so using it", self.NAME, self.cmd)
@@ -674,7 +675,7 @@ def modulefile_path(self, mod_name, strip_ext=False):
         :param strip_ext: strip (.lua) extension from module fileame (if present)"""
         # (possible relative) path is always followed by a ':', and may be prepended by whitespace
         # this works for both environment modules and Lmod
-        modpath_re = re.compile('^\s*(?P<modpath>[^/\n]*/[^\s]+):$', re.M)
+        modpath_re = re.compile(r'^\s*(?P<modpath>[^/\n]*/[^\s]+):$', re.M)
         modpath = self.get_value_from_modulefile(mod_name, modpath_re)
 
         if strip_ext and modpath.endswith('.lua'):
@@ -939,7 +940,7 @@ def file_join(res):
             """Helper function to compose joined path."""
             return os.path.join(*[x.strip('"') for x in res.groups()])
 
-        res = re.sub('\[\s+file\s+join\s+(.*)\s+(.*)\s+\]', file_join, res)
+        res = re.sub(r'\[\s+file\s+join\s+(.*)\s+(.*)\s+\]', file_join, res)
 
         # also interpret all $env(...) parts
         res = re.sub(r'\$env\((?P<key>[^)]*)\)', lambda res: os.getenv(res.group('key'), ''), res)
@@ -1158,7 +1159,7 @@ def run_module(self, *args, **kwargs):
         # this is required for the DEISA variant of modulecmd.tcl which is commonly used
         def tweak_stdout(txt):
             """Tweak stdout before it's exec'ed as Python code."""
-            modulescript_regex = "^exec\s+[\"'](?P<modulescript>/tmp/modulescript_[0-9_]+)[\"']$"
+            modulescript_regex = r"^exec\s+[\"'](?P<modulescript>/tmp/modulescript_[0-9_]+)[\"']$"
             return re.sub(modulescript_regex, r"execfile('\1')", txt)
 
         tweak_stdout_fn = None
@@ -1366,7 +1367,7 @@ def module_wrapper_exists(self, mod_name):
 
         # first consider .modulerc.lua with Lmod 7.8 (or newer)
         if StrictVersion(self.version) >= StrictVersion('7.8'):
-            mod_wrapper_regex_template = '^module_version\("(?P<wrapped_mod>.*)", "%s"\)$'
+            mod_wrapper_regex_template = r'^module_version\("(?P<wrapped_mod>.*)", "%s"\)$'
             res = super(Lmod, self).module_wrapper_exists(mod_name, modulerc_fn='.modulerc.lua',
                                                           mod_wrapper_regex_template=mod_wrapper_regex_template)
 

easybuild/tools/systemtools.py

@@ -114,6 +114,9 @@
     'AuthenticAMD': AMD,
     'GenuineIntel': INTEL,
     'IBM': IBM,
+    # IBM POWER9
+    '8335-GTH': IBM,
+    '8335-GTX': IBM,
 }
 # ARM Cortex part numbers from the corresponding ARM Processor Technical Reference Manuals,
 # see http://infocenter.arm.com - Cortex-A series processors, Section "Main ID Register"
@@ -276,7 +279,7 @@ def get_cpu_vendor():
         if arch == X86_64:
             vendor_regex = re.compile(r"vendor_id\s+:\s*(\S+)")
         elif arch == POWER:
-            vendor_regex = re.compile(r"model\s+:\s*(\w+)")
+            vendor_regex = re.compile(r"model\s+:\s*((\w|-)+)")
         elif arch in [AARCH32, AARCH64]:
             vendor_regex = re.compile(r"CPU implementer\s+:\s*(\S+)")
 

easybuild/tools/toolchain/options.py

@@ -86,7 +86,12 @@ def option(self, name, templatedict=None):
         """Return option value"""
         value = self.get(name, None)
         if value is None and name not in self.options_map:
-            self.log.warning("option: option with name %s returns None" % name)
+            msg = "option: option with name %s returns None" % name
+            # Empty options starting with _opt_ are allowed, so don't warn
+            if name.startswith('_opt_'):
+                self.log.devel(msg)
+            else:
+                self.log.warning(msg)
             res = None
         elif name in self.options_map:
             res = self.options_map[name]