Perl helper

easybuild/scripts/cpan2eb.py

@@ -0,0 +1,140 @@
+##
+# Copyright 2013-2014 Ghent University
+#
+# This file is part of EasyBuild,
+# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en),
+# with support of Ghent University (http://ugent.be/hpc),
+# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en),
+# the Hercules foundation (http://www.herculesstichting.be/in_English)
+# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en).
+#
+# http://github.com/hpcugent/easybuild
+#
+# EasyBuild is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation v2.
+#
+# EasyBuild is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with EasyBuild.  If not, see <http://www.gnu.org/licenses/>.
+##
+"""
+This script takes a Perl module name as argument, and generates
+a string compatible with the easyconfig 1.x format with metadata about the module
+and all it's dependencies
+
+@author: Jens Timmerman
+"""
+from urllib2 import HTTPError
+from vsc.utils.rest import RestClient
+from vsc.utils.generaloption import simple_option
+from vsc.utils import fancylogger
+
+
+logger = fancylogger.getLogger()
+
+
+class CpanMeta(object):
+    """This class gets meta information from cpan
+
+    This uses the metacpan.org api
+    """
+    def __init__(self):
+        """Constructor"""
+        dummy = {'download_url': 'example.com/bla', 'release': '0', 'version': '0', 'distribution': 'ExtUtils-MakeMaker',
+                 'modulename': 'ExtUtils::MakeMaker'}
+        self.cache = {'ExtUtils::MakeMaker': dummy,  'perl': dummy}
+        self.graph = {'ExtUtils::MakeMaker': [], 'perl': []}
+        self.client = RestClient('http://api.metacpan.org')
+
+    def get_module_data(self, modulename):
+        """Get some metadata about the current version of the module"""
+        try:
+            json = self.client.v0.module[modulename].get()[1]
+            depsjson = self.client.v0.release[json['author']][json['release']].get()
+        except HTTPError:
+            logger.error("API error for getting %s this will have to be resolved manually", modulename)
+            return {'release': '0', 'dependency': [] }
+        depsjson = depsjson[1]
+        depsjson.update(json)
+        depsjson.update({'modulename': modulename})
+        return depsjson
+
+    def get_recursive_data(self, modulename):
+        """Recursively get all data (so for all dependencies)"""
+        # check if we have been here before
+        if modulename in self.cache:
+            logger.info('%s module cached ', modulename)
+            return self.graph
+        data = self.get_module_data(modulename)
+        self.cache[modulename] = data
+        # module's are somtimes included in a release we already know, so skip this also
+        if data['release'] in self.cache:
+            logger.info('%s release cached', data['release'])
+            return self.graph
+        self.cache[data['release']] = data
+        dependencies = set()
+        # do the recursive thing
+        for dep in data['dependency']:
+            if "requires" in dep["relationship"]:
+                # we filter on dependendencies for the build and configure phase, otherwise we end up with circular
+                # dependencies
+                if dep['phase'] in ('build', 'configure') :
+                    self.get_recursive_data(dep['module'])
+                    dependencies.add(dep['module'])
+        self.graph[modulename] = dependencies
+        return self.graph
+
+
+def post_order(graph, root):
+    """Walk the graph from the given root in a post-order manner, by providing the correspoding generator."""
+    for node in graph[root]:
+        for child in post_order(graph, node):
+            yield child
+    yield root
+
+
+def topological_sort(graph, root):
+    """Perform a topological sorting of the given graph.
+
+    The graph needs to be in the following format:
+
+        g = { t1: [t2, t3],
+              t2: [t4, t5, t6],
+              t3: []
+              t4: []
+              t5: [t3]
+              t6: [t5]
+            }
+
+    where each node is mapped to a list of nodes it has an edge to.
+
+    @returns: generator for traversing the graph in the desired order
+    """
+    visited = set()
+    for node in post_order(graph, root):
+        if node not in visited:
+            yield node
+            visited.add(node)  # been there, done that.
+
+
+go = simple_option()
+
+cpan = CpanMeta()
+modules = cpan.get_recursive_data(go.args[0])
+print modules
+
+# topological soft, so we get correct dependencies order
+for module in topological_sort(modules, go.args[0]):
+    data = cpan.cache[module]
+    url, name = data['download_url'].rsplit("/", 1)
+    data.update({'url': url, 'tarball': name})  # distribution sometimes contains subdirs
+    if data['release'] is not '0' and data['version'] is not '0':
+        print    """('%(modulename)s', '%(version)s', {
+                    'source_tmpl': '%(tarball)s',
+                    'source_urls': ['%(url)s'],
+                }),""" % data

easybuild/tools/github.py

@@ -30,7 +30,6 @@
 """
 import base64
 import os
-import re
 import socket
 import tempfile
 import urllib
@@ -226,7 +225,7 @@ def download(url, path=None):
     pr_url = g.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr]
     try:
         status, pr_data = pr_url.get()
-    except socket.gaierror, err:
+    except socket.gaierror:
         status, pr_data = 0, None
     _log.debug("status: %d, data: %s" % (status, pr_data))
     if not status == HTTP_STATUS_OK:
@@ -269,7 +268,7 @@ def download(url, path=None):
     if not sorted(tmp_files) == sorted(all_files):
         _log.error("Not all patched files were downloaded to %s: %s vs %s" % (path, tmp_files, all_files))
 
-    ec_files = [os.path.join(path, fn) for fn in tmp_files]
+    ec_files = [os.path.join(path, fname) for fname in tmp_files]
 
     return ec_files