diff --git a/easybuild/scripts/cpan2eb.py b/easybuild/scripts/cpan2eb.py new file mode 100644 index 0000000000..c32e547294 --- /dev/null +++ b/easybuild/scripts/cpan2eb.py @@ -0,0 +1,140 @@ +## +# Copyright 2013-2014 Ghent University +# +# This file is part of EasyBuild, +# originally created by the HPC team of Ghent University (http://ugent.be/hpc/en), +# with support of Ghent University (http://ugent.be/hpc), +# the Flemish Supercomputer Centre (VSC) (https://vscentrum.be/nl/en), +# the Hercules foundation (http://www.herculesstichting.be/in_English) +# and the Department of Economy, Science and Innovation (EWI) (http://www.ewi-vlaanderen.be/en). +# +# http://github.com/hpcugent/easybuild +# +# EasyBuild is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation v2. +# +# EasyBuild is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with EasyBuild. If not, see . +## +""" +This script takes a Perl module name as argument, and generates +a string compatible with the easyconfig 1.x format with metadata about the module +and all it's dependencies + +@author: Jens Timmerman +""" +from urllib2 import HTTPError +from vsc.utils.rest import RestClient +from vsc.utils.generaloption import simple_option +from vsc.utils import fancylogger + + +logger = fancylogger.getLogger() + + +class CpanMeta(object): + """This class gets meta information from cpan + + This uses the metacpan.org api + """ + def __init__(self): + """Constructor""" + dummy = {'download_url': 'example.com/bla', 'release': '0', 'version': '0', 'distribution': 'ExtUtils-MakeMaker', + 'modulename': 'ExtUtils::MakeMaker'} + self.cache = {'ExtUtils::MakeMaker': dummy, 'perl': dummy} + self.graph = {'ExtUtils::MakeMaker': [], 'perl': []} + self.client = RestClient('http://api.metacpan.org') + + def get_module_data(self, modulename): + """Get some metadata about the current version of the module""" + try: + json = self.client.v0.module[modulename].get()[1] + depsjson = self.client.v0.release[json['author']][json['release']].get() + except HTTPError: + logger.error("API error for getting %s this will have to be resolved manually", modulename) + return {'release': '0', 'dependency': [] } + depsjson = depsjson[1] + depsjson.update(json) + depsjson.update({'modulename': modulename}) + return depsjson + + def get_recursive_data(self, modulename): + """Recursively get all data (so for all dependencies)""" + # check if we have been here before + if modulename in self.cache: + logger.info('%s module cached ', modulename) + return self.graph + data = self.get_module_data(modulename) + self.cache[modulename] = data + # module's are somtimes included in a release we already know, so skip this also + if data['release'] in self.cache: + logger.info('%s release cached', data['release']) + return self.graph + self.cache[data['release']] = data + dependencies = set() + # do the recursive thing + for dep in data['dependency']: + if "requires" in dep["relationship"]: + # we filter on dependendencies for the build and configure phase, otherwise we end up with circular + # dependencies + if dep['phase'] in ('build', 'configure') : + self.get_recursive_data(dep['module']) + dependencies.add(dep['module']) + self.graph[modulename] = dependencies + return self.graph + + +def post_order(graph, root): + """Walk the graph from the given root in a post-order manner, by providing the correspoding generator.""" + for node in graph[root]: + for child in post_order(graph, node): + yield child + yield root + + +def topological_sort(graph, root): + """Perform a topological sorting of the given graph. + + The graph needs to be in the following format: + + g = { t1: [t2, t3], + t2: [t4, t5, t6], + t3: [] + t4: [] + t5: [t3] + t6: [t5] + } + + where each node is mapped to a list of nodes it has an edge to. + + @returns: generator for traversing the graph in the desired order + """ + visited = set() + for node in post_order(graph, root): + if node not in visited: + yield node + visited.add(node) # been there, done that. + + +go = simple_option() + +cpan = CpanMeta() +modules = cpan.get_recursive_data(go.args[0]) +print modules + +# topological soft, so we get correct dependencies order +for module in topological_sort(modules, go.args[0]): + data = cpan.cache[module] + url, name = data['download_url'].rsplit("/", 1) + data.update({'url': url, 'tarball': name}) # distribution sometimes contains subdirs + if data['release'] is not '0' and data['version'] is not '0': + print """('%(modulename)s', '%(version)s', { + 'source_tmpl': '%(tarball)s', + 'source_urls': ['%(url)s'], + }),""" % data diff --git a/easybuild/tools/github.py b/easybuild/tools/github.py index ae544e28b4..9b0bb0ac52 100644 --- a/easybuild/tools/github.py +++ b/easybuild/tools/github.py @@ -30,7 +30,6 @@ """ import base64 import os -import re import socket import tempfile import urllib @@ -226,7 +225,7 @@ def download(url, path=None): pr_url = g.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr] try: status, pr_data = pr_url.get() - except socket.gaierror, err: + except socket.gaierror: status, pr_data = 0, None _log.debug("status: %d, data: %s" % (status, pr_data)) if not status == HTTP_STATUS_OK: @@ -269,7 +268,7 @@ def download(url, path=None): if not sorted(tmp_files) == sorted(all_files): _log.error("Not all patched files were downloaded to %s: %s vs %s" % (path, tmp_files, all_files)) - ec_files = [os.path.join(path, fn) for fn in tmp_files] + ec_files = [os.path.join(path, fname) for fname in tmp_files] return ec_files