Skip to content

include 'sources' key to entries in list returned by collect_exts_file_info #4054

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

include 'sources' key to entries in list returned by collect_exts_file_info #4054

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions easybuild/framework/easyblock.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -725,6 +725,11 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
# copy 'path' entry to 'src' for use with extensions
'src': src['path'],
})
filename = src['name']
else:
filename = source.get('filename')
if filename is not None:
ext_src['sources'] = [filename]

else:

Expand All @@ -738,6 +743,7 @@ def collect_exts_file_info(self, fetch_files=True, verify_checksums=True):
raise EasyBuildError(error_msg, type(src_fn).__name__, src_fn)

src_fn = resolve_template(src_fn, template_values)
ext_src['sources'] = [src_fn]
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe this should be [{'filename': src_fn}]?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It should be a list of strings, see https://github.com/easybuilders/easybuild-framework/pull/4054/files#r952807882. However I found a case where it might not be and fixed that


if fetch_files:
src_path = self.obtain_file(src_fn, extension=True, urls=source_urls,
Expand Down
47 changes: 41 additions & 6 deletions test/framework/easyblock.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2350,17 +2350,25 @@ def test_collect_exts_file_info(self):
toy_sources = os.path.join(testdir, 'sandbox', 'sources', 'toy')
toy_ext_sources = os.path.join(toy_sources, 'extensions')
toy_ec_file = os.path.join(testdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-gompi-2018a-test.eb')
toy_ec = process_easyconfig(toy_ec_file)[0]

test_ec = os.path.join(self.test_prefix, 'test.eb')
new_ext_txt = "('baz', '0.0', {'nosource': True})," # With nosource option
new_ext_txt += "('barbar', '0.0', {'sources': [SOURCE_TAR_GZ]})," # With sources containing a list
test_ectxt = re.sub(r'\(name, version', new_ext_txt+r"\g<0>", read_file(toy_ec_file))
write_file(test_ec, test_ectxt)

toy_ec = process_easyconfig(test_ec)[0]
toy_eb = EasyBlock(toy_ec['ec'])

exts_file_info = toy_eb.collect_exts_file_info()

self.assertIsInstance(exts_file_info, list)
self.assertEqual(len(exts_file_info), 4)
self.assertEqual(len(exts_file_info), 6)

self.assertEqual(exts_file_info[0], {'name': 'ulimit'})

self.assertEqual(exts_file_info[1]['name'], 'bar')
self.assertEqual(exts_file_info[1]['sources'], ['bar-0.0.tar.gz'])
self.assertEqual(exts_file_info[1]['src'], os.path.join(toy_ext_sources, 'bar-0.0.tar.gz'))
bar_patch1 = 'bar-0.0_fix-silly-typo-in-printf-statement.patch'
self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1)
Expand All @@ -2370,36 +2378,63 @@ def test_collect_exts_file_info(self):
self.assertEqual(exts_file_info[1]['patches'][1]['path'], os.path.join(toy_ext_sources, bar_patch2))

self.assertEqual(exts_file_info[2]['name'], 'barbar')
self.assertEqual(exts_file_info[2]['sources'], ['barbar-1.2.tar.gz'])
self.assertEqual(exts_file_info[2]['src'], os.path.join(toy_ext_sources, 'barbar-1.2.tar.gz'))
self.assertNotIn('patches', exts_file_info[2])

self.assertEqual(exts_file_info[3]['name'], 'toy')
self.assertEqual(exts_file_info[3]['src'], os.path.join(toy_sources, 'toy-0.0.tar.gz'))
self.assertEqual(exts_file_info[3]['name'], 'baz')
self.assertNotIn('sources', exts_file_info[3])
self.assertNotIn('sources', exts_file_info[3]['options'])
self.assertNotIn('src', exts_file_info[3])
self.assertNotIn('patches', exts_file_info[3])

self.assertEqual(exts_file_info[4]['name'], 'barbar')
self.assertEqual(exts_file_info[4]['sources'], ['barbar-0.0.tar.gz'])
self.assertEqual(exts_file_info[4]['src'], os.path.join(toy_ext_sources, 'barbar-0.0.tar.gz'))
self.assertNotIn('patches', exts_file_info[4])

self.assertEqual(exts_file_info[5]['name'], 'toy')
self.assertEqual(exts_file_info[5]['sources'], ['toy-0.0.tar.gz'])
self.assertEqual(exts_file_info[5]['src'], os.path.join(toy_sources, 'toy-0.0.tar.gz'))
self.assertNotIn('patches', exts_file_info[5])

# location of files is missing when fetch_files is set to False
exts_file_info = toy_eb.collect_exts_file_info(fetch_files=False, verify_checksums=False)

self.assertIsInstance(exts_file_info, list)
self.assertEqual(len(exts_file_info), 4)
self.assertEqual(len(exts_file_info), 6)

self.assertEqual(exts_file_info[0], {'name': 'ulimit'})

self.assertEqual(exts_file_info[1]['name'], 'bar')
self.assertEqual(exts_file_info[1]['sources'], ['bar-0.0.tar.gz'])
self.assertNotIn('src', exts_file_info[1])
self.assertEqual(exts_file_info[1]['patches'][0]['name'], bar_patch1)
self.assertNotIn('path', exts_file_info[1]['patches'][0])
self.assertEqual(exts_file_info[1]['patches'][1]['name'], bar_patch2)
self.assertNotIn('path', exts_file_info[1]['patches'][1])

self.assertEqual(exts_file_info[2]['name'], 'barbar')
self.assertEqual(exts_file_info[2]['sources'], ['barbar-1.2.tar.gz'])
self.assertNotIn('src', exts_file_info[2])
self.assertNotIn('patches', exts_file_info[2])

self.assertEqual(exts_file_info[3]['name'], 'toy')
self.assertEqual(exts_file_info[3]['name'], 'baz')
self.assertNotIn('sources', exts_file_info[3])
self.assertNotIn('sources', exts_file_info[3]['options'])
self.assertNotIn('src', exts_file_info[3])
self.assertNotIn('patches', exts_file_info[3])

self.assertEqual(exts_file_info[4]['name'], 'barbar')
self.assertEqual(exts_file_info[4]['sources'], ['barbar-0.0.tar.gz'])
self.assertNotIn('src', exts_file_info[4])
self.assertNotIn('patches', exts_file_info[4])

self.assertEqual(exts_file_info[5]['name'], 'toy')
self.assertEqual(exts_file_info[5]['sources'], ['toy-0.0.tar.gz'])
self.assertNotIn('src', exts_file_info[5])
self.assertNotIn('patches', exts_file_info[5])

error_msg = "Can't verify checksums for extension files if they are not being fetched"
self.assertErrorRegex(EasyBuildError, error_msg, toy_eb.collect_exts_file_info, fetch_files=False)

Expand Down