diff --git a/minecode_pipelines/__init__.py b/minecode_pipelines/__init__.py index 1d0ca7c8..53ff10fc 100644 --- a/minecode_pipelines/__init__.py +++ b/minecode_pipelines/__init__.py @@ -8,4 +8,4 @@ # -VERSION = "0.0.1b23" +VERSION = "0.0.1b24" diff --git a/minecode_pipelines/miners/swift.py b/minecode_pipelines/miners/swift.py index ff7c43e6..5f78d99a 100644 --- a/minecode_pipelines/miners/swift.py +++ b/minecode_pipelines/miners/swift.py @@ -81,16 +81,19 @@ def split_org_repo(url_like): return org, name -# FIXME duplicated with purl2vcs.find_source_repo.get_tags_and_commits_from_git_output def get_tags_and_commits_from_git_output(git_ls_remote): """ Yield tuples of (tag, commit), given a git ls-remote output """ + tags_and_commits = [] for line in git_ls_remote.split("\n"): # line: kjwfgeklngelkfjofjeo123 refs/tags/1.2.3 line_segments = line.split("\t") # segments: ["kjwfgeklngelkfjofjeo123", "refs/tags/1.2.3"] - if len(line_segments) > 1 and line_segments[1].startswith("refs/tags/"): + if len(line_segments) > 1 and ( + line_segments[1].startswith("refs/tags/") or line_segments[1] == "HEAD" + ): commit = line_segments[0] tag = line_segments[1].replace("refs/tags/", "") - yield tag, commit + tags_and_commits.append((tag, commit)) + return tags_and_commits diff --git a/minecode_pipelines/pipes/composer.py b/minecode_pipelines/pipes/composer.py index fcc19e5c..4b7a9173 100644 --- a/minecode_pipelines/pipes/composer.py +++ b/minecode_pipelines/pipes/composer.py @@ -88,7 +88,9 @@ def mine_and_publish_composer_purls(packages, cloned_data_repo, cloned_config_re purls.append(str(base_purl)) if purls and purl_files: - commit_and_push_changes(repo=cloned_data_repo, files_to_commit=purl_files, purls=purls) + commit_and_push_changes( + repo=cloned_data_repo, files_to_commit=purl_files, purls=purls, logger=logger + ) settings_data = { "date": str(datetime.now()), diff --git a/minecode_pipelines/pipes/cran.py b/minecode_pipelines/pipes/cran.py index a71288fe..fc46d1e3 100644 --- a/minecode_pipelines/pipes/cran.py +++ b/minecode_pipelines/pipes/cran.py @@ -62,7 +62,5 @@ def mine_and_publish_cran_packageurls(cloned_data_repo, db_path, logger): # After finishing the batch, commit & push if there’s something to save if purl_files and base_purls: commit_and_push_changes( - repo=cloned_data_repo, - files_to_commit=purl_files, - purls=base_purls, + repo=cloned_data_repo, files_to_commit=purl_files, purls=base_purls, logger=logger ) diff --git a/minecode_pipelines/pipes/swift.py b/minecode_pipelines/pipes/swift.py index 2703c529..ced60a8b 100644 --- a/minecode_pipelines/pipes/swift.py +++ b/minecode_pipelines/pipes/swift.py @@ -25,7 +25,6 @@ from datetime import datetime from pathlib import Path from aboutcode import hashid -from aboutcode.hashid import get_core_purl from packageurl import PackageURL from minecode_pipelines.miners.swift import fetch_git_tags_raw @@ -55,13 +54,21 @@ def store_swift_packages(package_repo_url, tags_and_commits, cloned_data_repo): """Collect Swift package versions into purls and write them to the repo.""" org, name = split_org_repo(package_repo_url) org = "github.com/" + org - purl = PackageURL(type="swift", namespace=org, name=name) - base_purl = get_core_purl(purl) - + base_purl = PackageURL(type="swift", namespace=org, name=name) updated_purls = [] - for tag, _ in tags_and_commits: - purl = PackageURL(type="swift", namespace=org, name=name, version=tag).to_string() - updated_purls.append(purl) + + for tag, commit in tags_and_commits: + purl = None + if tag == "HEAD": + if len(tags_and_commits) == 1: + purl = PackageURL( + type="swift", namespace=org, name=name, version=commit + ).to_string() + else: + purl = PackageURL(type="swift", namespace=org, name=name, version=tag).to_string() + + if purl: + updated_purls.append(purl) purl_yaml_path = cloned_data_repo.working_dir / hashid.get_package_purls_yml_file_path( base_purl @@ -134,9 +141,7 @@ def mine_and_publish_swift_packageurls(logger): if counter >= PACKAGE_BATCH_SIZE: if purls and purl_files: commit_and_push_changes( - repo=cloned_data_repo, - files_to_commit=purl_files, - purls=purls, + repo=cloned_data_repo, files_to_commit=purl_files, purls=purls, logger=logger ) purl_files = [] @@ -159,9 +164,7 @@ def mine_and_publish_swift_packageurls(logger): if purls and purl_files: commit_and_push_changes( - repo=cloned_data_repo, - files_to_commit=purl_files, - purls=purls, + repo=cloned_data_repo, files_to_commit=purl_files, purls=purls, logger=logger ) return [swift_index_repo, cloned_data_repo, cloned_config_repo] diff --git a/minecode_pipelines/tests/data/swift/commits_tags3txt b/minecode_pipelines/tests/data/swift/commits_tags3.txt similarity index 100% rename from minecode_pipelines/tests/data/swift/commits_tags3txt rename to minecode_pipelines/tests/data/swift/commits_tags3.txt diff --git a/minecode_pipelines/tests/data/swift/commits_tags4.txt b/minecode_pipelines/tests/data/swift/commits_tags4.txt new file mode 100644 index 00000000..de60e3ee --- /dev/null +++ b/minecode_pipelines/tests/data/swift/commits_tags4.txt @@ -0,0 +1,5 @@ +2ceb8e381b9ca5e2d3004d3206411191f38907b4 HEAD +2ceb8e381b9ca5e2d3004d3206411191f38907b4 refs/heads/master +f9a3154935e90dc2fc809f7c23edc68f500a416c refs/heads/test +069e27e6d2df851474fe659f2115c33ef04198fb refs/pull/1/head +7588ba8457013a6ad80c8e2b10f21cd7d2a51070 refs/pull/2/head \ No newline at end of file diff --git a/minecode_pipelines/tests/data/swift/expected4.yaml b/minecode_pipelines/tests/data/swift/expected4.yaml new file mode 100644 index 00000000..4d1aab61 --- /dev/null +++ b/minecode_pipelines/tests/data/swift/expected4.yaml @@ -0,0 +1 @@ +- pkg:swift/github.com/0xacdc/XCFSodium@2ceb8e381b9ca5e2d3004d3206411191f38907b4 \ No newline at end of file diff --git a/minecode_pipelines/tests/pipes/test_swift.py b/minecode_pipelines/tests/pipes/test_swift.py index 81667321..cf0b6ce9 100644 --- a/minecode_pipelines/tests/pipes/test_swift.py +++ b/minecode_pipelines/tests/pipes/test_swift.py @@ -100,3 +100,20 @@ def test_swift_financial(self, mock_write): "purls.yml", ], ) + + @patch("minecode_pipelines.pipes.swift.write_data_to_yaml_file") + def test_swift_xcf_sodium(self, mock_write): + self._run_package_test( + mock_write, + package_repo_url="https://github.com/0xacdc/XCFSodium", + commits_tags_file=DATA_DIR / "commits_tags4.txt", + expected_file=DATA_DIR / "expected4.yaml", + expected_path_parts=[ + "aboutcode-packages-swift-0", + "swift", + "github.com", + "0xacdc", + "XCFSodium", + "purls.yml", + ], + ) diff --git a/pyproject-minecode_pipelines.toml b/pyproject-minecode_pipelines.toml index 1db10685..fa4eb94d 100644 --- a/pyproject-minecode_pipelines.toml +++ b/pyproject-minecode_pipelines.toml @@ -4,7 +4,7 @@ build-backend = "flot.buildapi" [project] name = "minecode_pipelines" -version = "0.0.1b23" +version = "0.0.1b24" description = "A library for mining packageURLs and package metadata from ecosystem repositories." readme = "minecode_pipelines/README.rst" license = { text = "Apache-2.0" } @@ -60,7 +60,7 @@ mine_swift = "minecode_pipelines.pipelines.mine_swift:MineSwift" mine_composer = "minecode_pipelines.pipelines.mine_composer:MineComposer" [tool.bumpversion] -current_version = "0.0.1b23" +current_version = "0.0.1b24" allow_dirty = true files = [