diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index 763c5f27ee6..00000000000 --- a/.github/CODEOWNERS +++ /dev/null @@ -1,17 +0,0 @@ -# Pull requests concerning the listed files will automatically invite the respective maintainers as reviewers. -# This file is not used for denoting any kind of ownership, but is merely a tool for handling notifications. -# -# Merge permissions are required for maintaining an entry in this file. -# For documentation on this mechanism, see https://help.github.com/articles/about-codeowners/ - -# Default reviewers if nothing else matches -* @edolstra - -# This file -.github/CODEOWNERS @edolstra - -# Documentation of built-in functions -src/libexpr/primops.cc @roberth - -# Libstore layer -/src/libstore @ericson2314 diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index af94c3e9e5b..08a5851748d 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,10 +1,9 @@ --- name: Bug report about: Report unexpected or incorrect behaviour -title: '' +title: "" labels: bug -assignees: '' - +assignees: "" --- ## Describe the bug @@ -32,7 +31,9 @@ assignees: '' ## Metadata - + + + ## Additional context @@ -42,13 +43,9 @@ assignees: '' -- [ ] checked [latest Nix manual] \([source]) +- [ ] checked [latest Determinate Nix manual] \([source]) - [ ] checked [open bug issues and pull requests] for possible duplicates -[latest Nix manual]: https://nix.dev/manual/nix/development/ -[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source -[open bug issues and pull requests]: https://github.com/NixOS/nix/labels/bug - ---- - -Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). +[latest Determinate Nix manual]: https://manual.determinate.systems/ +[source]: https://github.com/DeterminateSystems/nix-src/tree/main/doc/manual/source +[open bug issues and pull requests]: https://github.com/DeterminateSystems/nix-src/labels/bug diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index fe9f9dd209d..b88e1093798 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,10 +1,9 @@ --- name: Feature request about: Suggest a new feature -title: '' +title: "" labels: feature -assignees: '' - +assignees: "" --- ## Is your feature request related to a problem? @@ -27,13 +26,9 @@ assignees: '' -- [ ] checked [latest Nix manual] \([source]) -- [ ] checked [open feature issues and pull requests] for possible duplicates - -[latest Nix manual]: https://nix.dev/manual/nix/development/ -[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source -[open feature issues and pull requests]: https://github.com/NixOS/nix/labels/feature - ---- +- [ ] checked [latest Determinate Nix manual] \([source]) +- [ ] checked [open bug issues and pull requests] for possible duplicates -Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). +[latest Determinate Nix manual]: https://manual.determinate.systems/ +[source]: https://github.com/DeterminateSystems/nix-src/tree/main/doc/manual/source +[open bug issues and pull requests]: https://github.com/DeterminateSystems/nix-src/labels/bug diff --git a/.github/ISSUE_TEMPLATE/installer.md b/.github/ISSUE_TEMPLATE/installer.md index 070e0bd9b25..430bef971aa 100644 --- a/.github/ISSUE_TEMPLATE/installer.md +++ b/.github/ISSUE_TEMPLATE/installer.md @@ -1,18 +1,17 @@ --- name: Installer issue about: Report problems with installation -title: '' +title: "" labels: installer -assignees: '' - +assignees: "" --- ## Platform - + -- [ ] Linux: - [ ] macOS +- [ ] Linux: - [ ] WSL ## Additional information @@ -35,13 +34,9 @@ assignees: '' -- [ ] checked [latest Nix manual] \([source]) -- [ ] checked [open installer issues and pull requests] for possible duplicates - -[latest Nix manual]: https://nix.dev/manual/nix/development/ -[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source -[open installer issues and pull requests]: https://github.com/NixOS/nix/labels/installer - ---- +- [ ] checked [latest Determinate Nix manual] \([source]) +- [ ] checked [open bug issues and pull requests] for possible duplicates -Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). +[latest Determinate Nix manual]: https://manual.determinate.systems/ +[source]: https://github.com/DeterminateSystems/nix-src/tree/main/doc/manual/source +[open bug issues and pull requests]: https://github.com/DeterminateSystems/nix-src/labels/bug diff --git a/.github/ISSUE_TEMPLATE/missing_documentation.md b/.github/ISSUE_TEMPLATE/missing_documentation.md index 4e05b626d39..fcdd0d20135 100644 --- a/.github/ISSUE_TEMPLATE/missing_documentation.md +++ b/.github/ISSUE_TEMPLATE/missing_documentation.md @@ -1,10 +1,9 @@ --- name: Missing or incorrect documentation about: Help us improve the reference manual -title: '' +title: "" labels: documentation -assignees: '' - +assignees: "" --- ## Problem @@ -19,13 +18,9 @@ assignees: '' -- [ ] checked [latest Nix manual] \([source]) -- [ ] checked [open documentation issues and pull requests] for possible duplicates - -[latest Nix manual]: https://nix.dev/manual/nix/development/ -[source]: https://github.com/NixOS/nix/tree/master/doc/manual/source -[open documentation issues and pull requests]: https://github.com/NixOS/nix/labels/documentation - ---- +- [ ] checked [latest Determinate Nix manual] \([source]) +- [ ] checked [open bug issues and pull requests] for possible duplicates -Add :+1: to [issues you find important](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc). +[latest Determinate Nix manual]: https://manual.determinate.systems/ +[source]: https://github.com/DeterminateSystems/nix-src/tree/main/doc/manual/source +[open bug issues and pull requests]: https://github.com/DeterminateSystems/nix-src/labels/bug diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index c6843d86fa7..d3e1f817736 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,22 +1,3 @@ - - ## Motivation @@ -30,9 +11,3 @@ so you understand the process and the expectations. - ---- - -Add :+1: to [pull requests you find important](https://github.com/NixOS/nix/pulls?q=is%3Aopen+sort%3Areactions-%2B1-desc). - -The Nix maintainer team uses a [GitHub project board](https://github.com/orgs/NixOS/projects/19) to [schedule and track reviews](https://github.com/NixOS/nix/tree/master/maintainers#project-board-protocol). diff --git a/.github/STALE-BOT.md b/.github/STALE-BOT.md index bc0005413f1..281d0f79a8b 100644 --- a/.github/STALE-BOT.md +++ b/.github/STALE-BOT.md @@ -2,34 +2,21 @@ - Thanks for your contribution! - To remove the stale label, just leave a new comment. -- _How to find the right people to ping?_ → [`git blame`](https://git-scm.com/docs/git-blame) to the rescue! (or GitHub's history and blame buttons.) -- You can always ask for help on [our Discourse Forum](https://discourse.nixos.org/) or on [Matrix - #users:nixos.org](https://matrix.to/#/#users:nixos.org). +- You can always ask for help on [Discord](https://determinate.systems/discord). ## Suggestions for PRs -1. GitHub sometimes doesn't notify people who commented / reviewed a PR previously, when you (force) push commits. If you have addressed the reviews you can [officially ask for a review](https://docs.github.com/en/free-pro-team@latest/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from those who commented to you or anyone else. -2. If it is unfinished but you plan to finish it, please mark it as a draft. -3. If you don't expect to work on it any time soon, closing it with a short comment may encourage someone else to pick up your work. -4. To get things rolling again, rebase the PR against the target branch and address valid comments. -5. If you need a review to move forward, ask in [the Discourse thread for PRs that need help](https://discourse.nixos.org/t/prs-in-distress/3604). -6. If all you need is a merge, check the git history to find and [request reviews](https://docs.github.com/en/github/collaborating-with-issues-and-pull-requests/requesting-a-pull-request-review) from people who usually merge related contributions. +1. If it is unfinished but you plan to finish it, please mark it as a draft. +1. If you don't expect to work on it any time soon, closing it with a short comment may encourage someone else to pick up your work. +1. To get things rolling again, rebase the PR against the target branch and address valid comments. +1. If you need a review to move forward, ask in [Discord](https://determinate.systems/discord). ## Suggestions for issues 1. If it is resolved (either for you personally, or in general), please consider closing it. 2. If this might still be an issue, but you are not interested in promoting its resolution, please consider closing it while encouraging others to take over and reopen an issue if they care enough. -3. If you still have interest in resolving it, try to ping somebody who you believe might have an interest in the topic. Consider discussing the problem in [our Discourse Forum](https://discourse.nixos.org/). -4. As with all open source projects, your best option is to submit a Pull Request that addresses this issue. We :heart: this attitude! +3. If you still have interest in resolving it, try to ping somebody who you believe might have an interest in the topic. Consider discussing the problem in [Discord](https://determinate.systems/discord). **Memorandum on closing issues** Don't be afraid to close an issue that holds valuable information. Closed issues stay in the system for people to search, read, cross-reference, or even reopen--nothing is lost! Closing obsolete issues is an important way to help maintainers focus their time and effort. - -## Useful GitHub search queries - -- [Open PRs with any stale-bot interaction](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+) -- [Open PRs with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22) -- [Open PRs with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/pulls?q=is%3Apr+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+) -- [Open Issues with any stale-bot interaction](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+) -- [Open Issues with any stale-bot interaction and `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+label%3A%22stale%22+) -- [Open Issues with any stale-bot interaction and NOT `stale`](https://github.com/NixOS/nix/issues?q=is%3Aissue+is%3Aopen+commenter%3Aapp%2Fstale+-label%3A%22stale%22+) diff --git a/.github/release-notes.sh b/.github/release-notes.sh new file mode 100755 index 00000000000..f641e146d2e --- /dev/null +++ b/.github/release-notes.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +# SC2002 disables "useless cat" warnings. +# I prefer pipelines that start with an explicit input, and go from there. +# Overly fussy. +# shellcheck disable=SC2002 + +scratch=$(mktemp -d -t tmp.XXXXXXXXXX) +finish() { + rm -rf "$scratch" +} +trap finish EXIT + +DATE=$(date +%Y-%m-%d) +DETERMINATE_NIX_VERSION=$(cat .version-determinate) +TAG_NAME="v${DETERMINATE_NIX_VERSION}" +NIX_VERSION=$(cat .version) +NIX_VERSION_MAJOR_MINOR=$(echo "$NIX_VERSION" | cut -d. -f1,2) +GITHUB_REPOSITORY="${GITHUB_REPOSITORY:-DeterminateSystems/nix-src}" + +gh api "/repos/${GITHUB_REPOSITORY}/releases/generate-notes" \ + -f "tag_name=${TAG_NAME}" > "$scratch/notes.json" + +trim_trailing_newlines() { + local text + text="$(cat)" + echo -n "${text}" +} + +linkify_gh() { + sed \ + -e 's!\(https://github.com/DeterminateSystems/nix-src/\(pull\|issue\)/\([[:digit:]]\+\)\)![DeterminateSystems/nix-src#\3](\1)!' \ + -e 's#\(https://github.com/DeterminateSystems/nix-src/compare/\([^ ]\+\)\)#[\2](\1)#' +} + +( + cat doc/manual/source/release-notes-determinate/changes.md \ + | sed 's/^.*\(\)$/This section lists the differences between upstream Nix '"$NIX_VERSION_MAJOR_MINOR"' and Determinate Nix '"$DETERMINATE_NIX_VERSION"'.\1/' \ + + printf "\n\n" "$DETERMINATE_NIX_VERSION" + cat "$scratch/notes.json" \ + | jq -r .body \ + | grep -v '^#' \ + | grep -v "Full Changelog" \ + | trim_trailing_newlines \ + | sed -e 's/^\* /\n* /' \ + | linkify_gh + echo "" # final newline +) > "$scratch/changes.md" + +( + printf "# Release %s (%s)\n\n" \ + "$DETERMINATE_NIX_VERSION" \ + "$DATE" + printf "* Based on [upstream Nix %s](../release-notes/rl-%s.md).\n\n" \ + "$NIX_VERSION" \ + "$NIX_VERSION_MAJOR_MINOR" + + cat "$scratch/notes.json" | jq -r .body | linkify_gh +) > "$scratch/rl.md" + +( + cat doc/manual/source/SUMMARY.md.in \ + | sed 's/\(\)$/\1\n - [Release '"$DETERMINATE_NIX_VERSION"' ('"$DATE"')](release-notes-determinate\/'"$TAG_NAME"'.md)/' +) > "$scratch/summary.md" + +mv "$scratch/changes.md" doc/manual/source/release-notes-determinate/changes.md +mv "$scratch/rl.md" "doc/manual/source/release-notes-determinate/v${DETERMINATE_NIX_VERSION}.md" +mv "$scratch/summary.md" doc/manual/source/SUMMARY.md.in diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000000..dd98d0d00f9 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,241 @@ +on: + workflow_call: + inputs: + system: + required: true + type: string + runner: + required: true + type: string + runner_for_virt: + required: true + type: string + runner_small: + required: true + type: string + if: + required: false + default: true + type: boolean + run_tests: + required: false + default: true + type: boolean + run_vm_tests: + required: false + default: false + type: boolean + run_regression_tests: + required: false + default: false + type: boolean + publish_manual: + required: false + default: false + type: boolean + secrets: + manual_netlify_auth_token: + required: false + manual_netlify_site_id: + required: false + +jobs: + build: + if: ${{ inputs.if }} + strategy: + fail-fast: false + runs-on: ${{ inputs.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: nix build .#packages.${{ inputs.system }}.default .#packages.${{ inputs.system }}.binaryTarball --no-link -L + - run: nix build .#packages.${{ inputs.system }}.binaryTarball --out-link tarball + - uses: actions/upload-artifact@v4 + with: + name: ${{ inputs.system }} + path: ./tarball/*.xz + + test: + if: ${{ inputs.if && inputs.run_tests}} + needs: build + strategy: + fail-fast: false + runs-on: ${{ inputs.runner }} + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: nix flake check -L --system ${{ inputs.system }} + + vm_tests_smoke: + if: inputs.run_vm_tests && github.event_name != 'merge_group' + needs: build + runs-on: ${{ inputs.runner_for_virt }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: | + nix build -L \ + .#hydraJobs.tests.functional_user \ + .#hydraJobs.tests.githubFlakes \ + .#hydraJobs.tests.nix-docker \ + .#hydraJobs.tests.tarballFlakes \ + ; + + vm_tests_all: + if: inputs.run_vm_tests && github.event_name == 'merge_group' + needs: build + runs-on: ${{ inputs.runner_for_virt }} + steps: + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - run: | + cmd() { + nix build -L --keep-going --timeout 600 \ + $(nix flake show --json \ + | jq -r ' + .hydraJobs.tests + | with_entries(select(.value.type == "derivation")) + | keys[] + | ".#hydraJobs.tests." + .') + } + + if ! cmd; then + echo "failed, retrying once ..." + printf "\n\n\n\n\n\n\n\n" + cmd + fi + + flake_regressions: + if: | + (inputs.run_regression_tests && github.event_name == 'merge_group') + || ( + inputs.run_regression_tests + && github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + && ( + (github.event.action == 'labeled' && github.event.label.name == 'flake-regression-test') + || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'flake-regression-test')) + ) + ) + needs: build + runs-on: ${{ inputs.runner }} + strategy: + matrix: + nix_config: + - "lazy-trees = true" + - "lazy-trees = false" + - "eval-cores = 24" + glob: + - "[0-d]*" + - "[e-l]*" + - "[m]*" + - "[n-r]*" + - "[s-z]*" + + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - name: Checkout flake-regressions + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions + path: flake-regressions + - name: Checkout flake-regressions-data + uses: actions/checkout@v4 + with: + repository: DeterminateSystems/flake-regressions-data + path: flake-regressions/tests + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - name: Run flake regression tests + env: + #PARALLEL: ${{ !contains(matrix.nix_config, 'eval-cores') && '-P 50%' || '-P 1' }} + PARALLEL: '-P 1' + FLAKE_REGRESSION_GLOB: ${{ matrix.glob }} + NIX_CONFIG: ${{ matrix.nix_config }} + PREFETCH: "1" + run: | + set -x + echo "PARALLEL: $PARALLEL" + echo "NIX_CONFIG: $NIX_CONFIG" + if [ ! -z "${NSC_CACHE_PATH:-}" ]; then + mkdir -p "${NSC_CACHE_PATH}/nix/xdg-cache" + export XDG_CACHE_HOME="${NSC_CACHE_PATH}/nix/xdg-cache" + fi + nix build -L --out-link ./new-nix + export PATH=$(pwd)/new-nix/bin:$PATH + [[ $(type -p nix) = $(pwd)/new-nix/bin/nix ]] + + nix config show lazy-trees + nix config show eval-cores + lscpu + nproc + + if ! flake-regressions/eval-all.sh; then + echo "Some failed, trying again" + printf "\n\n\n\n\n\n\n\n" + NIX_REMOTE=/tmp/nix flake-regressions/eval-all.sh + fi + + manual: + if: github.event_name != 'merge_group' + needs: build + runs-on: ${{ inputs.runner_small }} + permissions: + id-token: "write" + contents: "read" + pull-requests: "write" + statuses: "write" + deployments: "write" + steps: + - name: Checkout nix + uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-cache-action@main + - name: Build manual + if: inputs.system == 'x86_64-linux' + run: nix build .#hydraJobs.manual + - uses: nwtgck/actions-netlify@v3.0 + if: inputs.publish_manual && inputs.system == 'x86_64-linux' + with: + publish-dir: "./result/share/doc/nix/manual" + production-branch: main + github-token: ${{ secrets.GITHUB_TOKEN }} + deploy-message: "Deploy from GitHub Actions" + # NOTE(cole-h): We have a perpetual PR displaying our changes against upstream open, but + # its conversation is locked, so this PR comment can never be posted. + # https://github.com/DeterminateSystems/nix-src/pull/165 + enable-pull-request-comment: ${{ github.event.pull_request.number != 165 }} + enable-commit-comment: true + enable-commit-status: true + overwrites-pull-request-comment: true + env: + NETLIFY_AUTH_TOKEN: ${{ secrets.manual_netlify_auth_token }} + NETLIFY_SITE_ID: ${{ secrets.manual_netlify_site_id }} + + success: + needs: + - build + - test + - vm_tests_smoke + - vm_tests_all + - flake_regressions + - manual + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - run: "true" + - run: | + echo "A dependent in the build matrix failed:" + echo "$needs" + exit 1 + env: + needs: ${{ toJSON(needs) }} + if: | + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7e103b6320..29b6cbf36ea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,280 +2,178 @@ name: "CI" on: pull_request: - merge_group: push: - workflow_dispatch: - inputs: - dogfood: - description: 'Use dogfood Nix build' - required: false - default: true - type: boolean + branches: + # NOTE: make sure any branches here are also valid directory names, + # otherwise creating the directory and uploading to s3 will fail + - main + - master + merge_group: + release: + types: + - published -permissions: read-all +permissions: + id-token: "write" + contents: "read" + pull-requests: "write" + statuses: "write" + deployments: "write" jobs: eval: - runs-on: ubuntu-24.04 + runs-on: UbuntuLatest32Cores128G steps: - - uses: actions/checkout@v5 - with: - fetch-depth: 0 - - uses: ./.github/actions/install-nix-action - with: - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} - extra_nix_config: - experimental-features = nix-command flakes - github_token: ${{ secrets.GITHUB_TOKEN }} - - run: nix flake show --all-systems --json + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: DeterminateSystems/determinate-nix-action@main + - run: nix flake show --all-systems --json - tests: - strategy: - fail-fast: false - matrix: - include: - - scenario: on ubuntu - runs-on: ubuntu-24.04 - os: linux - instrumented: false - primary: true - stdenv: stdenv - - scenario: on macos - runs-on: macos-14 - os: darwin - instrumented: false - primary: true - stdenv: stdenv - - scenario: on ubuntu (with sanitizers / coverage) - runs-on: ubuntu-24.04 - os: linux - instrumented: true - primary: false - stdenv: clangStdenv - name: tests ${{ matrix.scenario }} - runs-on: ${{ matrix.runs-on }} - timeout-minutes: 60 - steps: - - uses: actions/checkout@v5 - with: - fetch-depth: 0 - - uses: ./.github/actions/install-nix-action - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} - # The sandbox would otherwise be disabled by default on Darwin - extra_nix_config: "sandbox = true" - - uses: DeterminateSystems/magic-nix-cache-action@main - # Since ubuntu 22.30, unprivileged usernamespaces are no longer allowed to map to the root user: - # https://ubuntu.com/blog/ubuntu-23-10-restricted-unprivileged-user-namespaces - - run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - if: matrix.os == 'linux' - - name: Run component tests - run: | - nix build --file ci/gha/tests/wrapper.nix componentTests -L \ - --arg withInstrumentation ${{ matrix.instrumented }} \ - --argstr stdenv "${{ matrix.stdenv }}" - - name: Run flake checks and prepare the installer tarball - run: | - ci/gha/tests/build-checks - ci/gha/tests/prepare-installer-for-github-actions - if: ${{ matrix.primary }} - - name: Collect code coverage - run: | - nix build --file ci/gha/tests/wrapper.nix codeCoverage.coverageReports -L \ - --arg withInstrumentation ${{ matrix.instrumented }} \ - --argstr stdenv "${{ matrix.stdenv }}" \ - --out-link coverage-reports - cat coverage-reports/index.txt >> $GITHUB_STEP_SUMMARY - if: ${{ matrix.instrumented }} - - name: Upload coverage reports - uses: actions/upload-artifact@v4 - with: - name: coverage-reports - path: coverage-reports/ - if: ${{ matrix.instrumented }} - - name: Upload installer tarball - uses: actions/upload-artifact@v4 - with: - name: installer-${{matrix.os}} - path: out/* - if: ${{ matrix.primary }} + build_x86_64-linux: + uses: ./.github/workflows/build.yml + with: + system: x86_64-linux + runner: namespace-profile-linuxamd32c64g-cache + runner_for_virt: UbuntuLatest32Cores128G + runner_small: ubuntu-latest + run_tests: true + run_vm_tests: true + run_regression_tests: true + publish_manual: true + secrets: + manual_netlify_auth_token: ${{ secrets.NETLIFY_AUTH_TOKEN }} + manual_netlify_site_id: ${{ secrets.NETLIFY_SITE_ID }} - installer_test: - needs: [tests] - strategy: - fail-fast: false - matrix: - include: - - scenario: on ubuntu - runs-on: ubuntu-24.04 - os: linux - - scenario: on macos - runs-on: macos-14 - os: darwin - name: installer test ${{ matrix.scenario }} - runs-on: ${{ matrix.runs-on }} - steps: - - uses: actions/checkout@v5 - - name: Download installer tarball - uses: actions/download-artifact@v5 - with: - name: installer-${{matrix.os}} - path: out - - name: Looking up the installer tarball URL - id: installer-tarball-url - run: echo "installer-url=file://$GITHUB_WORKSPACE/out" >> "$GITHUB_OUTPUT" - - uses: cachix/install-nix-action@v31 - with: - install_url: ${{ format('{0}/install', steps.installer-tarball-url.outputs.installer-url) }} - install_options: ${{ format('--tarball-url-prefix {0}', steps.installer-tarball-url.outputs.installer-url) }} - - run: sudo apt install fish zsh - if: matrix.os == 'linux' - - run: brew install fish - if: matrix.os == 'darwin' - - run: exec bash -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec sh -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec zsh -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec fish -c "nix-instantiate -E 'builtins.currentTime' --eval" - - run: exec bash -c "nix-channel --add https://releases.nixos.org/nixos/unstable/nixos-23.05pre466020.60c1d71f2ba nixpkgs" - - run: exec bash -c "nix-channel --update && nix-env -iA nixpkgs.hello && hello" + build_aarch64-linux: + uses: ./.github/workflows/build.yml + with: + if: ${{ + github.event_name != 'pull_request' + || ( + github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + && ( + (github.event.action == 'labeled' && github.event.label.name == 'upload to s3') + || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'upload to s3')) + ) + ) + }} + system: aarch64-linux + runner: UbuntuLatest32Cores128GArm + runner_for_virt: UbuntuLatest32Cores128GArm + runner_small: UbuntuLatest32Cores128GArm - # Steps to test CI automation in your own fork. - # 1. Sign-up for https://hub.docker.com/ - # 2. Store your dockerhub username as DOCKERHUB_USERNAME in "Repository secrets" of your fork repository settings (https://github.com/$githubuser/nix/settings/secrets/actions) - # 3. Create an access token in https://hub.docker.com/settings/security and store it as DOCKERHUB_TOKEN in "Repository secrets" of your fork - check_secrets: - permissions: - contents: none - name: Check presence of secrets - runs-on: ubuntu-24.04 - outputs: - docker: ${{ steps.secret.outputs.docker }} + build_x86_64-darwin: + uses: ./.github/workflows/build.yml + with: + if: ${{ + github.event_name != 'pull_request' + || ( + github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + && ( + (github.event.action == 'labeled' && github.event.label.name == 'upload to s3') + || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'upload to s3')) + ) + ) + }} + system: x86_64-darwin + runner: macos-latest-large + runner_for_virt: macos-latest-large + runner_small: macos-latest-large + run_tests: false + + build_aarch64-darwin: + uses: ./.github/workflows/build.yml + with: + system: aarch64-darwin + runner: namespace-profile-mac-m2-12c28g + runner_for_virt: namespace-profile-mac-m2-12c28g + runner_small: macos-latest-xlarge + + success: + runs-on: ubuntu-latest + needs: + - eval + - build_x86_64-linux + - build_aarch64-linux + - build_x86_64-darwin + - build_aarch64-darwin + if: ${{ always() }} steps: - - name: Check for DockerHub secrets - id: secret + - run: "true" + - run: | + echo "A dependent in the build matrix failed:" + echo "$needs" + exit 1 env: - _DOCKER_SECRETS: ${{ secrets.DOCKERHUB_USERNAME }}${{ secrets.DOCKERHUB_TOKEN }} - run: | - echo "docker=${{ env._DOCKER_SECRETS != '' }}" >> $GITHUB_OUTPUT + needs: ${{ toJSON(needs) }} + if: | + contains(needs.*.result, 'failure') || + contains(needs.*.result, 'cancelled') - docker_push_image: - needs: [tests, vm_tests, check_secrets] - permissions: - contents: read - packages: write - if: >- - needs.check_secrets.outputs.docker == 'true' && - github.event_name == 'push' && - github.ref_name == 'master' - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v5 - with: - fetch-depth: 0 - - uses: cachix/install-nix-action@v31 - with: - install_url: https://releases.nixos.org/nix/nix-2.20.3/install - - uses: DeterminateSystems/magic-nix-cache-action@main - - run: echo NIX_VERSION="$(nix --experimental-features 'nix-command flakes' eval .\#nix.version | tr -d \")" >> $GITHUB_ENV - - run: nix --experimental-features 'nix-command flakes' build .#dockerImage -L - - run: docker load -i ./result/image.tar.gz - - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION - - run: docker tag nix:$NIX_VERSION ${{ secrets.DOCKERHUB_USERNAME }}/nix:master - # We'll deploy the newly built image to both Docker Hub and Github Container Registry. - # - # Push to Docker Hub first - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:$NIX_VERSION - - run: docker push ${{ secrets.DOCKERHUB_USERNAME }}/nix:master - # Push to GitHub Container Registry as well - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - name: Push image - run: | - IMAGE_ID=ghcr.io/${{ github.repository_owner }}/nix - # Change all uppercase to lowercase - IMAGE_ID=$(echo $IMAGE_ID | tr '[A-Z]' '[a-z]') + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main - docker tag nix:$NIX_VERSION $IMAGE_ID:$NIX_VERSION - docker tag nix:$NIX_VERSION $IMAGE_ID:latest - docker push $IMAGE_ID:$NIX_VERSION - docker push $IMAGE_ID:latest - # deprecated 2024-02-24 - docker tag nix:$NIX_VERSION $IMAGE_ID:master - docker push $IMAGE_ID:master + - name: Create artifacts directory + run: mkdir -p ./artifacts - vm_tests: - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v5 - - uses: ./.github/actions/install-nix-action + - name: Fetch artifacts + uses: actions/download-artifact@v4 with: - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} - extra_nix_config: - experimental-features = nix-command flakes - github_token: ${{ secrets.GITHUB_TOKEN }} - - uses: DeterminateSystems/magic-nix-cache-action@main - - run: | - nix build -L \ - .#hydraJobs.tests.functional_user \ - .#hydraJobs.tests.githubFlakes \ - .#hydraJobs.tests.nix-docker \ - .#hydraJobs.tests.tarballFlakes \ - ; + path: downloaded + - name: Move downloaded artifacts to artifacts directory + run: | + for dir in ./downloaded/*; do + arch="$(basename "$dir")" + mv "$dir"/*.xz ./artifacts/"${arch}" + done - flake_regressions: - needs: vm_tests - runs-on: ubuntu-24.04 - steps: - - name: Checkout nix - uses: actions/checkout@v5 - - name: Checkout flake-regressions - uses: actions/checkout@v5 - with: - repository: NixOS/flake-regressions - path: flake-regressions - - name: Checkout flake-regressions-data - uses: actions/checkout@v5 - with: - repository: NixOS/flake-regressions-data - path: flake-regressions/tests - - uses: ./.github/actions/install-nix-action + - name: Build fallback-paths.nix + if: ${{ + github.event_name != 'pull_request' + || ( + github.event.pull_request.head.repo.full_name == 'DeterminateSystems/nix-src' + && ( + (github.event.action == 'labeled' && github.event.label.name == 'upload to s3') + || (github.event.action != 'labeled' && contains(github.event.pull_request.labels.*.name, 'upload to s3')) + ) + ) + }} + run: | + nix build .#fallbackPathsNix --out-link fallback + cat fallback > ./artifacts/fallback-paths.nix + + - uses: DeterminateSystems/push-artifact-ids@main with: - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} - extra_nix_config: - experimental-features = nix-command flakes - github_token: ${{ secrets.GITHUB_TOKEN }} - - uses: DeterminateSystems/magic-nix-cache-action@main - - run: nix build -L --out-link ./new-nix && PATH=$(pwd)/new-nix/bin:$PATH MAX_FLAKES=25 flake-regressions/eval-all.sh + s3_upload_role: ${{ secrets.AWS_S3_UPLOAD_ROLE_ARN }} + bucket: ${{ secrets.AWS_S3_UPLOAD_BUCKET_NAME }} + directory: ./artifacts + ids_project_name: determinate-nix + ids_binary_prefix: determinate-nix + skip_acl: true + allowed_branches: '["main"]' - profile_build: - needs: tests - runs-on: ubuntu-24.04 - timeout-minutes: 60 - if: >- - github.event_name == 'push' && - github.ref_name == 'master' + publish: + needs: + - success + if: (!github.repository.fork && (github.ref == format('refs/heads/{0}', github.event.repository.default_branch) || startsWith(github.ref, 'refs/tags/'))) + environment: ${{ github.event_name == 'release' && 'production' || '' }} + runs-on: ubuntu-latest + permissions: + contents: write + id-token: write steps: - - uses: actions/checkout@v5 - with: - fetch-depth: 0 - - uses: ./.github/actions/install-nix-action - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - dogfood: ${{ github.event_name == 'workflow_dispatch' && inputs.dogfood || github.event_name != 'workflow_dispatch' }} - extra_nix_config: | - experimental-features = flakes nix-command ca-derivations impure-derivations - max-jobs = 1 - - uses: DeterminateSystems/magic-nix-cache-action@main - - run: | - nix build -L --file ./ci/gha/profile-build buildTimeReport --out-link build-time-report.md - cat build-time-report.md >> $GITHUB_STEP_SUMMARY + - uses: actions/checkout@v4 + - uses: DeterminateSystems/determinate-nix-action@main + - uses: DeterminateSystems/flakehub-push@main + with: + rolling: ${{ github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} + visibility: "public" + tag: "${{ github.ref_name }}" + - name: Update the release notes + if: startsWith(github.ref, 'refs/tags/') + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG_NAME: ${{ github.ref_name }} + run: | + gh release edit "$TAG_NAME" --notes-file doc/manual/source/release-notes-determinate/"$TAG_NAME".md || true diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 23a5d9e51fc..16038cb213c 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-24.04 if: github.repository_owner == 'NixOS' steps: - - uses: actions/labeler@v5 + - uses: actions/labeler@v6 with: repo-token: ${{ secrets.GITHUB_TOKEN }} sync-labels: false diff --git a/.github/workflows/propose-release.yml b/.github/workflows/propose-release.yml new file mode 100644 index 00000000000..ea01e4b7afe --- /dev/null +++ b/.github/workflows/propose-release.yml @@ -0,0 +1,32 @@ +on: + workflow_dispatch: + inputs: + reference-id: + type: string + required: true + version: + type: string + required: true + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: true + +jobs: + propose-release: + uses: DeterminateSystems/propose-release/.github/workflows/workflow.yml@main + permissions: + id-token: write + contents: write + pull-requests: write + with: + update-flake: false + reference-id: ${{ inputs.reference-id }} + version: ${{ inputs.version }} + extra-commands-early: | + echo ${{ inputs.version }} > .version-determinate + git add .version-determinate + git commit -m "Set .version-determinate to ${{ inputs.version }}" || true + ./.github/release-notes.sh + git add doc + git commit -m "Generate release notes for ${{ inputs.version }}" || true diff --git a/.mergify.yml b/.mergify.yml index f49144113da..1c220045aba 100644 --- a/.mergify.yml +++ b/.mergify.yml @@ -161,3 +161,14 @@ pull_request_rules: labels: - automatic backport - merge-queue + + - name: backport patches to 2.31 + conditions: + - label=backport 2.31-maintenance + actions: + backport: + branches: + - "2.31-maintenance" + labels: + - automatic backport + - merge-queue diff --git a/.version b/.version index 70ff1993b10..7780cec2961 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -2.31.1 +2.32.1 diff --git a/.version-determinate b/.version-determinate new file mode 100644 index 00000000000..92536a9e485 --- /dev/null +++ b/.version-determinate @@ -0,0 +1 @@ +3.12.0 diff --git a/COPYING b/COPYING index 5ab7695ab8c..f6683e74e0f 100644 --- a/COPYING +++ b/COPYING @@ -1,8 +1,8 @@ - GNU LESSER GENERAL PUBLIC LICENSE - Version 2.1, February 1999 + GNU LESSER GENERAL PUBLIC LICENSE + Version 2.1, February 1999 Copyright (C) 1991, 1999 Free Software Foundation, Inc. - 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. @@ -10,7 +10,7 @@ as the successor of the GNU Library Public License, version 2, hence the version number 2.1.] - Preamble + Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public @@ -112,7 +112,7 @@ modification follow. Pay close attention to the difference between a former contains code derived from the library, whereas the latter must be combined with the library in order to run. - GNU LESSER GENERAL PUBLIC LICENSE + GNU LESSER GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License Agreement applies to any software library or other @@ -146,7 +146,7 @@ such a program is covered only if its contents constitute a work based on the Library (independent of the use of the Library in a tool for writing it). Whether that is true depends on what the Library does and what the program that uses the Library does. - + 1. You may copy and distribute verbatim copies of the Library's complete source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an @@ -432,7 +432,7 @@ decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. - NO WARRANTY + NO WARRANTY 15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. @@ -455,7 +455,7 @@ FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - END OF TERMS AND CONDITIONS + END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Libraries @@ -484,8 +484,7 @@ convey the exclusion of warranty; and each file should have at least the Lesser General Public License for more details. You should have received a copy of the GNU Lesser General Public - License along with this library; if not, write to the Free Software - Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + License along with this library; if not, see . Also add information on how to contact you by electronic and paper mail. @@ -496,9 +495,7 @@ necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the library `Frob' (a library for tweaking knobs) written by James Random Hacker. - , 1 April 1990 - Ty Coon, President of Vice + , 1 April 1990 + Moe Ghoul, President of Vice That's all there is to it! - - diff --git a/README.md b/README.md index 02498944cdb..4e304b28bf8 100644 --- a/README.md +++ b/README.md @@ -1,38 +1,92 @@ -# Nix +

+ +

+

+  Discord  +  Bluesky  +  Mastodon  +  Twitter  +  LinkedIn  +

-[![Open Collective supporters](https://opencollective.com/nixos/tiers/supporter/badge.svg?label=Supporters&color=brightgreen)](https://opencollective.com/nixos) -[![CI](https://github.com/NixOS/nix/workflows/CI/badge.svg)](https://github.com/NixOS/nix/actions/workflows/ci.yml) +# Determinate Nix -Nix is a powerful package manager for Linux and other Unix systems that makes package -management reliable and reproducible. Please refer to the [Nix manual](https://nix.dev/reference/nix-manual) -for more details. +[![CI](https://github.com/DeterminateSystems/nix-src/workflows/CI/badge.svg)](https://github.com/DeterminateSystems/nix-src/actions/workflows/ci.yml) -## Installation and first steps +This repository houses the source for [**Determinate Nix**][det-nix], a downstream distribution of [Nix][upstream] created and maintained by [Determinate Systems][detsys]. +Nix is a powerful [language], [package manager][package-management], and [CLI] for [macOS](#macos), [Linux](linux), and other Unix systems that enables you to create fully reproducible [development environments][envs], to build [packages] in sandboxed environments, to build entire Linux systems using [NixOS], and much more. -Visit [nix.dev](https://nix.dev) for [installation instructions](https://nix.dev/tutorials/install-nix) and [beginner tutorials](https://nix.dev/tutorials/first-steps). +Determinate Nix is part of the [Determinate platform][determinate], which also includes [FlakeHub], a secure flake repository with features like [FlakeHub Cache][cache], [private flakes][private-flakes], and [semantic versioning][semver] (SemVer) for [flakes]. -Full reference documentation can be found in the [Nix manual](https://nix.dev/reference/nix-manual). +## Installing Determinate -## Building and developing +You can install Determinate on [macOS](#macos), non-NixOS [Linux](#linux) and WSL, and [NixOS](#nixos). -Follow instructions in the Nix reference manual to [set up a development environment and build Nix from source](https://nix.dev/manual/nix/development/development/building.html). +### macOS -## Contributing +On macOS, we recommend using the graphical installer from Determinate Systems. +Click [here][gui] to download and run it. + +### Linux + +On Linux, including Windows Subsystem for Linux (WSL), we recommend installing Determinate Nix using [Determinate Nix Installer][installer]: + +```shell +curl -fsSL https://install.determinate.systems/nix | sh -s -- install --determinate +``` + +### NixOS + +On [NixOS], we recommend following our [dedicated installation guide][nixos-install]. -Check the [contributing guide](./CONTRIBUTING.md) if you want to get involved with developing Nix. +## Other resources -## Additional resources +Nix was created by [Eelco Dolstra][eelco] and developed as the subject of his 2006 PhD thesis, [The Purely Functional Software Deployment Model][thesis]. +Today, a worldwide developer community contributes to Nix and the ecosystem that has grown around it. -Nix was created by Eelco Dolstra and developed as the subject of his PhD thesis [The Purely Functional Software Deployment Model](https://edolstra.github.io/pubs/phd-thesis.pdf), published 2006. -Today, a world-wide developer community contributes to Nix and the ecosystem that has grown around it. +- [Zero to Nix][z2n], Determinate Systems' guide to Nix and [flakes] for beginners +- [Nixpkgs], a collection of well over 100,000 software packages that you can build and manage using Nix +- [NixOS] is a Linux distribution that can be configured fully declaratively +- The Nix, Nixpkgs, and NixOS community on [nixos.org][website] -- [The Nix, Nixpkgs, NixOS Community on nixos.org](https://nixos.org/) -- [Official documentation on nix.dev](https://nix.dev) -- [Nixpkgs](https://github.com/NixOS/nixpkgs) is [the largest, most up-to-date free software repository in the world](https://repology.org/repositories/graphs) -- [NixOS](https://github.com/NixOS/nixpkgs/tree/master/nixos) is a Linux distribution that can be configured fully declaratively -- [Discourse](https://discourse.nixos.org/) -- Matrix: [#users:nixos.org](https://matrix.to/#/#users:nixos.org) for user support and [#nix-dev:nixos.org](https://matrix.to/#/#nix-dev:nixos.org) for development +## Reference + +The primary documentation for Determinate and Determinate Nix is available at [docs.determinate.systems][determinate]. +For deeply technical reference material, see the [Determinate Nix manual][manual] which is based on the upstream Nix manual. ## License -Nix is released under the [LGPL v2.1](./COPYING). +[Upstream Nix][upstream] is released under the [LGPL v2.1][license] license. +[Determinate Nix][det-nix] is also released under LGPL v2.1 in accordance with the terms of the upstream license. + +## Contributing + +Check the [contributing guide][contributing] if you want to get involved with developing Nix. + +[cache]: https://docs.determinate.systems/flakehub/cache +[cli]: https://manual.determinate.systems/command-ref/new-cli/nix.html +[contributing]: ./CONTRIBUTING.md +[det-nix]: https://docs.determinate.systems/determinate-nix +[determinate]: https://docs.determinate.systems +[detsys]: https://determinate.systems +[dnixd]: https://docs.determinate.systems/determinate-nix#determinate-nixd +[eelco]: https://determinate.systems/people/eelco-dolstra +[envs]: https://zero-to-nix.com/concepts/dev-env +[flakehub]: https://flakehub.com +[flakes]: https://zero-to-nix.com/concepts/flakes +[gui]: https://install.determinate.systems/determinate-pkg/stable/Universal +[installer]: https://github.com/DeterminateSystems/nix-installer +[language]: https://zero-to-nix.com/concepts/nix-language +[license]: ./COPYING +[manual]: https://manual.determinate.systems +[nixpkgs]: https://github.com/NixOS/nixpkgs +[nixos]: https://github.com/NixOS/nixpkgs/tree/master/nixos +[nixos-install]: https://docs.determinate.systems/guides/advanced-installation#nixos +[packages]: https://zero-to-nix.com/concepts/packages +[package-management]: https://zero-to-nix.com/concepts/package-management +[private-flakes]: https://docs.determinate.systems/flakehub/private-flakes +[semver]: https://docs.determinate.systems/flakehub/concepts/semver +[thesis]: https://edolstra.github.io/pubs/phd-thesis.pdf +[upstream]: https://github.com/NixOS/nix +[website]: https://nixos.org +[z2n]: https://zero-to-nix.com diff --git a/ci/gha/tests/default.nix b/ci/gha/tests/default.nix index 74d0b8c7ec2..e181ee67024 100644 --- a/ci/gha/tests/default.nix +++ b/ci/gha/tests/default.nix @@ -24,16 +24,7 @@ let enableSanitizersLayer = finalAttrs: prevAttrs: { mesonFlags = (prevAttrs.mesonFlags or [ ]) - ++ [ - # Run all tests with UBSAN enabled. Running both with ubsan and - # without doesn't seem to have much immediate benefit for doubling - # the GHA CI workaround. - # - # TODO: Work toward enabling "address,undefined" if it seems feasible. - # This would maybe require dropping Boost coroutines and ignoring intentional - # memory leaks with detect_leaks=0. - (lib.mesonOption "b_sanitize" "undefined") - ] + ++ [ (lib.mesonOption "b_sanitize" "address,undefined") ] ++ (lib.optionals stdenv.cc.isClang [ # https://www.github.com/mesonbuild/meson/issues/764 (lib.mesonBool "b_lundef" false) @@ -71,8 +62,12 @@ rec { nixComponentsInstrumented = nixComponents.overrideScope ( final: prev: { nix-store-tests = prev.nix-store-tests.override { withBenchmarks = true; }; + # Boehm is incompatible with ASAN. + nix-expr = prev.nix-expr.override { enableGC = !withSanitizers; }; mesonComponentOverrides = lib.composeManyExtensions componentOverrides; + # Unclear how to make Perl bindings work with a dynamically linked ASAN. + nix-perl-bindings = if withSanitizers then null else prev.nix-perl-bindings; } ); @@ -82,7 +77,6 @@ rec { */ topLevel = { installerScriptForGHA = hydraJobs.installerScriptForGHA.${system}; - installTests = hydraJobs.installTests.${system}; nixpkgsLibTests = hydraJobs.tests.nixpkgsLibTests.${system}; rl-next = pkgs.buildPackages.runCommand "test-rl-next-release-notes" { } '' LANG=C.UTF-8 ${pkgs.changelog-d}/bin/changelog-d ${../../../doc/manual/rl-next} >$out diff --git a/ci/gha/tests/pre-commit-checks b/ci/gha/tests/pre-commit-checks new file mode 100755 index 00000000000..8c9f64d6c26 --- /dev/null +++ b/ci/gha/tests/pre-commit-checks @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +set -euo pipefail + +system=$(nix eval --raw --impure --expr builtins.currentSystem) + +echo "::group::Running pre-commit checks" + +if nix build ".#checks.$system.pre-commit" -L; then + echo "::endgroup::" + exit 0 +fi + +echo "::error ::Changes do not pass pre-commit checks" + +cat < **Warning** \ - > This program is - > [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) - > and its interface is subject to change. - # Name `${command}` - ${details.description} diff --git a/doc/manual/meson.build b/doc/manual/meson.build index 2e372deddee..7991c8e9993 100644 --- a/doc/manual/meson.build +++ b/doc/manual/meson.build @@ -5,6 +5,8 @@ project( license : 'LGPL-2.1-or-later', ) +fs = import('fs') + nix = find_program('nix', native : true) mdbook = find_program('mdbook', native : true) @@ -15,6 +17,7 @@ pymod = import('python') python = pymod.find_installation('python3') nix_env_for_docs = { + 'ASAN_OPTIONS' : 'abort_on_error=1:print_summary=1:detect_leaks=0', 'HOME' : '/dummy', 'NIX_CONF_DIR' : '/dummy', 'NIX_SSL_CERT_FILE' : '/dummy/no-ca-bundle.crt', @@ -22,7 +25,7 @@ nix_env_for_docs = { 'NIX_CONFIG' : 'cores = 0', } -nix_for_docs = [ nix, '--experimental-features', 'nix-command' ] +nix_for_docs = [ nix ] nix_eval_for_docs_common = nix_for_docs + [ 'eval', '-I', @@ -97,7 +100,7 @@ manual = custom_target( python.full_path(), mdbook.full_path(), meson.current_build_dir(), - meson.project_version(), + fs.read('../../.version-determinate').strip(), rsync.full_path(), ), ], diff --git a/doc/manual/package.nix b/doc/manual/package.nix index 69b7c0e49b0..a74ee3f57b2 100644 --- a/doc/manual/package.nix +++ b/doc/manual/package.nix @@ -24,7 +24,7 @@ let in mkMesonDerivation (finalAttrs: { - pname = "nix-manual"; + pname = "determinate-nix-manual"; inherit version; workDir = ./.; @@ -32,6 +32,7 @@ mkMesonDerivation (finalAttrs: { fileset.difference (fileset.unions [ ../../.version + ../../.version-determinate # Too many different types of files to filter for now ../../doc/manual ./. diff --git a/doc/manual/redirects.js b/doc/manual/redirects.js index 9612438481f..b2295cf4fc5 100644 --- a/doc/manual/redirects.js +++ b/doc/manual/redirects.js @@ -271,14 +271,10 @@ const redirects = { "sect-multi-user-installation": "installation/installing-binary.html#multi-user-installation", "sect-nix-install-binary-tarball": "installation/installing-binary.html#installing-from-a-binary-tarball", "sect-nix-install-pinned-version-url": "installation/installing-binary.html#installing-a-pinned-nix-version-from-a-url", - "sect-single-user-installation": "installation/installing-binary.html#single-user-installation", "ch-installing-source": "installation/installing-source.html", "ssec-multi-user": "installation/multi-user.html", - "ch-nix-security": "installation/nix-security.html", "sec-obtaining-source": "installation/obtaining-source.html", "sec-prerequisites-source": "installation/prerequisites-source.html", - "sec-single-user": "installation/single-user.html", - "ch-supported-platforms": "installation/supported-platforms.html", "ch-upgrading-nix": "installation/upgrading.html", "ch-about-nix": "introduction.html", "chap-introduction": "introduction.html", diff --git a/doc/manual/rl-next/shorter-build-dir-names.md b/doc/manual/rl-next/shorter-build-dir-names.md new file mode 100644 index 00000000000..e87fa5d04fb --- /dev/null +++ b/doc/manual/rl-next/shorter-build-dir-names.md @@ -0,0 +1,6 @@ +--- +synopsis: "Temporary build directories no longer include derivation names" +prs: [13839] +--- + +Temporary build directories created during derivation builds no longer include the derivation name in their path to avoid build failures when the derivation name is too long. This change ensures predictable prefix lengths for build directories under `/nix/var/nix/builds`. \ No newline at end of file diff --git a/doc/manual/source/SUMMARY.md.in b/doc/manual/source/SUMMARY.md.in index 8fed98c2c1b..9f703d3eac1 100644 --- a/doc/manual/source/SUMMARY.md.in +++ b/doc/manual/source/SUMMARY.md.in @@ -3,17 +3,12 @@ - [Introduction](introduction.md) - [Quick Start](quick-start.md) - [Installation](installation/index.md) - - [Supported Platforms](installation/supported-platforms.md) - - [Installing a Binary Distribution](installation/installing-binary.md) - [Installing Nix from Source](installation/installing-source.md) - [Prerequisites](installation/prerequisites-source.md) - [Obtaining a Source Distribution](installation/obtaining-source.md) - [Building Nix from Source](installation/building-source.md) - [Using Nix within Docker](installation/installing-docker.md) - [Security](installation/nix-security.md) - - [Single-User Mode](installation/single-user.md) - - [Multi-User Mode](installation/multi-user.md) - - [Environment Variables](installation/env-variables.md) - [Upgrading Nix](installation/upgrading.md) - [Uninstalling Nix](installation/uninstall.md) - [Nix Store](store/index.md) @@ -61,8 +56,11 @@ - [Command Reference](command-ref/index.md) - [Common Options](command-ref/opt-common.md) - [Common Environment Variables](command-ref/env-common.md) - - [Main Commands](command-ref/main-commands.md) + - [Subcommands](command-ref/subcommands.md) +{{#include ./command-ref/new-cli/SUMMARY.md}} + - [Deprecated Commands](command-ref/main-commands.md) - [nix-build](command-ref/nix-build.md) + - [nix-channel](command-ref/nix-channel.md) - [nix-shell](command-ref/nix-shell.md) - [nix-store](command-ref/nix-store.md) - [nix-store --add-fixed](command-ref/nix-store/add-fixed.md) @@ -98,22 +96,17 @@ - [nix-env --uninstall](command-ref/nix-env/uninstall.md) - [nix-env --upgrade](command-ref/nix-env/upgrade.md) - [Utilities](command-ref/utilities.md) - - [nix-channel](command-ref/nix-channel.md) - [nix-collect-garbage](command-ref/nix-collect-garbage.md) - [nix-copy-closure](command-ref/nix-copy-closure.md) - [nix-daemon](command-ref/nix-daemon.md) - [nix-hash](command-ref/nix-hash.md) - [nix-instantiate](command-ref/nix-instantiate.md) - [nix-prefetch-url](command-ref/nix-prefetch-url.md) - - [Experimental Commands](command-ref/experimental-commands.md) -{{#include ./command-ref/new-cli/SUMMARY.md}} - [Files](command-ref/files.md) - [nix.conf](command-ref/conf-file.md) - [Profiles](command-ref/files/profiles.md) - [manifest.nix](command-ref/files/manifest.nix.md) - [manifest.json](command-ref/files/manifest.json.md) - - [Channels](command-ref/files/channels.md) - - [Default Nix expression](command-ref/files/default-nix-expression.md) - [Architecture and Design](architecture/architecture.md) - [Formats and Protocols](protocols/index.md) - [JSON Formats](protocols/json/index.md) @@ -136,68 +129,46 @@ - [C++ style guide](development/cxx.md) - [Experimental Features](development/experimental-features.md) - [Contributing](development/contributing.md) -- [Releases](release-notes/index.md) +- [Determinate Nix Release Notes](release-notes-determinate/index.md) + - [Changes between Nix and Determinate Nix](release-notes-determinate/changes.md) + - [Release 3.12.0 (2025-10-23)](release-notes-determinate/v3.12.0.md) + - [Release 3.11.3 (2025-10-09)](release-notes-determinate/v3.11.3.md) + - [Release 3.11.2 (2025-09-12)](release-notes-determinate/v3.11.2.md) + - [Release 3.11.1 (2025-09-04)](release-notes-determinate/v3.11.1.md) + - [Release 3.11.0 (2025-09-03)](release-notes-determinate/v3.11.0.md) + - [Release 3.10.1 (2025-09-02)](release-notes-determinate/v3.10.1.md) + - [Release 3.10.0 (2025-09-02)](release-notes-determinate/v3.10.0.md) + - [Release 3.9.1 (2025-08-28)](release-notes-determinate/v3.9.1.md) + - [Release 3.9.0 (2025-08-26)](release-notes-determinate/v3.9.0.md) + - [Release 3.8.6 (2025-08-19)](release-notes-determinate/v3.8.6.md) + - [Release 3.8.5 (2025-08-04)](release-notes-determinate/rl-3.8.5.md) + - [Release 3.8.4 (2025-07-21)](release-notes-determinate/rl-3.8.4.md) + - [Release 3.8.3 (2025-07-18)](release-notes-determinate/rl-3.8.3.md) + - [Release 3.8.2 (2025-07-12)](release-notes-determinate/rl-3.8.2.md) + - [Release 3.8.1 (2025-07-11)](release-notes-determinate/rl-3.8.1.md) + - [Release 3.8.0 (2025-07-10)](release-notes-determinate/rl-3.8.0.md) + - [Release 3.7.0 (2025-07-03)](release-notes-determinate/rl-3.7.0.md) + - [Release 3.6.8 (2025-06-25)](release-notes-determinate/rl-3.6.8.md) + - [Release 3.6.7 (2025-06-24)](release-notes-determinate/rl-3.6.7.md) + - [Release 3.6.6 (2025-06-17)](release-notes-determinate/rl-3.6.6.md) + - [Release 3.6.5 (2025-06-16)](release-notes-determinate/rl-3.6.5.md) + - [Release 3.6.2 (2025-06-02)](release-notes-determinate/rl-3.6.2.md) + - [Release 3.6.1 (2025-05-24)](release-notes-determinate/rl-3.6.1.md) + - [Release 3.6.0 (2025-05-22)](release-notes-determinate/rl-3.6.0.md) + - [Release 3.5.2 (2025-05-12)](release-notes-determinate/rl-3.5.2.md) + - [Release 3.5.1 (2025-05-09)](release-notes-determinate/rl-3.5.1.md) + - [~~Release 3.5.0 (2025-05-09)~~](release-notes-determinate/rl-3.5.0.md) + - [Release 3.4.2 (2025-05-05)](release-notes-determinate/rl-3.4.2.md) + - [Release 3.4.0 (2025-04-25)](release-notes-determinate/rl-3.4.0.md) + - [Release 3.3.0 (2025-04-11)](release-notes-determinate/rl-3.3.0.md) + - [Release 3.1.0 (2025-03-27)](release-notes-determinate/rl-3.1.0.md) + - [Release 3.0.0 (2025-03-04)](release-notes-determinate/rl-3.0.0.md) +- [Nix Release Notes](release-notes/index.md) {{#include ./SUMMARY-rl-next.md}} + - [Release 2.32 (2025-10-06)](release-notes/rl-2.32.md) - [Release 2.31 (2025-08-21)](release-notes/rl-2.31.md) - [Release 2.30 (2025-07-07)](release-notes/rl-2.30.md) - [Release 2.29 (2025-05-14)](release-notes/rl-2.29.md) - [Release 2.28 (2025-04-02)](release-notes/rl-2.28.md) - [Release 2.27 (2025-03-03)](release-notes/rl-2.27.md) - [Release 2.26 (2025-01-22)](release-notes/rl-2.26.md) - - [Release 2.25 (2024-11-07)](release-notes/rl-2.25.md) - - [Release 2.24 (2024-07-31)](release-notes/rl-2.24.md) - - [Release 2.23 (2024-06-03)](release-notes/rl-2.23.md) - - [Release 2.22 (2024-04-23)](release-notes/rl-2.22.md) - - [Release 2.21 (2024-03-11)](release-notes/rl-2.21.md) - - [Release 2.20 (2024-01-29)](release-notes/rl-2.20.md) - - [Release 2.19 (2023-11-17)](release-notes/rl-2.19.md) - - [Release 2.18 (2023-09-20)](release-notes/rl-2.18.md) - - [Release 2.17 (2023-07-24)](release-notes/rl-2.17.md) - - [Release 2.16 (2023-05-31)](release-notes/rl-2.16.md) - - [Release 2.15 (2023-04-11)](release-notes/rl-2.15.md) - - [Release 2.14 (2023-02-28)](release-notes/rl-2.14.md) - - [Release 2.13 (2023-01-17)](release-notes/rl-2.13.md) - - [Release 2.12 (2022-12-06)](release-notes/rl-2.12.md) - - [Release 2.11 (2022-08-25)](release-notes/rl-2.11.md) - - [Release 2.10 (2022-07-11)](release-notes/rl-2.10.md) - - [Release 2.9 (2022-05-30)](release-notes/rl-2.9.md) - - [Release 2.8 (2022-04-19)](release-notes/rl-2.8.md) - - [Release 2.7 (2022-03-07)](release-notes/rl-2.7.md) - - [Release 2.6 (2022-01-24)](release-notes/rl-2.6.md) - - [Release 2.5 (2021-12-13)](release-notes/rl-2.5.md) - - [Release 2.4 (2021-11-01)](release-notes/rl-2.4.md) - - [Release 2.3 (2019-09-04)](release-notes/rl-2.3.md) - - [Release 2.2 (2019-01-11)](release-notes/rl-2.2.md) - - [Release 2.1 (2018-09-02)](release-notes/rl-2.1.md) - - [Release 2.0 (2018-02-22)](release-notes/rl-2.0.md) - - [Release 1.11.10 (2017-06-12)](release-notes/rl-1.11.10.md) - - [Release 1.11 (2016-01-19)](release-notes/rl-1.11.md) - - [Release 1.10 (2015-09-03)](release-notes/rl-1.10.md) - - [Release 1.9 (2015-06-12)](release-notes/rl-1.9.md) - - [Release 1.8 (2014-12-14)](release-notes/rl-1.8.md) - - [Release 1.7 (2014-04-11)](release-notes/rl-1.7.md) - - [Release 1.6.1 (2013-10-28)](release-notes/rl-1.6.1.md) - - [Release 1.6 (2013-09-10)](release-notes/rl-1.6.md) - - [Release 1.5.2 (2013-05-13)](release-notes/rl-1.5.2.md) - - [Release 1.5 (2013-02-27)](release-notes/rl-1.5.md) - - [Release 1.4 (2013-02-26)](release-notes/rl-1.4.md) - - [Release 1.3 (2013-01-04)](release-notes/rl-1.3.md) - - [Release 1.2 (2012-12-06)](release-notes/rl-1.2.md) - - [Release 1.1 (2012-07-18)](release-notes/rl-1.1.md) - - [Release 1.0 (2012-05-11)](release-notes/rl-1.0.md) - - [Release 0.16 (2010-08-17)](release-notes/rl-0.16.md) - - [Release 0.15 (2010-03-17)](release-notes/rl-0.15.md) - - [Release 0.14 (2010-02-04)](release-notes/rl-0.14.md) - - [Release 0.13 (2009-11-05)](release-notes/rl-0.13.md) - - [Release 0.12 (2008-11-20)](release-notes/rl-0.12.md) - - [Release 0.11 (2007-12-31)](release-notes/rl-0.11.md) - - [Release 0.10.1 (2006-10-11)](release-notes/rl-0.10.1.md) - - [Release 0.10 (2006-10-06)](release-notes/rl-0.10.md) - - [Release 0.9.2 (2005-09-21)](release-notes/rl-0.9.2.md) - - [Release 0.9.1 (2005-09-20)](release-notes/rl-0.9.1.md) - - [Release 0.9 (2005-09-16)](release-notes/rl-0.9.md) - - [Release 0.8.1 (2005-04-13)](release-notes/rl-0.8.1.md) - - [Release 0.8 (2005-04-11)](release-notes/rl-0.8.md) - - [Release 0.7 (2005-01-12)](release-notes/rl-0.7.md) - - [Release 0.6 (2004-11-14)](release-notes/rl-0.6.md) - - [Release 0.5 and earlier](release-notes/rl-0.5.md) diff --git a/doc/manual/source/command-ref/env-common.md b/doc/manual/source/command-ref/env-common.md index e0fd2b00eec..fe6e822ff16 100644 --- a/doc/manual/source/command-ref/env-common.md +++ b/doc/manual/source/command-ref/env-common.md @@ -102,7 +102,7 @@ Most Nix commands interpret the following environment variables: This variable should be set to `daemon` if you want to use the Nix daemon to execute Nix operations. This is necessary in [multi-user - Nix installations](@docroot@/installation/multi-user.md). If the Nix + Nix installations](@docroot@/installation/nix-security.md#multi-user-model). If the Nix daemon's Unix socket is at some non-standard path, this variable should be set to `unix://path/to/socket`. Otherwise, it should be left unset. diff --git a/doc/manual/source/command-ref/experimental-commands.md b/doc/manual/source/command-ref/experimental-commands.md deleted file mode 100644 index 1190729a230..00000000000 --- a/doc/manual/source/command-ref/experimental-commands.md +++ /dev/null @@ -1,8 +0,0 @@ -# Experimental Commands - -This section lists [experimental commands](@docroot@/development/experimental-features.md#xp-feature-nix-command). - -> **Warning** -> -> These commands may be removed in the future, or their syntax may -> change in incompatible ways. diff --git a/doc/manual/source/command-ref/files/default-nix-expression.md b/doc/manual/source/command-ref/files/default-nix-expression.md index 2bd45ff5deb..e886e3ff499 100644 --- a/doc/manual/source/command-ref/files/default-nix-expression.md +++ b/doc/manual/source/command-ref/files/default-nix-expression.md @@ -31,12 +31,12 @@ Then, the resulting expression is interpreted like this: The file [`manifest.nix`](@docroot@/command-ref/files/manifest.nix.md) is always ignored. -The command [`nix-channel`] places a symlink to the current user's [channels] in this directory, the [user channel link](#user-channel-link). +The command [`nix-channel`] places a symlink to the current user's channels in this directory, the [user channel link](#user-channel-link). This makes all subscribed channels available as attributes in the default expression. ## User channel link -A symlink that ensures that [`nix-env`] can find the current user's [channels]: +A symlink that ensures that [`nix-env`] can find the current user's channels: - `~/.nix-defexpr/channels` - `$XDG_STATE_HOME/defexpr/channels` if [`use-xdg-base-directories`] is set to `true`. @@ -51,4 +51,3 @@ In a multi-user installation, you may also have `~/.nix-defexpr/channels_root`, [`nix-channel`]: @docroot@/command-ref/nix-channel.md [`nix-env`]: @docroot@/command-ref/nix-env.md [`use-xdg-base-directories`]: @docroot@/command-ref/conf-file.md#conf-use-xdg-base-directories -[channels]: @docroot@/command-ref/files/channels.md diff --git a/doc/manual/source/command-ref/files/profiles.md b/doc/manual/source/command-ref/files/profiles.md index b5c7378800f..e46e2418b4c 100644 --- a/doc/manual/source/command-ref/files/profiles.md +++ b/doc/manual/source/command-ref/files/profiles.md @@ -67,7 +67,7 @@ By default, this symlink points to: - `$NIX_STATE_DIR/profiles/per-user/root/profile` for `root` The `PATH` environment variable should include `/bin` subdirectory of the profile link (e.g. `~/.nix-profile/bin`) for the user environment to be visible to the user. -The [installer](@docroot@/installation/installing-binary.md) sets this up by default, unless you enable [`use-xdg-base-directories`]. +The installer sets this up by default, unless you enable [`use-xdg-base-directories`]. [`nix-env`]: @docroot@/command-ref/nix-env.md [`nix profile`]: @docroot@/command-ref/new-cli/nix3-profile.md diff --git a/doc/manual/source/command-ref/meson.build b/doc/manual/source/command-ref/meson.build index 92998dec126..06aed261a60 100644 --- a/doc/manual/source/command-ref/meson.build +++ b/doc/manual/source/command-ref/meson.build @@ -2,6 +2,7 @@ xp_features_json = custom_target( command : [ nix, '__dump-xp-features' ], capture : true, output : 'xp-features.json', + env : nix_env_for_docs, ) experimental_features_shortlist_md = custom_target( diff --git a/doc/manual/source/command-ref/nix-channel.md b/doc/manual/source/command-ref/nix-channel.md index ed9cbb41fbf..a65ec97c558 100644 --- a/doc/manual/source/command-ref/nix-channel.md +++ b/doc/manual/source/command-ref/nix-channel.md @@ -8,6 +8,12 @@ # Description +> **Warning** +> +> nix-channel is deprecated in favor of flakes in Determinate Nix. +> For a guide on Nix flakes, see: . +> For details and to offer feedback on the deprecation process, see: . + Channels are a mechanism for referencing remote Nix expressions and conveniently retrieving their latest version. The moving parts of channels are: diff --git a/doc/manual/source/command-ref/nix-env.md b/doc/manual/source/command-ref/nix-env.md index bda02149ed0..d01caaf7f78 100644 --- a/doc/manual/source/command-ref/nix-env.md +++ b/doc/manual/source/command-ref/nix-env.md @@ -52,7 +52,7 @@ These pages can be viewed offline: `nix-env` can obtain packages from multiple sources: - An attribute set of derivations from: - - The [default Nix expression](@docroot@/command-ref/files/default-nix-expression.md) (by default) + - The default Nix expression (by default) - A Nix file, specified via `--file` - A [profile](@docroot@/command-ref/files/profiles.md), specified via `--from-profile` - A Nix expression that is a function which takes default expression as argument, specified via `--from-expression` diff --git a/doc/manual/source/command-ref/nix-env/install.md b/doc/manual/source/command-ref/nix-env/install.md index 527fd8f90d8..26a32aa6b6b 100644 --- a/doc/manual/source/command-ref/nix-env/install.md +++ b/doc/manual/source/command-ref/nix-env/install.md @@ -22,12 +22,11 @@ It is based on the current generation of the active [profile](@docroot@/command- The arguments *args* map to store paths in a number of possible ways: -- By default, *args* is a set of names denoting derivations in the [default Nix expression]. +- By default, *args* is a set of names denoting derivations in the default Nix expression. These are [realised], and the resulting output paths are installed. Currently installed derivations with a name equal to the name of a derivation being added are removed unless the option `--preserve-installed` is specified. [derivation expression]: @docroot@/glossary.md#gloss-derivation-expression - [default Nix expression]: @docroot@/command-ref/files/default-nix-expression.md [realised]: @docroot@/glossary.md#gloss-realise If there are multiple derivations matching a name in *args* that @@ -45,7 +44,7 @@ The arguments *args* map to store paths in a number of possible ways: gcc-3.3.6 gcc-4.1.1` will install both version of GCC (and will probably cause a user environment conflict\!). -- If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the [default Nix expression]. +- If [`--attr`](#opt-attr) / `-A` is specified, the arguments are *attribute paths* that select attributes from the default Nix expression. This is faster than using derivation names and unambiguous. Show the attribute paths of available packages with [`nix-env --query`](./query.md): @@ -58,7 +57,7 @@ The arguments *args* map to store paths in a number of possible ways: easy way to copy user environment elements from one profile to another. -- If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/syntax.md#functions) that are called with the [default Nix expression] as their single argument. +- If `--from-expression` is given, *args* are [Nix language functions](@docroot@/language/syntax.md#functions) that are called with the default Nix expression as their single argument. The derivations returned by those function calls are installed. This allows derivations to be specified in an unambiguous way, which is necessary if there are multiple derivations with the same name. diff --git a/doc/manual/source/command-ref/nix-store/query.md b/doc/manual/source/command-ref/nix-store/query.md index b5ba63adae2..94eee05b8a8 100644 --- a/doc/manual/source/command-ref/nix-store/query.md +++ b/doc/manual/source/command-ref/nix-store/query.md @@ -103,6 +103,13 @@ symlink. example when *paths* were substituted from a binary cache. Use `--valid-derivers` instead to obtain valid paths only. + > **Note** + > + > `nix-store --query --deriver` is replaced with the following `nix` command: + > + > nix path-info --json ... | jq -r '.[].deriver' + + [deriver]: @docroot@/glossary.md#gloss-deriver - `--valid-derivers` diff --git a/doc/manual/source/command-ref/subcommands.md b/doc/manual/source/command-ref/subcommands.md new file mode 100644 index 00000000000..6a26732338d --- /dev/null +++ b/doc/manual/source/command-ref/subcommands.md @@ -0,0 +1,3 @@ +# Subcommands + +This section lists all the subcommands of the `nix` CLI. diff --git a/doc/manual/source/development/building.md b/doc/manual/source/development/building.md index 33b7b2d5c56..111d46d7232 100644 --- a/doc/manual/source/development/building.md +++ b/doc/manual/source/development/building.md @@ -1,73 +1,5 @@ # Building Nix -This section provides some notes on how to start hacking on Nix. -To get the latest version of Nix from GitHub: - -```console -$ git clone https://github.com/NixOS/nix.git -$ cd nix -``` - -> **Note** -> -> The following instructions assume you already have some version of Nix installed locally, so that you can use it to set up the development environment. -> If you don't have it installed, follow the [installation instructions](../installation/index.md). - - -To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: - -```console -$ nix-shell -``` - -To get a shell with one of the other [supported compilation environments](#compilation-environments): - -```console -$ nix-shell --attr devShells.x86_64-linux.native-clangStdenvPackages -``` - -> **Note** -> -> You can use `native-ccacheStdenv` to drastically improve rebuild time. -> By default, [ccache](https://ccache.dev) keeps artifacts in `~/.cache/ccache/`. - -To build Nix itself in this shell: - -```console -[nix-shell]$ mesonFlags+=" --prefix=$(pwd)/outputs/out" -[nix-shell]$ dontAddPrefix=1 configurePhase -[nix-shell]$ buildPhase -``` - -To test it: - -```console -[nix-shell]$ checkPhase -``` - -To install it in `$(pwd)/outputs`: - -```console -[nix-shell]$ installPhase -[nix-shell]$ ./outputs/out/bin/nix --version -nix (Nix) 2.12 -``` - -To build a release version of Nix for the current operating system and CPU architecture: - -```console -$ nix-build -``` - -You can also build Nix for one of the [supported platforms](#platforms). - -## Building Nix with flakes - -This section assumes you are using Nix with the [`flakes`] and [`nix-command`] experimental features enabled. - -[`flakes`]: @docroot@/development/experimental-features.md#xp-feature-flakes -[`nix-command`]: @docroot@/development/experimental-features.md#xp-nix-command - To build all dependencies and start a shell in which all environment variables are set up so that those dependencies can be found: ```console @@ -145,12 +77,6 @@ platform. Common solutions include [remote build machines] and [binary format em Given such a setup, executing the build only requires selecting the respective attribute. For example, to compile for `aarch64-linux`: -```console -$ nix-build --attr packages.aarch64-linux.default -``` - -or for Nix with the [`flakes`] and [`nix-command`] experimental features enabled: - ```console $ nix build .#packages.aarch64-linux.default ``` @@ -243,20 +169,12 @@ To build with one of those environments, you can use $ nix build .#nix-cli-ccacheStdenv ``` -for flake-enabled Nix, or - -```console -$ nix-build --attr nix-cli-ccacheStdenv -``` - -for classic Nix. - You can use any of the other supported environments in place of `nix-cli-ccacheStdenv`. ## Editor integration The `clangd` LSP server is installed by default on the `clang`-based `devShell`s. -See [supported compilation environments](#compilation-environments) and instructions how to set up a shell [with flakes](#nix-with-flakes) or in [classic Nix](#classic-nix). +See [supported compilation environments](#compilation-environments) and instructions how to [set up a shell with flakes](#nix-with-flakes). To use the LSP with your editor, you will want a `compile_commands.json` file telling `clangd` how we are compiling the code. Meson's configure always produces this inside the build directory. diff --git a/doc/manual/source/development/debugging.md b/doc/manual/source/development/debugging.md index 98456841af1..ccc6614b75a 100644 --- a/doc/manual/source/development/debugging.md +++ b/doc/manual/source/development/debugging.md @@ -24,6 +24,19 @@ It is also possible to build without debugging for faster build: (The first line is needed because `fortify` hardening requires at least some optimization.) +## Building Nix with sanitizers + +Nix can be built with [Address](https://clang.llvm.org/docs/AddressSanitizer.html) and +[UB](https://clang.llvm.org/docs/UndefinedBehaviorSanitizer.html) sanitizers using LLVM +or GCC. This is useful when debugging memory corruption issues. + +```console +[nix-shell]$ export mesonBuildType=debugoptimized +[nix-shell]$ appendToVar mesonFlags "-Dlibexpr:gc=disabled" # Disable Boehm +[nix-shell]$ appendToVar mesonFlags "-Dbindings=false" # Disable nix-perl +[nix-shell]$ appendToVar mesonFlags "-Db_sanitize=address,undefined" +``` + ## Debugging the Nix Binary Obtain your preferred debugger within the development shell: diff --git a/doc/manual/source/development/experimental-features.md b/doc/manual/source/development/experimental-features.md index ad5cffa91ee..56a45b23890 100644 --- a/doc/manual/source/development/experimental-features.md +++ b/doc/manual/source/development/experimental-features.md @@ -6,7 +6,7 @@ Experimental features are considered unstable, which means that they can be chan Users must explicitly enable them by toggling the associated [experimental feature flags](@docroot@/command-ref/conf-file.md#conf-experimental-features). This allows accessing unstable functionality without unwittingly relying on it. -Experimental feature flags were first introduced in [Nix 2.4](@docroot@/release-notes/rl-2.4.md). +Experimental feature flags were first introduced in [Nix 2.4](https://nix.dev/manual/nix/latest/release-notes/rl-2.4). Before that, Nix did have experimental features, but they were not guarded by flags and were merely documented as unstable. This was a source of confusion and controversy. diff --git a/doc/manual/source/development/meson.build b/doc/manual/source/development/meson.build index 4831cf8f083..b3fb110230d 100644 --- a/doc/manual/source/development/meson.build +++ b/doc/manual/source/development/meson.build @@ -7,5 +7,6 @@ experimental_feature_descriptions_md = custom_target( xp_features_json, ], capture : true, + env : nix_env_for_docs, output : 'experimental-feature-descriptions.md', ) diff --git a/doc/manual/source/favicon.png b/doc/manual/source/favicon.png deleted file mode 100644 index 1ed2b5fe0fd..00000000000 Binary files a/doc/manual/source/favicon.png and /dev/null differ diff --git a/doc/manual/source/favicon.svg b/doc/manual/source/favicon.svg index 1d2a6e835d5..55fb9479b06 100644 --- a/doc/manual/source/favicon.svg +++ b/doc/manual/source/favicon.svg @@ -1 +1,29 @@ - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/manual/source/glossary.md b/doc/manual/source/glossary.md index e6a294e7de7..9e76ad37b96 100644 --- a/doc/manual/source/glossary.md +++ b/doc/manual/source/glossary.md @@ -353,14 +353,6 @@ See [Nix Archive](store/file-system-object/content-address.html#serial-nix-archive) for details. -- [`∅`]{#gloss-empty-set} - - The empty set symbol. In the context of profile history, this denotes a package is not present in a particular version of the profile. - -- [`ε`]{#gloss-epsilon} - - The epsilon symbol. In the context of a package, this means the version is empty. More precisely, the derivation does not have a version attribute. - - [package]{#package} A software package; files that belong together for a particular purpose, and metadata. diff --git a/doc/manual/source/installation/env-variables.md b/doc/manual/source/installation/env-variables.md deleted file mode 100644 index 0350904211a..00000000000 --- a/doc/manual/source/installation/env-variables.md +++ /dev/null @@ -1,62 +0,0 @@ -# Environment Variables - -To use Nix, some environment variables should be set. In particular, -`PATH` should contain the directories `prefix/bin` and -`~/.nix-profile/bin`. The first directory contains the Nix tools -themselves, while `~/.nix-profile` is a symbolic link to the current -*user environment* (an automatically generated package consisting of -symlinks to installed packages). The simplest way to set the required -environment variables is to include the file -`prefix/etc/profile.d/nix.sh` in your `~/.profile` (or similar), like -this: - -```bash -source prefix/etc/profile.d/nix.sh -``` - -# `NIX_SSL_CERT_FILE` - -If you need to specify a custom certificate bundle to account for an -HTTPS-intercepting man in the middle proxy, you must specify the path to -the certificate bundle in the environment variable `NIX_SSL_CERT_FILE`. - -If you don't specify a `NIX_SSL_CERT_FILE` manually, Nix will install -and use its own certificate bundle. - -Set the environment variable and install Nix - -```console -$ export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt -$ curl -L https://nixos.org/nix/install | sh -``` - -In the shell profile and rc files (for example, `/etc/bashrc`, -`/etc/zshrc`), add the following line: - -```bash -export NIX_SSL_CERT_FILE=/etc/ssl/my-certificate-bundle.crt -``` - -> **Note** -> -> You must not add the export and then do the install, as the Nix -> installer will detect the presence of Nix configuration, and abort. - -If you use the Nix daemon, you should also add the following to -`/etc/nix/nix.conf`: - -``` -ssl-cert-file = /etc/ssl/my-certificate-bundle.crt -``` - -## Proxy Environment Variables - -The Nix installer has special handling for these proxy-related -environment variables: `http_proxy`, `https_proxy`, `ftp_proxy`, -`all_proxy`, `no_proxy`, `HTTP_PROXY`, `HTTPS_PROXY`, `FTP_PROXY`, -`ALL_PROXY`, `NO_PROXY`. - -If any of these variables are set when running the Nix installer, then -the installer will create an override file at -`/etc/systemd/system/nix-daemon.service.d/override.conf` so `nix-daemon` -will use them. diff --git a/doc/manual/source/installation/index.md b/doc/manual/source/installation/index.md index 3c09f103184..21aca146fd2 100644 --- a/doc/manual/source/installation/index.md +++ b/doc/manual/source/installation/index.md @@ -1,44 +1,11 @@ # Installation -This section describes how to install and configure Nix for first-time use. - -The current recommended option on Linux and MacOS is [multi-user](#multi-user). - -## Multi-user - -This installation offers better sharing, improved isolation, and more security -over a single user installation. - -This option requires either: - -* Linux running systemd, with SELinux disabled -* MacOS - -> **Updating to macOS 15 Sequoia** -> -> If you recently updated to macOS 15 Sequoia and are getting -> ```console -> error: the user '_nixbld1' in the group 'nixbld' does not exist -> ``` -> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling. +We recommend that macOS users install Determinate Nix using our graphical installer, [Determinate.pkg][pkg]. +For Linux and Windows Subsystem for Linux (WSL) users: ```console -$ curl -L https://nixos.org/nix/install | sh -s -- --daemon -``` - -## Single-user - -> Single-user is not supported on Mac. - -> `warning: installing Nix as root is not supported by this script!` - -This installation has less requirements than the multi-user install, however it -cannot offer equivalent sharing, isolation, or security. - -This option is suitable for systems without systemd. - -```console -$ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon +curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | \ + sh -s -- install --determinate ``` ## Distributions @@ -46,3 +13,5 @@ $ curl -L https://nixos.org/nix/install | sh -s -- --no-daemon The Nix community maintains installers for several distributions. They can be found in the [`nix-community/nix-installers`](https://github.com/nix-community/nix-installers) repository. + +[pkg]: https://install.determinate.systems/determinate-pkg/stable/Universal diff --git a/doc/manual/source/installation/installing-binary.md b/doc/manual/source/installation/installing-binary.md deleted file mode 100644 index 21c15637437..00000000000 --- a/doc/manual/source/installation/installing-binary.md +++ /dev/null @@ -1,158 +0,0 @@ -# Installing a Binary Distribution - -> **Updating to macOS 15 Sequoia** -> -> If you recently updated to macOS 15 Sequoia and are getting -> ```console -> error: the user '_nixbld1' in the group 'nixbld' does not exist -> ``` -> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling. - -To install the latest version Nix, run the following command: - -```console -$ curl -L https://nixos.org/nix/install | sh -``` - -This performs the default type of installation for your platform: - -- [Multi-user](#multi-user-installation): - - Linux with systemd and without SELinux - - macOS -- [Single-user](#single-user-installation): - - Linux without systemd - - Linux with SELinux - -We recommend the multi-user installation if it supports your platform and you can authenticate with `sudo`. - -The installer can be configured with various command line arguments and environment variables. -To show available command line flags: - -```console -$ curl -L https://nixos.org/nix/install | sh -s -- --help -``` - -To check what it does and how it can be customised further, [download and edit the second-stage installation script](#installing-from-a-binary-tarball). - -# Installing a pinned Nix version from a URL - -Version-specific installation URLs for all Nix versions since 1.11.16 can be found at [releases.nixos.org](https://releases.nixos.org/?prefix=nix/). -The directory for each version contains the corresponding SHA-256 hash. - -All installation scripts are invoked the same way: - -```console -$ export VERSION=2.19.2 -$ curl -L https://releases.nixos.org/nix/nix-$VERSION/install | sh -``` - -# Multi User Installation - -The multi-user Nix installation creates system users and a system service for the Nix daemon. - -Supported systems: - -- Linux running systemd, with SELinux disabled -- macOS - -To explicitly instruct the installer to perform a multi-user installation on your system: - -```console -$ bash <(curl -L https://nixos.org/nix/install) --daemon -``` - -You can run this under your usual user account or `root`. -The script will invoke `sudo` as needed. - -# Single User Installation - -To explicitly select a single-user installation on your system: - -```console -$ bash <(curl -L https://nixos.org/nix/install) --no-daemon -``` - -In a single-user installation, `/nix` is owned by the invoking user. -The script will invoke `sudo` to create `/nix` if it doesn’t already exist. -If you don’t have `sudo`, manually create `/nix` as `root`: - -```console -$ su root -# mkdir /nix -# chown alice /nix -``` - -# Installing from a binary tarball - -You can also download a binary tarball that contains Nix and all its dependencies: -- Choose a [version](https://releases.nixos.org/?prefix=nix/) and [system type](../development/building.md#platforms) -- Download and unpack the tarball -- Run the installer - -> **Example** -> -> ```console -> $ pushd $(mktemp -d) -> $ export VERSION=2.19.2 -> $ export SYSTEM=x86_64-linux -> $ curl -LO https://releases.nixos.org/nix/nix-$VERSION/nix-$VERSION-$SYSTEM.tar.xz -> $ tar xfj nix-$VERSION-$SYSTEM.tar.xz -> $ cd nix-$VERSION-$SYSTEM -> $ ./install -> $ popd -> ``` - -The installer can be customised with the environment variables declared in the file named `install-multi-user`. - -## Native packages for Linux distributions - -The Nix community maintains installers for some Linux distributions in their native packaging format(https://nix-community.github.io/nix-installers/). - -# macOS Installation - - -[]{#sect-macos-installation-change-store-prefix}[]{#sect-macos-installation-encrypted-volume}[]{#sect-macos-installation-symlink}[]{#sect-macos-installation-recommended-notes} - -We believe we have ironed out how to cleanly support the read-only root file system -on modern macOS. New installs will do this automatically. - -This section previously detailed the situation, options, and trade-offs, -but it now only outlines what the installer does. You don't need to know -this to run the installer, but it may help if you run into trouble: - -- create a new APFS volume for your Nix store -- update `/etc/synthetic.conf` to direct macOS to create a "synthetic" - empty root directory to mount your volume -- specify mount options for the volume in `/etc/fstab` - - `rw`: read-write - - `noauto`: prevent the system from auto-mounting the volume (so the - LaunchDaemon mentioned below can control mounting it, and to avoid - masking problems with that mounting service). - - `nobrowse`: prevent the Nix Store volume from showing up on your - desktop; also keeps Spotlight from spending resources to index - this volume - -- if you have FileVault enabled - - generate an encryption password - - put it in your system Keychain - - use it to encrypt the volume -- create a system LaunchDaemon to mount this volume early enough in the - boot process to avoid problems loading or restoring any programs that - need access to your Nix store - diff --git a/doc/manual/source/installation/nix-security.md b/doc/manual/source/installation/nix-security.md index 1e9036b68b2..61cad24c2b3 100644 --- a/doc/manual/source/installation/nix-security.md +++ b/doc/manual/source/installation/nix-security.md @@ -1,15 +1,85 @@ # Security -Nix has two basic security models. First, it can be used in “single-user -mode”, which is similar to what most other package management tools do: -there is a single user (typically root) who performs all package -management operations. All other users can then use the installed -packages, but they cannot perform package management operations -themselves. - -Alternatively, you can configure Nix in “multi-user mode”. In this -model, all users can perform package management operations — for -instance, every user can install software without requiring root -privileges. Nix ensures that this is secure. For instance, it’s not -possible for one user to overwrite a package used by another user with a -Trojan horse. +Nix follows a [**multi-user**](#multi-user-model) security model in which all +users can perform package management operations. Every user can, for example, +install software without requiring root privileges, and Nix ensures that this +is secure. It's *not* possible for one user to, for example, overwrite a +package used by another user with a Trojan horse. + +## Multi-User model + +To allow a Nix store to be shared safely among multiple users, it is +important that users are not able to run builders that modify the Nix +store or database in arbitrary ways, or that interfere with builds +started by other users. If they could do so, they could install a Trojan +horse in some package and compromise the accounts of other users. + +To prevent this, the Nix store and database are owned by some privileged +user (usually `root`) and builders are executed under special user +accounts (usually named `nixbld1`, `nixbld2`, etc.). When a unprivileged +user runs a Nix command, actions that operate on the Nix store (such as +builds) are forwarded to a *Nix daemon* running under the owner of the +Nix store/database that performs the operation. + +> **Note** +> +> Multi-user mode has one important limitation: only root and a set of +> trusted users specified in `nix.conf` can specify arbitrary binary +> caches. So while unprivileged users may install packages from +> arbitrary Nix expressions, they may not get pre-built binaries. + +### Setting up the build users + +The *build users* are the special UIDs under which builds are performed. +They should all be members of the *build users group* `nixbld`. This +group should have no other members. The build users should not be +members of any other group. On Linux, you can create the group and users +as follows: + +```console +$ groupadd -r nixbld +$ for n in $(seq 1 10); do useradd -c "Nix build user $n" \ + -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(which nologin)" \ + nixbld$n; done +``` + +This creates 10 build users. There can never be more concurrent builds +than the number of build users, so you may want to increase this if you +expect to do many builds at the same time. + +### Running the daemon + +The [Nix daemon](../command-ref/nix-daemon.md) should be started as +follows (as `root`): + +```console +$ nix-daemon +``` + +You’ll want to put that line somewhere in your system’s boot scripts. + +To let unprivileged users use the daemon, they should set the +[`NIX_REMOTE` environment variable](../command-ref/env-common.md) to +`daemon`. So you should put a line like + +```console +export NIX_REMOTE=daemon +``` + +into the users’ login scripts. + +### Restricting access + +To limit which users can perform Nix operations, you can use the +permissions on the directory `/nix/var/nix/daemon-socket`. For instance, +if you want to restrict the use of Nix to the members of a group called +`nix-users`, do + +```console +$ chgrp nix-users /nix/var/nix/daemon-socket +$ chmod ug=rwx,o= /nix/var/nix/daemon-socket +``` + +This way, users who are not in the `nix-users` group cannot connect to +the Unix domain socket `/nix/var/nix/daemon-socket/socket`, so they +cannot perform Nix operations. diff --git a/doc/manual/source/installation/single-user.md b/doc/manual/source/installation/single-user.md deleted file mode 100644 index f9a3b26edf4..00000000000 --- a/doc/manual/source/installation/single-user.md +++ /dev/null @@ -1,9 +0,0 @@ -# Single-User Mode - -In single-user mode, all Nix operations that access the database in -`prefix/var/nix/db` or modify the Nix store in `prefix/store` must be -performed under the user ID that owns those directories. This is -typically root. (If you install from RPM packages, that’s in fact the -default ownership.) However, on single-user machines, it is often -convenient to `chown` those directories to your normal user account so -that you don’t have to `su` to root all the time. diff --git a/doc/manual/source/installation/supported-platforms.md b/doc/manual/source/installation/supported-platforms.md deleted file mode 100644 index 8ca3ce8d445..00000000000 --- a/doc/manual/source/installation/supported-platforms.md +++ /dev/null @@ -1,7 +0,0 @@ -# Supported Platforms - -Nix is currently supported on the following platforms: - - - Linux (i686, x86\_64, aarch64). - - - macOS (x86\_64, aarch64). diff --git a/doc/manual/source/installation/uninstall.md b/doc/manual/source/installation/uninstall.md index 69d59847b6f..e95634c213a 100644 --- a/doc/manual/source/installation/uninstall.md +++ b/doc/manual/source/installation/uninstall.md @@ -1,197 +1,15 @@ # Uninstalling Nix -## Multi User - -Removing a [multi-user installation](./installing-binary.md#multi-user-installation) depends on the operating system. - -### Linux - -If you are on Linux with systemd: - -1. Remove the Nix daemon service: - - ```console - sudo systemctl stop nix-daemon.service - sudo systemctl disable nix-daemon.socket nix-daemon.service - sudo systemctl daemon-reload - ``` - -Remove files created by Nix: +To uninstall Determinate Nix, use the uninstallation utility built into the [Determinate Nix Installer][installer]: ```console -sudo rm -rf /etc/nix /etc/profile.d/nix.sh /etc/tmpfiles.d/nix-daemon.conf /nix ~root/.nix-channels ~root/.nix-defexpr ~root/.nix-profile ~root/.cache/nix +$ /nix/nix-installer uninstall ``` -Remove build users and their group: +If you're certain that you want to uninstall, you can skip the confirmation step: ```console -for i in $(seq 1 32); do - sudo userdel nixbld$i -done -sudo groupdel nixbld +$ /nix/nix-installer uninstall --no-confirm ``` -There may also be references to Nix in - -- `/etc/bash.bashrc` -- `/etc/bashrc` -- `/etc/profile` -- `/etc/zsh/zshrc` -- `/etc/zshrc` - -which you may remove. - -### FreeBSD - -1. Stop and remove the Nix daemon service: - - ```console - sudo service nix-daemon stop - sudo rm -f /usr/local/etc/rc.d/nix-daemon - sudo sysrc -x nix_daemon_enable - ``` - -2. Remove files created by Nix: - - ```console - sudo rm -rf /etc/nix /usr/local/etc/profile.d/nix.sh /nix ~root/.nix-channels ~root/.nix-defexpr ~root/.nix-profile ~root/.cache/nix - ``` - -3. Remove build users and their group: - - ```console - for i in $(seq 1 32); do - sudo pw userdel nixbld$i - done - sudo pw groupdel nixbld - ``` - -4. There may also be references to Nix in: - - `/usr/local/etc/bashrc` - - `/usr/local/etc/zshrc` - - Shell configuration files in users' home directories - - which you may remove. - -### macOS - -> **Updating to macOS 15 Sequoia** -> -> If you recently updated to macOS 15 Sequoia and are getting -> ```console -> error: the user '_nixbld1' in the group 'nixbld' does not exist -> ``` -> when running Nix commands, refer to GitHub issue [NixOS/nix#10892](https://github.com/NixOS/nix/issues/10892) for instructions to fix your installation without reinstalling. - -1. If system-wide shell initialisation files haven't been altered since installing Nix, use the backups made by the installer: - - ```console - sudo mv /etc/zshrc.backup-before-nix /etc/zshrc - sudo mv /etc/bashrc.backup-before-nix /etc/bashrc - sudo mv /etc/bash.bashrc.backup-before-nix /etc/bash.bashrc - ``` - - Otherwise, edit `/etc/zshrc`, `/etc/bashrc`, and `/etc/bash.bashrc` to remove the lines sourcing `nix-daemon.sh`, which should look like this: - - ```bash - # Nix - if [ -e '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' ]; then - . '/nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh' - fi - # End Nix - ``` - -2. Stop and remove the Nix daemon services: - - ```console - sudo launchctl unload /Library/LaunchDaemons/org.nixos.nix-daemon.plist - sudo rm /Library/LaunchDaemons/org.nixos.nix-daemon.plist - sudo launchctl unload /Library/LaunchDaemons/org.nixos.darwin-store.plist - sudo rm /Library/LaunchDaemons/org.nixos.darwin-store.plist - ``` - - This stops the Nix daemon and prevents it from being started next time you boot the system. - -3. Remove the `nixbld` group and the `_nixbuildN` users: - - ```console - sudo dscl . -delete /Groups/nixbld - for u in $(sudo dscl . -list /Users | grep _nixbld); do sudo dscl . -delete /Users/$u; done - ``` - - This will remove all the build users that no longer serve a purpose. - -4. Edit fstab using `sudo vifs` to remove the line mounting the Nix Store volume on `/nix`, which looks like - - ``` - UUID= /nix apfs rw,noauto,nobrowse,suid,owners - ``` - or - - ``` - LABEL=Nix\040Store /nix apfs rw,nobrowse - ``` - - by setting the cursor on the respective line using the arrow keys, and pressing `dd`, and then `:wq` to save the file. - - This will prevent automatic mounting of the Nix Store volume. - -5. Edit `/etc/synthetic.conf` to remove the `nix` line. - If this is the only line in the file you can remove it entirely: - - ```bash - if [ -f /etc/synthetic.conf ]; then - if [ "$(cat /etc/synthetic.conf)" = "nix" ]; then - sudo rm /etc/synthetic.conf - else - sudo vi /etc/synthetic.conf - fi - fi - ``` - - This will prevent the creation of the empty `/nix` directory. - -6. Remove the files Nix added to your system, except for the store: - - ```console - sudo rm -rf /etc/nix /var/root/.nix-profile /var/root/.nix-defexpr /var/root/.nix-channels ~/.nix-profile ~/.nix-defexpr ~/.nix-channels - ``` - - -7. Remove the Nix Store volume: - - ```console - sudo diskutil apfs deleteVolume /nix - ``` - - This will remove the Nix Store volume and everything that was added to the store. - - If the output indicates that the command couldn't remove the volume, you should make sure you don't have an _unmounted_ Nix Store volume. - Look for a "Nix Store" volume in the output of the following command: - - ```console - diskutil list - ``` - - If you _do_ find a "Nix Store" volume, delete it by running `diskutil apfs deleteVolume` with the store volume's `diskXsY` identifier. - - If you get an error that the volume is in use by the kernel, reboot and immediately delete the volume before starting any other process. - -> **Note** -> -> After you complete the steps here, you will still have an empty `/nix` directory. -> This is an expected sign of a successful uninstall. -> The empty `/nix` directory will disappear the next time you reboot. -> -> You do not have to reboot to finish uninstalling Nix. -> The uninstall is complete. -> macOS (Catalina+) directly controls root directories, and its read-only root will prevent you from manually deleting the empty `/nix` mountpoint. - -## Single User - -To remove a [single-user installation](./installing-binary.md#single-user-installation) of Nix, run: - -```console -rm -rf /nix ~/.nix-channels ~/.nix-defexpr ~/.nix-profile -``` -You might also want to manually remove references to Nix from your `~/.profile`. +[installer]: https://github.com/DeterminateSystems/nix-installer diff --git a/doc/manual/source/installation/upgrading.md b/doc/manual/source/installation/upgrading.md index a433f1d30e6..8fe342b09b7 100644 --- a/doc/manual/source/installation/upgrading.md +++ b/doc/manual/source/installation/upgrading.md @@ -1,40 +1,10 @@ # Upgrading Nix -> **Note** -> -> These upgrade instructions apply where Nix was installed following the [installation instructions in this manual](./index.md). - -Check which Nix version will be installed, for example from one of the [release channels](http://channels.nixos.org/) such as `nixpkgs-unstable`: - -```console -$ nix-shell -p nix -I nixpkgs=channel:nixpkgs-unstable --run "nix --version" -nix (Nix) 2.18.1 -``` - -> **Warning** -> -> Writing to the [local store](@docroot@/store/types/local-store.md) with a newer version of Nix, for example by building derivations with [`nix-build`](@docroot@/command-ref/nix-build.md) or [`nix-store --realise`](@docroot@/command-ref/nix-store/realise.md), may change the database schema! -> Reverting to an older version of Nix may therefore require purging the store database before it can be used. - -## Linux multi-user +You can upgrade Determinate Nix using Determinate Nixd: ```console -$ sudo su -# nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable -# systemctl daemon-reload -# systemctl restart nix-daemon +sudo determinate-nixd upgrade ``` -## macOS multi-user +Note that the `sudo` is necessary here and upgrading fails without it. -```console -$ sudo nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable -$ sudo launchctl remove org.nixos.nix-daemon -$ sudo launchctl load /Library/LaunchDaemons/org.nixos.nix-daemon.plist -``` - -## Single-user all platforms - -```console -$ nix-env --install --file '' --attr nix cacert -I nixpkgs=channel:nixpkgs-unstable -``` diff --git a/doc/manual/source/introduction.md b/doc/manual/source/introduction.md index e70411c11f5..fedb5595a1d 100644 --- a/doc/manual/source/introduction.md +++ b/doc/manual/source/introduction.md @@ -1,4 +1,19 @@ -# Introduction +# Determinate Nix + +**Determinate Nix** is a downstream distribution of [Nix], a purely functional language, CLI tool, and package management system. +It's available on Linux, macOS, and Windows Subsystem for Linux (WSL). + +## Installing + +We recommend that macOS users install Determinate Nix using our graphical installer, [Determinate.pkg][pkg]. +For Linux and Windows Subsystem for Linux (WSL) users: + +```console +curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | \ + sh -s -- install --determinate +``` + +## How Nix works Nix is a _purely functional package manager_. This means that it treats packages like values in a purely functional programming language @@ -184,10 +199,14 @@ to build configuration files in `/etc`). This means, among other things, that it is easy to roll back the entire configuration of the system to an earlier state. Also, users can install software without root privileges. For more information and downloads, see the [NixOS -homepage](https://nixos.org/). +homepage][nix]. ## License Nix is released under the terms of the [GNU LGPLv2.1 or (at your option) any later -version](http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html). +version][license]. + +[license]: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html +[pkg]: https://install.determinate.systems/determinate-pkg/stable/Universal +[site]: https://nixos.org diff --git a/doc/manual/source/language/builtins-prefix.md b/doc/manual/source/language/builtins-prefix.md index fb983bb7f3c..fff0f7cb5e4 100644 --- a/doc/manual/source/language/builtins-prefix.md +++ b/doc/manual/source/language/builtins-prefix.md @@ -5,12 +5,28 @@ All built-ins are available through the global [`builtins`](#builtins-builtins) Some built-ins are also exposed directly in the global scope: - - - [`derivation`](#builtins-derivation) -- [`import`](#builtins-import) +- `derivationStrict` - [`abort`](#builtins-abort) +- [`baseNameOf`](#builtins-baseNameOf) +- [`break`](#builtins-break) +- [`dirOf`](#builtins-dirOf) +- [`false`](#builtins-false) +- [`fetchGit`](#builtins-fetchGit) +- `fetchMercurial` +- [`fetchTarball`](#builtins-fetchTarball) +- [`fetchTree`](#builtins-fetchTree) +- [`fromTOML`](#builtins-fromTOML) +- [`import`](#builtins-import) +- [`isNull`](#builtins-isNull) +- [`map`](#builtins-map) +- [`null`](#builtins-null) +- [`placeholder`](#builtins-placeholder) +- [`removeAttrs`](#builtins-removeAttrs) +- `scopedImport` - [`throw`](#builtins-throw) +- [`toString`](#builtins-toString) +- [`true`](#builtins-true)
derivation attrs
diff --git a/doc/manual/source/protocols/json/derivation.md b/doc/manual/source/protocols/json/derivation.md index 04881776abc..86854cf2655 100644 --- a/doc/manual/source/protocols/json/derivation.md +++ b/doc/manual/source/protocols/json/derivation.md @@ -1,11 +1,5 @@ # Derivation JSON Format -> **Warning** -> -> This JSON format is currently -> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) -> and subject to change. - The JSON serialization of a [derivations](@docroot@/glossary.md#gloss-store-derivation) is a JSON object with the following fields: @@ -14,6 +8,21 @@ is a JSON object with the following fields: The name of the derivation. This is used when calculating the store paths of the derivation's outputs. +* `version`: + Must be `3`. + This is a guard that allows us to continue evolving this format. + The choice of `3` is fairly arbitrary, but corresponds to this informal version: + + - Version 0: A-Term format + + - Version 1: Original JSON format, with ugly `"r:sha256"` inherited from A-Term format. + + - Version 2: Separate `method` and `hashAlgo` fields in output specs + + - Verison 3: Drop store dir from store paths, just include base name. + + Note that while this format is experimental, the maintenance of versions is best-effort, and not promised to identify every change. + * `outputs`: Information about the output paths of the derivation. This is a JSON object with one member per output, where the key is the output name and the value is a JSON object with these fields: @@ -52,7 +61,6 @@ is a JSON object with the following fields: > ```json > "outputs": { > "out": { - > "path": "/nix/store/2543j7c6jn75blc3drf4g5vhb1rhdq29-source", > "method": "nar", > "hashAlgo": "sha256", > "hash": "6fc80dcc62179dbc12fc0b5881275898f93444833d21b89dfe5f7fbcbb1d0d62" @@ -63,6 +71,15 @@ is a JSON object with the following fields: * `inputSrcs`: A list of store paths on which this derivation depends. + > **Example** + > + > ```json + > "inputSrcs": [ + > "47y241wqdhac3jm5l7nv0x4975mb1975-separate-debug-info.sh", + > "56d0w71pjj9bdr363ym3wj1zkwyqq97j-fix-pop-var-context-error.patch" + > ] + > ``` + * `inputDrvs`: A JSON object specifying the derivations on which this derivation depends, and what outputs of those derivations. @@ -70,8 +87,8 @@ is a JSON object with the following fields: > > ```json > "inputDrvs": { - > "/nix/store/6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], - > "/nix/store/fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] + > "6lkh5yi7nlb7l6dr8fljlli5zfd9hq58-curl-7.73.0.drv": ["dev"], + > "fn3kgnfzl5dzym26j8g907gq3kbm8bfh-unzip-6.0.drv": ["out"] > } > ``` diff --git a/doc/manual/source/protocols/json/store-object-info.md b/doc/manual/source/protocols/json/store-object-info.md index b7348538c35..4b029c40b5d 100644 --- a/doc/manual/source/protocols/json/store-object-info.md +++ b/doc/manual/source/protocols/json/store-object-info.md @@ -1,11 +1,5 @@ # Store object info JSON format -> **Warning** -> -> This JSON format is currently -> [**experimental**](@docroot@/development/experimental-features.md#xp-feature-nix-command) -> and subject to change. - Info about a [store object]. * `path`: diff --git a/doc/manual/source/quick-start.md b/doc/manual/source/quick-start.md index 9eb7a326590..ffb87aa725f 100644 --- a/doc/manual/source/quick-start.md +++ b/doc/manual/source/quick-start.md @@ -3,10 +3,13 @@ This chapter is for impatient people who don't like reading documentation. For more in-depth information you are kindly referred to subsequent chapters. -1. Install Nix: +1. Install Nix. + We recommend that macOS users install Determinate Nix using our graphical installer, [Determinate.pkg][pkg]. + For Linux and Windows Subsystem for Linux (WSL) users: ```console - $ curl -L https://nixos.org/nix/install | sh + $ curl --proto '=https' --tlsv1.2 -sSf -L https://install.determinate.systems/nix | \ + sh -s -- install --determinate ``` The install script will use `sudo`, so make sure you have sufficient rights. @@ -41,3 +44,5 @@ For more in-depth information you are kindly referred to subsequent chapters. ```console $ nix-collect-garbage ``` + +[pkg]: https://install.determinate.systems/determinate-pkg/stable/Universal diff --git a/doc/manual/source/release-notes-determinate/changes.md b/doc/manual/source/release-notes-determinate/changes.md new file mode 100644 index 00000000000..c9960bd1e8d --- /dev/null +++ b/doc/manual/source/release-notes-determinate/changes.md @@ -0,0 +1,131 @@ +# Changes between Nix and Determinate Nix + +This section lists the differences between upstream Nix 2.32 and Determinate Nix 3.12.0. + +* In Determinate Nix, flakes are stable. You no longer need to enable the `flakes` experimental feature. + +* In Determinate Nix, the new Nix CLI (i.e. the `nix` command) is stable. You no longer need to enable the `nix-command` experimental feature. + +* Determinate Nix has a setting [`json-log-path`](@docroot@/command-ref/conf-file.md#conf-json-log-path) to send a copy of all Nix log messages (in JSON format) to a file or Unix domain socket. + +* Determinate Nix has made `nix profile install` an alias to `nix profile add`, a more symmetrical antonym of `nix profile remove`. + +* `nix-channel` and `channel:` url syntax (like `channel:nixos-24.11`) is deprecated, see: https://github.com/DeterminateSystems/nix-src/issues/34 + +* Using indirect flake references and implicit inputs is deprecated, see: https://github.com/DeterminateSystems/nix-src/issues/37 + +* Warnings around "dirty trees" are updated to reduce "dirty" jargon, and now refers to "uncommitted changes". + + + + + + + +* `nix upgrade-nix` is now inert, and suggests using `determinate-nixd upgrade` -- [DeterminateSystems/nix-src#55](https://github.com/DeterminateSystems/nix-src/pull/55) + +* Lazy Trees support has been merged. ([DeterminateSystems/nix-src#27](https://github.com/DeterminateSystems/nix-src/pull/27), [DeterminateSystems/nix-src#56](https://github.com/DeterminateSystems/nix-src/pull/56)) + + + + + + + + + +* Faster `nix store copy-sigs` by @edolstra in [DeterminateSystems/nix-src#80](https://github.com/DeterminateSystems/nix-src/pull/80) + +* Document how to replicate nix-store --query --deriver with the nix cli by @grahamc in [DeterminateSystems/nix-src#82](https://github.com/DeterminateSystems/nix-src/pull/82) + +* nix profile: Replace ε and ∅ with descriptive English words by @grahamc in [DeterminateSystems/nix-src#81](https://github.com/DeterminateSystems/nix-src/pull/81) + +* Call out that `--keep-failed` with remote builders will keep the failed build directory on that builder by @cole-h in [DeterminateSystems/nix-src#85](https://github.com/DeterminateSystems/nix-src/pull/85) + + + + + + +* When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) + +* Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) + +* Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) + +* Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) + +* Improve caching of inputs in dry-run mode by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98) + + + + + + + +* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) + + + +* `nix store delete` now explains why deletion fails by @edolstra in [DeterminateSystems/nix-src#130](https://github.com/DeterminateSystems/nix-src/pull/130) + + + + + + + +* ci: don't run the full test suite for x86_64-darwin by @grahamc in [DeterminateSystems/nix-src#144](https://github.com/DeterminateSystems/nix-src/pull/144) + + + + + + + +* Tab completing arguments to Nix avoids network access [DeterminateSystems/nix-src#161](https://github.com/DeterminateSystems/nix-src/pull/161) + +* Importing Nixpkgs and other tarballs to the cache is 2-4x faster [DeterminateSystems/nix-src#149](https://github.com/DeterminateSystems/nix-src/pull/149) + +* Adding paths to the store is significantly faster [DeterminateSystems/nix-src#162](https://github.com/DeterminateSystems/nix-src/pull/162) + + + + + +* Build-time flake inputs [DeterminateSystems/nix-src#49](https://github.com/DeterminateSystems/nix-src/pull/49) + + + +* The default `nix flake init` template is much more useful [DeterminateSystems/nix-src#180](https://github.com/DeterminateSystems/nix-src/pull/180) + + + + + + + + +* Multithreaded evaluation support [DeterminateSystems/nix-src#125](https://github.com/DeterminateSystems/nix-src/pull/125) + + + + + + +* Fix some interactions with the registry and flakes that include a `?dir=` parameter [DeterminateSystems/nix-src#196](https://github.com/DeterminateSystems/nix-src/pull/196), [DeterminateSystems/nix-src#199](https://github.com/DeterminateSystems/nix-src/pull/199) + +* Only try to substitute input if fetching from its original location fails [DeterminateSystems/nix-src#202](https://github.com/DeterminateSystems/nix-src/pull/202) + + + + + + +* `nix flake clone` supports arbitrary input types. [DeterminateSystems/nix-src#229](https://github.com/DeterminateSystems/nix-src/pull/229) + +* A new command `nix nario` that replaces `nix-store --export|--export`. It also has a new file format (`--format 2`) that supports store path attributes such as signatures, and that can be imported more efficiently. +[DeterminateSystems/nix-src#215](https://github.com/DeterminateSystems/nix-src/pull/215) + +* Determinate Nix prints the Nix version when using `-vv` or higher verbosity. [DeterminateSystems/nix-src#237](https://github.com/DeterminateSystems/nix-src/pull/237) + diff --git a/doc/manual/source/release-notes-determinate/index.md b/doc/manual/source/release-notes-determinate/index.md new file mode 100644 index 00000000000..bba33084424 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/index.md @@ -0,0 +1,3 @@ +# Determinate Nix Release Notes + +This chapter lists the differences between Nix and Determinate Nix, as well as the release history of Determinate Nix. diff --git a/doc/manual/source/release-notes-determinate/rl-3.0.0.md b/doc/manual/source/release-notes-determinate/rl-3.0.0.md new file mode 100644 index 00000000000..d60786e9a72 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.0.0.md @@ -0,0 +1,5 @@ +# Release 3.0.0 (2025-03-04) + +* Initial release of Determinate Nix. + +* Based on [upstream Nix 2.26.2](../release-notes/rl-2.26.md). diff --git a/doc/manual/source/release-notes-determinate/rl-3.1.0.md b/doc/manual/source/release-notes-determinate/rl-3.1.0.md new file mode 100644 index 00000000000..96b7819d08d --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.1.0.md @@ -0,0 +1,5 @@ +# Release 3.1.0 (2025-03-27) + +* Based on [upstream Nix 2.27.1](../release-notes/rl-2.27.md). + +* New setting `json-log-path` that sends a copy of all Nix log messages (in JSON format) to a file or Unix domain socket. diff --git a/doc/manual/source/release-notes-determinate/rl-3.3.0.md b/doc/manual/source/release-notes-determinate/rl-3.3.0.md new file mode 100644 index 00000000000..badf96415df --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.3.0.md @@ -0,0 +1,5 @@ +# Release 3.3.0 (2025-04-11) + +* Based on [upstream Nix 2.28.1](../release-notes/rl-2.28.md). + +* The `nix profile install` command is now an alias to `nix profile add`, a more symmetrical antonym of `nix profile remove`. diff --git a/doc/manual/source/release-notes-determinate/rl-3.4.0.md b/doc/manual/source/release-notes-determinate/rl-3.4.0.md new file mode 100644 index 00000000000..24ae03ca554 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.4.0.md @@ -0,0 +1,50 @@ +# Release 3.4.0 (2025-04-25) + +* Based on [upstream Nix 2.28.2](../release-notes/rl-2.28.md). + +* **Warn users that `nix-channel` is deprecated.** + +This is the first change accomplishing our roadmap item of deprecating Nix channels: https://github.com/DeterminateSystems/nix-src/issues/34 + +This is due to user confusion and surprising behavior of channels, especially in the context of user vs. root channels. + +The goal of this change is to make the user experience of Nix more predictable. +In particular, these changes are to support users with lower levels of experience who are following guides that focus on channels as the mechanism of distribution. + +Users will now see this message: + +> nix-channel is deprecated in favor of flakes in Determinate Nix. For a guide on Nix flakes, see: https://zero-to-nix.com/. or details and to offer feedback on the deprecation process, see: https://github.com/DeterminateSystems/nix-src/issues/34. + + +* **Warn users that `channel:` URLs are deprecated.** + +This is the second change regarding our deprecation of Nix channels. +Using a `channel:` URL (like `channel:nixos-24.11`) will yield a warning like this: + +> Channels are deprecated in favor of flakes in Determinate Nix. Instead of 'channel:nixos-24.11', use 'https://nixos.org/channels/nixos-24.11/nixexprs.tar.xz'. For a guide on Nix flakes, see: https://zero-to-nix.com/. For details and to offer feedback on the deprecation process, see: https://github.com/DeterminateSystems/nix-src/issues/34. + +* **Warn users against indirect flake references in `flake.nix` inputs** + +This is the first change accomplishing our roadmap item of deprecating implicit and indirect flake inputs: https://github.com/DeterminateSystems/nix-src/issues/37 + +The flake registry provides an important UX affordance for using Nix flakes and remote sources in command line uses. +For that reason, the registry is not being deprecated entirely and will still be used for command-line incantations, like nix run. + +This move will eliminate user confusion and surprising behavior around global and local registries during flake input resolution. + +The goal of this change is to make the user experience of Nix more predictable. +We have seen a pattern of confusion when using automatic flake inputs and local registries. +Specifically, users' flake inputs resolving and locking inconsistently depending on the configuration of the host system. + +Users will now see the following warning if their flake.nix uses an implicit or indirect Flake reference input: + +> Flake input 'nixpkgs' uses the flake registry. Using the registry in flake inputs is deprecated in Determinate Nix. To make your flake future-proof, add the following to 'xxx/flake.nix': +> +> inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.11"; +> +> For more information, see: https://github.com/DeterminateSystems/nix-src/issues/37 + + +### Other updates: +* Improve the "dirty tree" message. Determinate Nix will now say `Git tree '...' has uncommitted changes` instead of `Git tree '...' is dirty` +* Stop warning about uncommitted changes in a Git repository when using `nix develop` diff --git a/doc/manual/source/release-notes-determinate/rl-3.4.2.md b/doc/manual/source/release-notes-determinate/rl-3.4.2.md new file mode 100644 index 00000000000..8acabd4425f --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.4.2.md @@ -0,0 +1,4 @@ +# Release 3.4.2 (2025-05-05) + +* Based on [upstream Nix 2.28.3](../release-notes/rl-2.28.md). + diff --git a/doc/manual/source/release-notes-determinate/rl-3.5.0.md b/doc/manual/source/release-notes-determinate/rl-3.5.0.md new file mode 100644 index 00000000000..d5b26b9419e --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.5.0.md @@ -0,0 +1,4 @@ +# Release 3.5.0 (2025-05-09) + +* Based on [upstream Nix 2.28.3](../release-notes/rl-2.28.md). + diff --git a/doc/manual/source/release-notes-determinate/rl-3.5.1.md b/doc/manual/source/release-notes-determinate/rl-3.5.1.md new file mode 100644 index 00000000000..b0813ca59c9 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.5.1.md @@ -0,0 +1,57 @@ +# Release 3.5.1 (2025-05-09) + +* Based on [upstream Nix 2.28.3](../release-notes/rl-2.28.md). + +## What's Changed + +Most notably, Lazy Trees has merged in to Determinate Nix and is in Feature Preview status, but remains disabled by default. +Lazy trees massively improves performance in virtually all scenarios because it enables Nix to avoid making unnecessary copies of files into the Nix store. +In testing, we saw iteration times on Nixpkgs **drop from over 12 seconds to 3.5 seconds**. + +After upgrading to Determinate Nix 3.5.1 with `sudo determinate-nixd upgrade`, enable lazy trees by adding this to `/etc/nix/nix.custom.conf`: + +``` +lazy-trees = true +``` + +Please note that our full flake regression test suite passes with no changes with lazy trees, and please report compatibility issues. + +Read [this GitHub comment](https://github.com/DeterminateSystems/nix-src/pull/27#pullrequestreview-2822153088) for further details and next steps. +We'll be publishing an update on the [Determinate Systems blog](https://determinate.systems/posts/) in the next few days with more information as well. + +Relevant PRs: +* Lazy trees v2 by @edolstra in [DeterminateSystems/nix-src#27](https://github.com/DeterminateSystems/nix-src/pull/27) +* Improve lazy trees backward compatibility by @edolstra in [DeterminateSystems/nix-src#56](https://github.com/DeterminateSystems/nix-src/pull/56) + + +### Additional changes in this release: +* Bug fix: Flake input URLs are canonicalized before checking flake.lock file staleness, avoiding needlessly regenerating flake.lock files with `dir` in URL-style flakerefs by @edolstra in [DeterminateSystems/nix-src#57](https://github.com/DeterminateSystems/nix-src/pull/57) +* `nix upgrade-nix` is deprecated in favor of `determinate-nixd upgrade`, by @gustavderdrache in [DeterminateSystems/nix-src#55](https://github.com/DeterminateSystems/nix-src/pull/55) +* UX: Improved build failure and dependency failure error messages to include needed output paths by @edolstra in [DeterminateSystems/nix-src#58](https://github.com/DeterminateSystems/nix-src/pull/58). + +Previously: + +``` +error: builder for '/nix/store/[...]-nested-failure-bottom.drv' failed with exit code 1 +error: 1 dependencies of derivation '/nix/store/[...]-nested-failure-middle.drv' failed to build +error: 1 dependencies of derivation '/nix/store/[...]-nested-failure-top.drv' failed to build +``` + +Now: + +``` +error: Cannot build '/nix/store/w37gflm9wz9dcnsgy3sfrmnlvm8qigaj-nested-failure-bottom.drv'. + Reason: builder failed with exit code 1. + Output paths: + /nix/store/yzybs8kp35dfipbzdlqcc6lxz62hax04-nested-failure-bottom +error: Cannot build '/nix/store/00gr5hlxfc03x2675w6nn3pwfrz2fr62-nested-failure-middle.drv'. + Reason: 1 dependency failed. + Output paths: + /nix/store/h781j5h4bdchmb4c2lvy8qzh8733azhz-nested-failure-middle +error: Cannot build '/nix/store/8am0ng1gyx8sbzyr0yx6jd5ix3yy5szc-nested-failure-top.drv'. + Reason: 1 dependency failed. + Output paths: + /nix/store/fh12637kgvp906s9yhi9w2dc7ghfwxs1-nested-failure-top +``` + +**Full Changelog**: [v3.4.2...v3.5.1](https://github.com/DeterminateSystems/nix-src/compare/v3.4.2...v3.5.1) diff --git a/doc/manual/source/release-notes-determinate/rl-3.5.2.md b/doc/manual/source/release-notes-determinate/rl-3.5.2.md new file mode 100644 index 00000000000..bc5396c255b --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.5.2.md @@ -0,0 +1,11 @@ +# Release 3.5.2 (2025-05-12) + +* Based on [upstream Nix 2.28.3](../release-notes/rl-2.28.md). + +## What's Changed +* Fix a regression where narHash was not added to lock files when lazy trees were disabled by @edolstra in [DeterminateSystems/nix-src#63](https://github.com/DeterminateSystems/nix-src/pull/63) + +* Tell users a source is corrupted ("cannot read file from tarball: Truncated tar archive detected while reading data"), improving over the previous 'cannot read file from tarball' error by @edolstra in [DeterminateSystems/nix-src#64](https://github.com/DeterminateSystems/nix-src/pull/64) + + +**Full Changelog**: [v3.5.1...v3.5.2](https://github.com/DeterminateSystems/nix-src/compare/v3.5.1...v3.5.2) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.0.md b/doc/manual/source/release-notes-determinate/rl-3.6.0.md new file mode 100644 index 00000000000..453ab6c301d --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.0.md @@ -0,0 +1,11 @@ +# Release 3.6.0 (2025-05-22) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Install 'nix profile add' manpage by @edolstra in [DeterminateSystems/nix-src#69](https://github.com/DeterminateSystems/nix-src/pull/69) +* Sync with upstream 2.29.0 by @edolstra in [DeterminateSystems/nix-src#67](https://github.com/DeterminateSystems/nix-src/pull/67) +* Emit warnings when using import-from-derivation by setting the `trace-import-from-derivation` option to `true` by @gustavderdrache in [DeterminateSystems/nix-src#70](https://github.com/DeterminateSystems/nix-src/pull/70) + + +**Full Changelog**: [v3.5.2...v3.6.0](https://github.com/DeterminateSystems/nix-src/compare/v3.5.2...v3.6.0) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.1.md b/doc/manual/source/release-notes-determinate/rl-3.6.1.md new file mode 100644 index 00000000000..12505afee27 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.1.md @@ -0,0 +1,9 @@ +# Release 3.6.1 (2025-05-24) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Fix nlohmann error in fromStructuredAttrs() by @edolstra in [DeterminateSystems/nix-src#73](https://github.com/DeterminateSystems/nix-src/pull/73) + + +**Full Changelog**: [v3.6.0...v3.6.1](https://github.com/DeterminateSystems/nix-src/compare/v3.6.0...v3.6.1) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.2.md b/doc/manual/source/release-notes-determinate/rl-3.6.2.md new file mode 100644 index 00000000000..882c142f00c --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.2.md @@ -0,0 +1,15 @@ +# Release 3.6.2 (2025-06-02) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* Dramatically improve the performance of nix store copy-sigs: Use http-connections setting to control parallelism by @edolstra in [DeterminateSystems/nix-src#80](https://github.com/DeterminateSystems/nix-src/pull/80) +* Document how to replicate nix-store --query --deriver with the nix cli by @grahamc in [DeterminateSystems/nix-src#82](https://github.com/DeterminateSystems/nix-src/pull/82) +* The garbage collector no longer gives up if it encounters an undeletable file, by @edolstra in [DeterminateSystems/nix-src#83](https://github.com/DeterminateSystems/nix-src/pull/83) +* nix profile: Replace ε and ∅ with descriptive English words by @grahamc in [DeterminateSystems/nix-src#81](https://github.com/DeterminateSystems/nix-src/pull/81) +* Rework README to clarify that this distribution is our distribution, by @lucperkins in [DeterminateSystems/nix-src#84](https://github.com/DeterminateSystems/nix-src/pull/84) +* Include the source location when warning about inefficient double copies by @edolstra in [DeterminateSystems/nix-src#79](https://github.com/DeterminateSystems/nix-src/pull/79) +* Call out that `--keep-failed` with remote builders will keep the failed build directory on that builder by @cole-h in [DeterminateSystems/nix-src#85](https://github.com/DeterminateSystems/nix-src/pull/85) + + +**Full Changelog**: [v3.6.1...v3.6.2](https://github.com/DeterminateSystems/nix-src/compare/v3.6.1...v3.6.2) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.5.md b/doc/manual/source/release-notes-determinate/rl-3.6.5.md new file mode 100644 index 00000000000..8ef5be0fd0d --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.5.md @@ -0,0 +1,19 @@ +# Release 3.6.5 (2025-06-12) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed +* When remote building with --keep-failed, only show "you can rerun" message if the derivation's platform is supported on this machine by @cole-h in [DeterminateSystems/nix-src#87](https://github.com/DeterminateSystems/nix-src/pull/87) +* Indicate that sandbox-paths specifies a missing file in the corresponding error message. by @cole-h in [DeterminateSystems/nix-src#88](https://github.com/DeterminateSystems/nix-src/pull/88) +* Render lazy tree paths in messages withouth the/nix/store/hash... prefix in substituted source trees by @edolstra in [DeterminateSystems/nix-src#91](https://github.com/DeterminateSystems/nix-src/pull/91) +* Use FlakeHub inputs by @lucperkins in [DeterminateSystems/nix-src#89](https://github.com/DeterminateSystems/nix-src/pull/89) +* Proactively cache more flake inputs and fetches by @edolstra in [DeterminateSystems/nix-src#93](https://github.com/DeterminateSystems/nix-src/pull/93) +* Fix: register extra builtins just once by @edolstra in [DeterminateSystems/nix-src#97](https://github.com/DeterminateSystems/nix-src/pull/97) +* Fix the link to `builders-use-substitutes` documentation for `builders` by @lucperkins in [DeterminateSystems/nix-src#102](https://github.com/DeterminateSystems/nix-src/pull/102) +* Improve error messages that use the hypothetical future tense of "will" by @lucperkins in [DeterminateSystems/nix-src#92](https://github.com/DeterminateSystems/nix-src/pull/92) +* Make the `nix repl` test more stable by @edolstra in [DeterminateSystems/nix-src#103](https://github.com/DeterminateSystems/nix-src/pull/103) +* Run nixpkgsLibTests against lazy trees by @edolstra in [DeterminateSystems/nix-src#100](https://github.com/DeterminateSystems/nix-src/pull/100) +* Run the Nix test suite against lazy trees by @edolstra in [DeterminateSystems/nix-src#105](https://github.com/DeterminateSystems/nix-src/pull/105) +* Improve caching of inputs by @edolstra in [DeterminateSystems/nix-src#98](https://github.com/DeterminateSystems/nix-src/pull/98), [DeterminateSystems/nix-src#110](https://github.com/DeterminateSystems/nix-src/pull/110), and [DeterminateSystems/nix-src#115](https://github.com/DeterminateSystems/nix-src/pull/115) + +**Full Changelog**: [v3.6.2...v3.6.5](https://github.com/DeterminateSystems/nix-src/compare/v3.6.2...v3.6.4) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.6.md b/doc/manual/source/release-notes-determinate/rl-3.6.6.md new file mode 100644 index 00000000000..bf4e3690afa --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.6.md @@ -0,0 +1,7 @@ +# Release 3.6.6 (2025-06-17) + +* Based on [upstream Nix 2.29.0](../release-notes/rl-2.29.md). + +## What's Changed + +* No-op release on the nix-src side, due to a regression on nix-darwin in determinate-nixd. diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.7.md b/doc/manual/source/release-notes-determinate/rl-3.6.7.md new file mode 100644 index 00000000000..197587f1b3a --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.7.md @@ -0,0 +1,17 @@ +# Release 3.6.7 (2025-06-24) + +* Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). + +## What's Changed + +### Security contents + +* Patched against GHSA-g948-229j-48j3 + +### Lazy trees: + +* Lazy trees now produces `flake.lock` files with NAR hashes unless `lazy-locks` is set to `true` by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) +* Improved caching with lazy-trees when using --impure, with enhanced testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) + + +**Full Changelog**: [v3.6.6...v3.6.7](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.7) diff --git a/doc/manual/source/release-notes-determinate/rl-3.6.8.md b/doc/manual/source/release-notes-determinate/rl-3.6.8.md new file mode 100644 index 00000000000..c4b4b96c9e7 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.6.8.md @@ -0,0 +1,12 @@ +# Release 3.6.8 (2025-06-25) + +* Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). + +## What's Changed +* Fix fetchToStore() caching with --impure, improve testing by @edolstra in [DeterminateSystems/nix-src#117](https://github.com/DeterminateSystems/nix-src/pull/117) +* Add lazy-locks setting by @edolstra in [DeterminateSystems/nix-src#113](https://github.com/DeterminateSystems/nix-src/pull/113) +* Sync 2.29.1 by @edolstra in [DeterminateSystems/nix-src#124](https://github.com/DeterminateSystems/nix-src/pull/124) +* Release v3.6.7 by @github-actions in [DeterminateSystems/nix-src#126](https://github.com/DeterminateSystems/nix-src/pull/126) + + +**Full Changelog**: [v3.6.6...v3.6.8](https://github.com/DeterminateSystems/nix-src/compare/v3.6.6...v3.6.8) diff --git a/doc/manual/source/release-notes-determinate/rl-3.7.0.md b/doc/manual/source/release-notes-determinate/rl-3.7.0.md new file mode 100644 index 00000000000..8e5fc9ca6a1 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.7.0.md @@ -0,0 +1,63 @@ +# Release 3.7.0 (2025-07-03) + +- Based on [upstream Nix 2.29.1](../release-notes/rl-2.29.md). + +## What's Changed + +### Prefetch flake inputs in parallel + +By @edolstra in [DeterminateSystems/nix-src#127](https://github.com/DeterminateSystems/nix-src/pull/127) + +This release brings the command `nix flake prefetch-inputs`. + +Flake inputs are typically fetched "just in time." +That means Nix fetches a flake input when the evaluator needs it, and not before. +When the evaluator needs an input, evaluation is paused until the source is available. + +This causes a significant slow-down on projects with lots of flake inputs. + +The new command `nix flake prefetch-inputs` fetches all flake inputs in parallel. +We expect running this new command before building will dramatically improve evaluation performance for most projects, especially in CI. +Note that projects which with many unused flake inputs may not benefit from this change, since the new command fetches every input whether they're used or not. + +### Deep flake input overrides now work as expected + +By @edolstra in [DeterminateSystems/nix-src#108](https://github.com/DeterminateSystems/nix-src/pull/108) + +An override like: + +``` +inputs.foo.inputs.bar.inputs.nixpkgs.follows = "nixpkgs"; +``` + +implicitly set `inputs.foo.inputs.bar` to `flake:bar`, which led to an unexpected error like: + +``` +error: cannot find flake 'flake:bar' in the flake registries +``` + +We now no longer create a parent override (like for `foo.bar` in the example above) if it doesn't set an explicit ref or follows attribute. +We only recursively apply its child overrides. + +### `nix store delete` now shows you why deletion was not possible + +By @edolstra in [DeterminateSystems/nix-src#130](https://github.com/DeterminateSystems/nix-src/pull/130) + +For example: + +``` +error: Cannot delete path '/nix/store/6fcrjgfjip2ww3sx51rrmmghfsf60jvi-patchelf-0.14.3' + because it's referenced by the GC root '/home/eelco/Dev/nix-master/build/result'. + +error: Cannot delete path '/nix/store/rn0qyn3kmky26xgpr2n10vr787g57lff-cowsay-3.8.4' + because it's referenced by the GC root '/proc/3600568/environ'. + +error: Cannot delete path '/nix/store/klyng5rpdkwi5kbxkncy4gjwb490dlhb-foo.drv' + because it's in use by '{nix-process:3605324}'. +``` + +### Lazy-tree improvements + +- Improved lazy-tree evaluation caching for flakes accessed with a `path` flakeref by @edolstra in [DeterminateSystems/nix-src#131](https://github.com/DeterminateSystems/nix-src/pull/131) + +**Full Changelog**: [v3.6.8...v3.7.0](https://github.com/DeterminateSystems/nix-src/compare/v3.6.8...v3.7.0) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.0.md b/doc/manual/source/release-notes-determinate/rl-3.8.0.md new file mode 100644 index 00000000000..4103d6df94e --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.0.md @@ -0,0 +1,29 @@ +# Release 3.8.0 (2025-07-10) + +* Based on [upstream Nix 2.30.0](../release-notes/rl-2.30.md). + +## What's Changed + +### Faster CI with `nix flake check` + +`nix flake check` no longer downloads flake outputs if no building is necessary. + +This command is intended to validate that a flake can fully evaluate and all outputs can build. +If the outputs are available in a binary cache then both properties are confirmed to be true. +Notably, downloading the output from the binary cache is not strictly necessary for the validation. + +Previously, `nix flake check` would download a flake output if the full build is available in a binary cache. + +Some users will find this change significantly reduces costly bandwidth and CI workflow time. + +PR: [DeterminateSystems/nix-src#134](https://github.com/DeterminateSystems/nix-src/pull/134) + +### Improved flake locking of transitive dependencies + +Determinate Nix now re-locks all transitive dependencies when changing a flake input's source URL. + +This fixes an issue where in some scenarios Nix would not re-lock those inputs and incorrectly use the old inputs' dependencies. + +PR: [DeterminateSystems/nix-src#137](https://github.com/DeterminateSystems/nix-src/pull/137) + +**Full Changelog**: [v3.7.0...v3.8.0](https://github.com/DeterminateSystems/nix-src/compare/v3.7.0...v3.8.0) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.1.md b/doc/manual/source/release-notes-determinate/rl-3.8.1.md new file mode 100644 index 00000000000..90dc328f6ec --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.1.md @@ -0,0 +1,9 @@ +# Release 3.8.1 (2025-07-11) + +* Based on [upstream Nix 2.30.0](../release-notes/rl-2.30.md). + +## What's Changed +* Address ifdef problem with macOS/BSD sandboxing by @gustavderdrache in [DeterminateSystems/nix-src#142](https://github.com/DeterminateSystems/nix-src/pull/142) + + +**Full Changelog**: [v3.8.0...v3.8.1](https://github.com/DeterminateSystems/nix-src/compare/v3.8.0...v3.8.1) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.2.md b/doc/manual/source/release-notes-determinate/rl-3.8.2.md new file mode 100644 index 00000000000..638d90f6841 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.2.md @@ -0,0 +1,10 @@ +# Release 3.8.2 (2025-07-12) + +* Based on [upstream Nix 2.30.0](../release-notes/rl-2.30.md). + +## What's Changed +* ci: don't run the full test suite for x86_64-darwin by @grahamc in [DeterminateSystems/nix-src#144](https://github.com/DeterminateSystems/nix-src/pull/144) +* Try publishing the manual again by @grahamc in [DeterminateSystems/nix-src#145](https://github.com/DeterminateSystems/nix-src/pull/145) + + +**Full Changelog**: [v3.8.1...v3.8.2](https://github.com/DeterminateSystems/nix-src/compare/v3.8.1...v3.8.2) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.3.md b/doc/manual/source/release-notes-determinate/rl-3.8.3.md new file mode 100644 index 00000000000..d3eb02bc7ea --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.3.md @@ -0,0 +1,26 @@ +# Release 3.8.3 (2025-07-18) + +* Based on [upstream Nix 2.30.1](../release-notes/rl-2.30.md). + +## What's Changed + +### Non-blocking evaluation caching + +Users reported evaluation would occasionally block other evaluation processes. + +The evaluation cache database is now opened in write-ahead mode to prevent delaying evaluations. + +PR: [DeterminateSystems/nix-src#150](https://github.com/DeterminateSystems/nix-src/pull/150) + +### New experimental feature: `external-builders` + +This experimental feature allows Nix to call an external program for the build environment. + +The interface and behavior of this feature may change at any moment without a correspondingly major semver version change. + +PRs: +- [DeterminateSystems/nix-src#141](https://github.com/DeterminateSystems/nix-src/pull/141) +- [DeterminateSystems/nix-src#152](https://github.com/DeterminateSystems/nix-src/pull/152) +- [DeterminateSystems/nix-src#78](https://github.com/DeterminateSystems/nix-src/pull/78) + +**Full Changelog**: [v3.8.2...v3.8.3](https://github.com/DeterminateSystems/nix-src/compare/v3.8.2...v3.8.3) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.4.md b/doc/manual/source/release-notes-determinate/rl-3.8.4.md new file mode 100644 index 00000000000..7c73e75ca02 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.4.md @@ -0,0 +1,9 @@ +# Release 3.8.4 (2025-07-21) + +* Based on [upstream Nix 2.30.1](../release-notes/rl-2.30.md). + +## What's Changed +* Revert "Use WAL mode for SQLite cache databases" by @grahamc in [DeterminateSystems/nix-src#155](https://github.com/DeterminateSystems/nix-src/pull/155) + + +**Full Changelog**: [v3.8.3...v3.8.4](https://github.com/DeterminateSystems/nix-src/compare/v3.8.3...v3.8.4) diff --git a/doc/manual/source/release-notes-determinate/rl-3.8.5.md b/doc/manual/source/release-notes-determinate/rl-3.8.5.md new file mode 100644 index 00000000000..0f1bbe6f99d --- /dev/null +++ b/doc/manual/source/release-notes-determinate/rl-3.8.5.md @@ -0,0 +1,58 @@ +## What's Changed + +### Less time "unpacking into the Git cache" + +Unpacking sources into the user's cache is now takes 1/2 to 1/4 of the time it used to. +Previously, Nix serially unpacked sources into the cache. +This change takes better advantage of our users' hardware by parallelizing the import. +Real life testing shows an initial Nixpkgs import takes 3.6s on Linux, when it used to take 11.7s. + +PR: [DeterminateSystems/nix-src#149](https://github.com/DeterminateSystems/nix-src/pull/149) + +### Copy paths to the daemon in parallel + +Determinate Nix's evaluator no longer blocks evaluation when copying paths to the store. +Previously, Nix would pause evaluation when it needed to add files to the store. +Now, the copying is performed in the background allowing evaluation to proceed. + +PR: [DeterminateSystems/nix-src#162](https://github.com/DeterminateSystems/nix-src/pull/162) + +### Faster Nix evaluation by reducing duplicate Nix daemon queries + +Determinate Nix more effectively caches store path validity data within a single evaluation. +Previously, the Nix client would perform many thousands of exra Nix daemon requests. +Each extra request takes real time, and this change reduced a sample evaluation by over 12,000 requests. + +PR: [DeterminateSystems/nix-src#157](https://github.com/DeterminateSystems/nix-src/pull/157) + +### More responsive tab completion + +Tab completion now implies the "--offline" flag, which disables most network requests. +Previously, tab completing Nix arguments would attempt to fetch sources and access binary caches. +Operating in offline mode improves the interactive experience of Nix when tab completing. + +PR: [DeterminateSystems/nix-src#161](https://github.com/DeterminateSystems/nix-src/pull/161) + +### ZFS users: we fixed the mysterious stall. + +Opening the Nix database is usually instantaneous but sometimes has a several second latency. +Determinate Nix works around this issue, eliminating the frustrating random stall when running Nix commands. + +PR: [DeterminateSystems/nix-src#158](https://github.com/DeterminateSystems/nix-src/pull/158) + +### Other changes + +* Determinate Nix is now fully formatted by clang-format, making it easier than ever to contribute to the project. + +PR: [DeterminateSystems/nix-src#159](https://github.com/DeterminateSystems/nix-src/pull/159) + +* Determinate Nix is now based on upstream Nix 2.30.2. + +PR: [DeterminateSystems/nix-src#160](https://github.com/DeterminateSystems/nix-src/pull/160) + +* Determinate Nix now uses `main` as our development branch, moving away from `detsys-main`. + +PRs: +* [DeterminateSystems/nix-src#164](https://github.com/DeterminateSystems/nix-src/pull/164) +* [DeterminateSystems/nix-src#166](https://github.com/DeterminateSystems/nix-src/pull/166) + diff --git a/doc/manual/source/release-notes-determinate/v3.10.0.md b/doc/manual/source/release-notes-determinate/v3.10.0.md new file mode 100644 index 00000000000..c644dd78744 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.10.0.md @@ -0,0 +1,10 @@ +# Release 3.10.0 (2025-09-02) + +* Based on [upstream Nix 2.31.0](../release-notes/rl-2.31.md). + +## What's Changed + +This release rebases Determinate Nix on upstream Nix 2.31.0. + + +**Full Changelog**: [v3.9.1...v3.10.0](https://github.com/DeterminateSystems/nix-src/compare/v3.9.1...v3.10.0) diff --git a/doc/manual/source/release-notes-determinate/v3.10.1.md b/doc/manual/source/release-notes-determinate/v3.10.1.md new file mode 100644 index 00000000000..08cbe4fd058 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.10.1.md @@ -0,0 +1,9 @@ +# Release 3.10.1 (2025-09-02) + +* Based on [upstream Nix 2.31.1](../release-notes/rl-2.31.md). + +## What's Changed +This release rebases Determinate Nix on upstream Nix 2.31.1. + + +**Full Changelog**: [v3.10.0...v3.10.1](https://github.com/DeterminateSystems/nix-src/compare/v3.10.0...v3.10.1) diff --git a/doc/manual/source/release-notes-determinate/v3.11.0.md b/doc/manual/source/release-notes-determinate/v3.11.0.md new file mode 100644 index 00000000000..7abb665a5a9 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.11.0.md @@ -0,0 +1,36 @@ +# Release 3.11.0 (2025-09-03) + +- Based on [upstream Nix 2.31.1](../release-notes/rl-2.31.md). + +## What's Changed + +### Parallel evaluation + +The following commands are now able to evaluate Nix expressions in parallel: + +- `nix search` +- `nix flake check` +- `nix flake show` +- `nix eval --json` + +This is currently in developer preview, and we'll be turning it on for more users in the coming weeks. +If you would like to try it right away, specify `eval-cores` in your `/etc/nix/nix.custom.conf`: + +```ini +eval-cores = 0 # Evaluate across all cores +``` + +Further, we introduced a new builtin: `builtins.parallel`. +This new builtin allows users to explicitly parallelize evaluation within a Nix expression. + +Using this new builtin requires turning on an additional experimental feature: + +```ini +extra-experimental-features = parallel-eval +``` + +Please note that this new builtin is subject to change semantics or even go away during the developer preview. + +PR: [DeterminateSystems/nix-src#125](https://github.com/DeterminateSystems/nix-src/pull/125) + +**Full Changelog**: [v3.10.1...v3.11.0](https://github.com/DeterminateSystems/nix-src/compare/v3.10.1...v3.11.0) diff --git a/doc/manual/source/release-notes-determinate/v3.11.1.md b/doc/manual/source/release-notes-determinate/v3.11.1.md new file mode 100644 index 00000000000..30597164333 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.11.1.md @@ -0,0 +1,9 @@ +# Release 3.11.1 (2025-09-04) + +* Based on [upstream Nix 2.31.1](../release-notes/rl-2.31.md). + +## What's Changed +* Fix race condition in Value::isTrivial() by @edolstra in [DeterminateSystems/nix-src#192](https://github.com/DeterminateSystems/nix-src/pull/192) + + +**Full Changelog**: [v3.11.0...v3.11.1](https://github.com/DeterminateSystems/nix-src/compare/v3.11.0...v3.11.1) diff --git a/doc/manual/source/release-notes-determinate/v3.11.2.md b/doc/manual/source/release-notes-determinate/v3.11.2.md new file mode 100644 index 00000000000..ac4fe569dff --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.11.2.md @@ -0,0 +1,24 @@ +# Release 3.11.2 (2025-09-12) + +* Based on [upstream Nix 2.31.1](../release-notes/rl-2.31.md). + +## What's Changed + +### Fix some interactions with the registry and flakes that include a `?dir=` parameter + +Some users were experiencing issues when their flake registry contained a flake that included a `?dir=` parameter, causing commands like `nix eval registry-with-flake-in-subdir#output` and those that used --inputs-from` to fail or behave incorrectly. + +This is now fixed, so use your flakes inside subdirs without fear! + +PRs: [DeterminateSystems/nix-src#196](https://github.com/DeterminateSystems/nix-src/pull/196), [DeterminateSystems/nix-src#199](https://github.com/DeterminateSystems/nix-src/pull/199) + +### Only substitute inputs if they haven't already been fetched + +When using `lazy-trees`, you might have noticed Nix fetching some source inputs from a cache, even though you could have sworn it already fetched those inputs! + +This fixes that behavior such that Nix will try to fetch inputs from their original location, and only if that fails fall back to fetching from a substituter. + +PR: [DeterminateSystems/nix-src#202](https://github.com/DeterminateSystems/nix-src/pull/202) + + +**Full Changelog**: [v3.11.1...v3.11.2](https://github.com/DeterminateSystems/nix-src/compare/v3.11.1...v3.11.2) diff --git a/doc/manual/source/release-notes-determinate/v3.11.3.md b/doc/manual/source/release-notes-determinate/v3.11.3.md new file mode 100644 index 00000000000..fab5ed51a4b --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.11.3.md @@ -0,0 +1,34 @@ +# Release 3.11.3 (2025-10-09) + +* Based on [upstream Nix 2.31.2](../release-notes/rl-2.31.md). + +## What's Changed + +### Fix some bugs and interactions with parallel eval + +We received some reports of parallel eval having issues, such as not being able to be interrupted, infinite recursion hanging forever, and segfaults when using the experimental `builtins.parallel`. + +Those have now been fixed. + +Additionally, the debugger now disables parallel eval, because the two features are incompatible. + +PRs: [DeterminateSystems/nix-src#206](https://github.com/DeterminateSystems/nix-src/pull/206), [DeterminateSystems/nix-src#213](https://github.com/DeterminateSystems/nix-src/pull/213), [DeterminateSystems/nix-src#218](https://github.com/DeterminateSystems/nix-src/pull/218), [DeterminateSystems/nix-src#205](https://github.com/DeterminateSystems/nix-src/pull/205) + +### `NIX_SSHOPTS` + `ssh-ng://root@localhost` fix + +We noticed that specifying `NIX_SSHOPTS=-p2222` when using a command that uses SSH (such as `nix copy --to ssh-ng://root@localhost`) stopped respecting the `NIX_SSHOPTS` setting because of an incorrect comparison. + +This has been fixed, so `NIX_SSHOPTS` and SSH stores that are accessed like `user@localhost` work again. + +PR: [DeterminateSystems/nix-src#219](https://github.com/DeterminateSystems/nix-src/pull/219) + +### Fix `error: [json.exception.type_error.302] type must be string, but is array` when using `exportReferencesGraph` + +We received a report of a `nix build` failing on a specific flake due to its expression using `exportReferencesGraph` with a heterogeneous array of dependencies, causing this inscrutable error. + +This specific case has been broken since Nix 2.29.0, and is now fixed. + +PRs: [DeterminateSystems/nix-src#221](https://github.com/DeterminateSystems/nix-src/pull/221), [DeterminateSystems/nix-src#225](https://github.com/DeterminateSystems/nix-src/pull/225) + + +**Full Changelog**: [v3.11.2...v3.11.3](https://github.com/DeterminateSystems/nix-src/compare/v3.11.2...v3.11.3) diff --git a/doc/manual/source/release-notes-determinate/v3.12.0.md b/doc/manual/source/release-notes-determinate/v3.12.0.md new file mode 100644 index 00000000000..55c1f10bf15 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.12.0.md @@ -0,0 +1,17 @@ +# Release 3.12.0 (2025-10-23) + +* Based on [upstream Nix 2.32.1](../release-notes/rl-2.32.md). + +## What's Changed + +### `nix nario` + +Determinate Nix has a new command, `nix nario`, that replaces the commands `nix-store --export` and `nix-store --import` from the old CLI. `nix nario` allows you to serialize store paths to a file that can be imported into another Nix store. It is backwards compatible with the file format generated by `nix-store --export`. It also provides a new format (selected by passing `--format 2`) that supports store path attributes such as signatures, and allows store paths to be imported more efficiently. + +### Other changes + +`nix flake clone` now supports arbitrary input types. In particular, this allows you to clone tarball flakes, such as flakes on FlakeHub. + +When using `-vv`, Determinate Nix now prints the Nix version. This is useful when diagnosing Nix problems from the debug output of a Nix run. + +**Full Changelog**: [v3.11.3...v3.12.0](https://github.com/DeterminateSystems/nix-src/compare/v3.11.3...v3.12.0) diff --git a/doc/manual/source/release-notes-determinate/v3.8.6.md b/doc/manual/source/release-notes-determinate/v3.8.6.md new file mode 100644 index 00000000000..8f917f2362f --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.8.6.md @@ -0,0 +1,14 @@ +# Release 3.8.6 (2025-08-19) + +* Based on [upstream Nix 2.30.2](../release-notes/rl-2.30.md). + +## What's Changed +* Auto update release notes by @grahamc in [DeterminateSystems/nix-src#170](https://github.com/DeterminateSystems/nix-src/pull/170) +* Use WAL mode for SQLite cache databases (2nd attempt) by @edolstra in [DeterminateSystems/nix-src#167](https://github.com/DeterminateSystems/nix-src/pull/167) +* Enable parallel marking in boehm-gc by @edolstra in [DeterminateSystems/nix-src#168](https://github.com/DeterminateSystems/nix-src/pull/168) +* BasicClientConnection::queryPathInfo(): Don't throw exception for invalid paths by @edolstra in [DeterminateSystems/nix-src#172](https://github.com/DeterminateSystems/nix-src/pull/172) +* Fix queryPathInfo() negative caching by @edolstra in [DeterminateSystems/nix-src#173](https://github.com/DeterminateSystems/nix-src/pull/173) +* forceDerivation(): Wait for async path write after forcing value by @edolstra in [DeterminateSystems/nix-src#176](https://github.com/DeterminateSystems/nix-src/pull/176) + + +**Full Changelog**: [v3.8.5...v3.8.6](https://github.com/DeterminateSystems/nix-src/compare/v3.8.5...v3.8.6) diff --git a/doc/manual/source/release-notes-determinate/v3.9.0.md b/doc/manual/source/release-notes-determinate/v3.9.0.md new file mode 100644 index 00000000000..66deb69b619 --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.9.0.md @@ -0,0 +1,45 @@ +# Release 3.9.0 (2025-08-26) + +* Based on [upstream Nix 2.30.2](../release-notes/rl-2.30.md). + +## What's Changed + +### Build-time flake inputs + +Some of our users have hundreds or thousands of flake inputs. +In those cases, it is painfully slow for Nix to fetch all the inputs during evaluation of the flake. + +Determinate Nix has an experimental feature for deferring the fetching to build time of the dependent derivations. + +This is currently in developer preview. +If you would like to try it, add the experimental feature to your `/etc/nix/nix.custom.conf`: + +```ini +extra-experimental-features = build-time-fetch-tree +``` + +Then, mark an input to be fetched at build time: + +```nix +inputs.example = { + type = "github"; + owner = "DeterminateSystems"; + repo = "example"; + flake = false; # <-- currently required + buildTime = true; +}; +``` + +Let us know what you think! + +PR: [DeterminateSystems/nix-src#49](https://github.com/DeterminateSystems/nix-src/pull/49) + +### Corrected inconsistent behavior of `nix flake check` + +Users reported that `nix flake check` would not consistently validate the entire flake. + +We've fixed this issue and improved our testing around `nix flake check`. + +PR: [DeterminateSystems/nix-src#182](https://github.com/DeterminateSystems/nix-src/pull/182) + +**Full Changelog**: [v3.8.6...v3.9.0](https://github.com/DeterminateSystems/nix-src/compare/v3.8.6...v3.9.0) diff --git a/doc/manual/source/release-notes-determinate/v3.9.1.md b/doc/manual/source/release-notes-determinate/v3.9.1.md new file mode 100644 index 00000000000..38d17199c2c --- /dev/null +++ b/doc/manual/source/release-notes-determinate/v3.9.1.md @@ -0,0 +1,20 @@ +# Release 3.9.1 (2025-08-28) + +- Based on [upstream Nix 2.30.2](../release-notes/rl-2.30.md). + +### A useful `nix flake init` template default + +Nix's default flake template is [extremely bare bones](https://github.com/NixOS/templates/blob/ad0e221dda33c4b564fad976281130ce34a20cb9/trivial/flake.nix), and not a useful starting point. + +Deteminate Nix now uses [a more fleshed out default template](https://github.com/DeterminateSystems/flake-templates/blob/8af99b99627da41f16897f60eb226db30c775e76/default/flake.nix), including targeting multiple systems. + +PR: [DeterminateSystems/nix-src#180](https://github.com/DeterminateSystems/nix-src/pull/180) + +### Build cancellation is repaired on macOS + +A recent macOS update changed how signals are handled by Nix and broke using Ctrl-C to stop a build. +Determinate Nix on macOS correctly handles these signals and stops the build. + +PR: [DeterminateSystems/nix-src#184](https://github.com/DeterminateSystems/nix-src/pull/184) + +**Full Changelog**: [v3.9.0...v3.9.1](https://github.com/DeterminateSystems/nix-src/compare/v3.9.0...v3.9.1) diff --git a/doc/manual/source/release-notes/rl-2.19.md b/doc/manual/source/release-notes/rl-2.19.md index 06c704324dd..47a0dd3db99 100644 --- a/doc/manual/source/release-notes/rl-2.19.md +++ b/doc/manual/source/release-notes/rl-2.19.md @@ -69,7 +69,7 @@ This makes it match `nix derivation show`, which also maps store paths to information. -- When Nix is installed using the [binary installer](@docroot@/installation/installing-binary.md), in supported shells (Bash, Zsh, Fish) +- When Nix is installed using the binary installer, in supported shells (Bash, Zsh, Fish) [`XDG_DATA_DIRS`](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html#variables) is now populated with the path to the `/share` subdirectory of the current profile. This means that command completion scripts, `.desktop` files, and similar artifacts installed via [`nix-env`](@docroot@/command-ref/nix-env.md) or [`nix profile`](@docroot@/command-ref/new-cli/nix3-profile.md) (experimental) can be found by any program that follows the [XDG Base Directory Specification](https://specifications.freedesktop.org/basedir-spec/basedir-spec-latest.html). diff --git a/doc/manual/source/release-notes/rl-2.24.md b/doc/manual/source/release-notes/rl-2.24.md index d4af3cb5174..33fc0db03f9 100644 --- a/doc/manual/source/release-notes/rl-2.24.md +++ b/doc/manual/source/release-notes/rl-2.24.md @@ -268,6 +268,21 @@ be configured using the `warn-large-path-threshold` setting, e.g. `--warn-large-path-threshold 100M`. +- Wrap filesystem exceptions more correctly [#11378](https://github.com/NixOS/nix/pull/11378) + + With the switch to `std::filesystem` in different places, Nix started to throw `std::filesystem::filesystem_error` in many places instead of its own exceptions. + + This led to no longer generating error traces, for example when listing a non-existing directory. + + This version catches these types of exception correctly and wraps them into Nix's own exeception type. + + Author: [**@Mic92**](https://github.com/Mic92) + +- `` uses TLS verification [#11585](https://github.com/NixOS/nix/pull/11585) + + Previously `` did not do TLS verification. This was because the Nix sandbox in the past did not have access to TLS certificates, and Nix checks the hash of the fetched file anyway. However, this can expose authentication data from `netrc` and URLs to man-in-the-middle attackers. In addition, Nix now in some cases (such as when using impure derivations) does *not* check the hash. Therefore we have now enabled TLS verification. This means that downloads by `` will now fail if you're fetching from a HTTPS server that does not have a valid certificate. + + `` is also known as the builtin derivation builder `builtin:fetchurl`. It's not to be confused with the evaluation-time function `builtins.fetchurl`, which was not affected by this issue. ## Contributors diff --git a/doc/manual/source/release-notes/rl-2.32.md b/doc/manual/source/release-notes/rl-2.32.md new file mode 100644 index 00000000000..3a925198dd4 --- /dev/null +++ b/doc/manual/source/release-notes/rl-2.32.md @@ -0,0 +1,130 @@ +# Release 2.32.0 (2025-10-06) + +## Incompatible changes + +- Removed support for daemons and clients older than Nix 2.0 [#13951](https://github.com/NixOS/nix/pull/13951) + + We have dropped support in the daemon worker protocol for daemons and clients that don't speak at least version 18 of the protocol. This first Nix release that supports this version is Nix 2.0, released in February 2018. + +- Derivation JSON format now uses store path basenames only [#13570](https://github.com/NixOS/nix/issues/13570) [#13980](https://github.com/NixOS/nix/pull/13980) + + Experience with many JSON frameworks (e.g. nlohmann/json in C++, Serde in Rust, and Aeson in Haskell) has shown that the use of the store directory in JSON formats is an impediment to systematic JSON formats, because it requires the serializer/deserializer to take an extra paramater (the store directory). + + We ultimately want to rectify this issue with all JSON formats to the extent allowed by our stability promises. To start with, we are changing the JSON format for derivations because the `nix derivation` commands are — in addition to being formally unstable — less widely used than other unstable commands. + + See the documentation on the [JSON format for derivations](@docroot@/protocols/json/derivation.md) for further details. + +- C API: `nix_get_attr_name_byidx`, `nix_get_attr_byidx` take a `nix_value *` instead of `const nix_value *` [#13987](https://github.com/NixOS/nix/pull/13987) + + In order to accommodate a more optimized internal representation of attribute set merges these functions require + a mutable `nix_value *` that might be modified on access. This does *not* break the ABI of these functions. + +## New features + +- C API: Add lazy attribute and list item accessors [#14030](https://github.com/NixOS/nix/pull/14030) + + The C API now includes lazy accessor functions for retrieving values from lists and attribute sets without forcing evaluation: + + - `nix_get_list_byidx_lazy()` - Get a list element without forcing its evaluation + - `nix_get_attr_byname_lazy()` - Get an attribute value by name without forcing evaluation + - `nix_get_attr_byidx_lazy()` - Get an attribute by index without forcing evaluation + + These functions are useful when forwarding unevaluated sub-values to other lists, attribute sets, or function calls. They allow more efficient handling of Nix values by deferring evaluation until actually needed. + + Additionally, bounds checking has been improved for all `_byidx` functions to properly validate indices before access, preventing potential out-of-bounds errors. + + The documentation for `NIX_ERR_KEY` error handling has also been clarified to specify when this error code is returned. + +- HTTP binary caches now support transparent compression for metadata + + HTTP binary cache stores can now compress `.narinfo`, `.ls`, and build log files before uploading them, + reducing bandwidth usage and storage requirements. The compression is applied transparently using the + `Content-Encoding` header, allowing compatible clients to automatically decompress the files. + + Three new configuration options control this behavior: + - `narinfo-compression`: Compression method for `.narinfo` files + - `ls-compression`: Compression method for `.ls` files + - `log-compression`: Compression method for build logs in `log/` directory + + Example usage: + ``` + nix copy --to 'http://cache.example.com?narinfo-compression=gzip&ls-compression=gzip' /nix/store/... + nix store copy-log --to 'http://cache.example.com?log-compression=br' /nix/store/... + ``` + +- Temporary build directories no longer include derivation names [#13839](https://github.com/NixOS/nix/pull/13839) + + Temporary build directories created during derivation builds no longer include the derivation name in their path to avoid build failures when the derivation name is too long. This change ensures predictable prefix lengths for build directories under `/nix/var/nix/builds`. + +- External derivation builders [#14145](https://github.com/NixOS/nix/pull/14145) + + These are helper programs that Nix calls to perform derivations for specified system types, e.g. by using QEMU to emulate a different type of platform. For more information, see the [`external-builders` setting](../command-ref/conf-file.md#conf-external-builders). + + This is currently an experimental feature. + +## Performance improvements + +- Optimize memory usage of attribute set merges [#13987](https://github.com/NixOS/nix/pull/13987) + + [Attribute set update operations](@docroot@/language/operators.md#update) have been optimized to + reduce reallocations in cases when the second operand is small. + + For typical evaluations of nixpkgs this optimization leads to ~20% less memory allocated in total + without significantly affecting evaluation performance. + + See [eval-attrset-update-layer-rhs-threshold](@docroot@/command-ref/conf-file.md#conf-eval-attrset-update-layer-rhs-threshold) + +- Substituted flake inputs are no longer re-copied to the store [#14041](https://github.com/NixOS/nix/pull/14041) + + Since 2.25, Nix would fail to store a cache entry for substituted flake inputs, which in turn would cause them to be re-copied to the store on initial evaluation. Caching these inputs results in a near doubling of performance in some cases — especially on I/O-bound machines and when using commands that fetch many inputs, like `nix flake [archive|prefetch-inputs]`. + +- `nix flake check` now skips derivations that can be substituted [#13574](https://github.com/NixOS/nix/pull/13574) + + Previously, `nix flake check` would evaluate and build/substitute all + derivations. Now, it will skip downloading derivations that can be substituted. + This can drastically decrease the time invocations take in environments where + checks may already be cached (like in CI). + +- `fetchTarball` and `fetchurl` now correctly substitute (#14138) + + At some point we stopped substituting calls to `fetchTarball` and `fetchurl` with a set `narHash` to avoid incorrectly substituting things in `fetchTree`, even though it would be safe to substitute when calling the legacy `fetch{Tarball,url}`. This fixes that regression where it is safe. +- Started moving AST allocations into a bump allocator [#14088](https://github.com/NixOS/nix/issues/14088) + + This leaves smaller, immutable structures in the AST. So far this saves about 2% memory on a NixOS config evaluation. +## Contributors + +This release was made possible by the following 32 contributors: + +- Farid Zakaria [**(@fzakaria)**](https://github.com/fzakaria) +- dram [**(@dramforever)**](https://github.com/dramforever) +- Ephraim Siegfried [**(@EphraimSiegfried)**](https://github.com/EphraimSiegfried) +- Robert Hensing [**(@roberth)**](https://github.com/roberth) +- Taeer Bar-Yam [**(@Radvendii)**](https://github.com/Radvendii) +- Emily [**(@emilazy)**](https://github.com/emilazy) +- Jens Petersen [**(@juhp)**](https://github.com/juhp) +- Bernardo Meurer [**(@lovesegfault)**](https://github.com/lovesegfault) +- Jörg Thalheim [**(@Mic92)**](https://github.com/Mic92) +- Leandro Emmanuel Reina Kiperman [**(@kip93)**](https://github.com/kip93) +- Marie [**(@NyCodeGHG)**](https://github.com/NyCodeGHG) +- Ethan Evans [**(@ethanavatar)**](https://github.com/ethanavatar) +- Yaroslav Bolyukin [**(@CertainLach)**](https://github.com/CertainLach) +- Matej Urbas [**(@urbas)**](https://github.com/urbas) +- Jami Kettunen [**(@JamiKettunen)**](https://github.com/JamiKettunen) +- Clayton [**(@netadr)**](https://github.com/netadr) +- Grégory Marti [**(@gmarti)**](https://github.com/gmarti) +- Eelco Dolstra [**(@edolstra)**](https://github.com/edolstra) +- rszyma [**(@rszyma)**](https://github.com/rszyma) +- Philip Wilk [**(@philipwilk)**](https://github.com/philipwilk) +- John Ericson [**(@Ericson2314)**](https://github.com/Ericson2314) +- Tom Westerhout [**(@twesterhout)**](https://github.com/twesterhout) +- Tristan Ross [**(@RossComputerGuy)**](https://github.com/RossComputerGuy) +- Sergei Zimmerman [**(@xokdvium)**](https://github.com/xokdvium) +- Jean-François Roche [**(@jfroche)**](https://github.com/jfroche) +- Seth Flynn [**(@getchoo)**](https://github.com/getchoo) +- éclairevoyant [**(@eclairevoyant)**](https://github.com/eclairevoyant) +- Glen Huang [**(@hgl)**](https://github.com/hgl) +- osman - オスマン [**(@osbm)**](https://github.com/osbm) +- David McFarland [**(@corngood)**](https://github.com/corngood) +- Cole Helbling [**(@cole-h)**](https://github.com/cole-h) +- Sinan Mohd [**(@sinanmohd)**](https://github.com/sinanmohd) +- Philipp Otterbein diff --git a/doc/manual/source/store/store-object.md b/doc/manual/source/store/store-object.md index 10c2384fa53..71ec772fb52 100644 --- a/doc/manual/source/store/store-object.md +++ b/doc/manual/source/store/store-object.md @@ -20,7 +20,8 @@ The graph of references excluding self-references thus forms a [directed acyclic [directed acyclic graph]: @docroot@/glossary.md#gloss-directed-acyclic-graph -We can take the [transitive closure] of the references graph, which any pair of store objects have an edge not if there is a single reference from the first to the second, but a path of one or more references from the first to the second. +We can take the [transitive closure] of the references graph, in which any pair of store objects have an edge if a *path* of one or more references exists from the first to the second object. +(A single reference always forms a path which is one reference long, but longer paths may connect objects which have no direct reference between them.) The *requisites* of a store object are all store objects reachable by paths of references which start with given store object's references. [transitive closure]: https://en.wikipedia.org/wiki/Transitive_closure diff --git a/docker.nix b/docker.nix index 619e75c5440..c209c507e84 100644 --- a/docker.nix +++ b/docker.nix @@ -281,7 +281,10 @@ let # may get replaced by pkgs.dockerTools.caCertificates mkdir -p $out/etc/ssl/certs + # Old NixOS compatibility. ln -s /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt $out/etc/ssl/certs + # NixOS canonical location + ln -s /nix/var/nix/profiles/default/etc/ssl/certs/ca-bundle.crt $out/etc/ssl/certs/ca-certificates.crt cat $passwdContentsPath > $out/etc/passwd echo "" >> $out/etc/passwd @@ -333,7 +336,7 @@ let globalFlakeRegistryPath="$nixCacheDir/flake-registry.json" ln -s ${flake-registry-path} $out$globalFlakeRegistryPath mkdir -p $out/nix/var/nix/gcroots/auto - rootName=$(${lib.getExe' nix "nix"} --extra-experimental-features nix-command hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath)) + rootName=$(${lib.getExe' nix "nix"} hash file --type sha1 --base32 <(echo -n $globalFlakeRegistryPath)) ln -s $globalFlakeRegistryPath $out/nix/var/nix/gcroots/auto/$rootName '') ); diff --git a/flake.lock b/flake.lock index cc2b2f27e72..919115594d7 100644 --- a/flake.lock +++ b/flake.lock @@ -3,11 +3,11 @@ "flake-compat": { "flake": false, "locked": { - "lastModified": 1733328505, - "narHash": "sha256-NeCCThCEP3eCl2l/+27kNNK7QrwZB1IJCrXfrbv5oqU=", + "lastModified": 1696426674, + "narHash": "sha256-kvjfFW7WAETZlt09AgDn1MrtKzP7t90Vf7vypd3OL1U=", "owner": "edolstra", "repo": "flake-compat", - "rev": "ff81ac966bb2cae68946d5ed5fc4994f96d0ffec", + "rev": "0f9255e01c2351cc7d116c072cb317785dd33b33", "type": "github" }, "original": { @@ -23,58 +23,51 @@ ] }, "locked": { - "lastModified": 1733312601, - "narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=", - "owner": "hercules-ci", - "repo": "flake-parts", - "rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9", - "type": "github" + "lastModified": 1748821116, + "narHash": "sha256-F82+gS044J1APL0n4hH50GYdPRv/5JWm34oCJYmVKdE=", + "rev": "49f0870db23e8c1ca0b5259734a02cd9e1e371a1", + "revCount": 377, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/hercules-ci/flake-parts/0.1.377%2Brev-49f0870db23e8c1ca0b5259734a02cd9e1e371a1/01972f28-554a-73f8-91f4-d488cc502f08/source.tar.gz" }, "original": { - "owner": "hercules-ci", - "repo": "flake-parts", - "type": "github" + "type": "tarball", + "url": "https://flakehub.com/f/hercules-ci/flake-parts/0.1" } }, "git-hooks-nix": { "inputs": { - "flake-compat": [], + "flake-compat": "flake-compat", "gitignore": [], "nixpkgs": [ "nixpkgs" - ], - "nixpkgs-stable": [ - "nixpkgs" ] }, "locked": { - "lastModified": 1734279981, - "narHash": "sha256-NdaCraHPp8iYMWzdXAt5Nv6sA3MUzlCiGiR586TCwo0=", - "owner": "cachix", - "repo": "git-hooks.nix", - "rev": "aa9f40c906904ebd83da78e7f328cd8aeaeae785", - "type": "github" + "lastModified": 1747372754, + "narHash": "sha256-2Y53NGIX2vxfie1rOW0Qb86vjRZ7ngizoo+bnXU9D9k=", + "rev": "80479b6ec16fefd9c1db3ea13aeb038c60530f46", + "revCount": 1026, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/cachix/git-hooks.nix/0.1.1026%2Brev-80479b6ec16fefd9c1db3ea13aeb038c60530f46/0196d79a-1b35-7b8e-a021-c894fb62163d/source.tar.gz" }, "original": { - "owner": "cachix", - "repo": "git-hooks.nix", - "type": "github" + "type": "tarball", + "url": "https://flakehub.com/f/cachix/git-hooks.nix/0.1.941" } }, "nixpkgs": { "locked": { - "lastModified": 1756178832, - "narHash": "sha256-O2CIn7HjZwEGqBrwu9EU76zlmA5dbmna7jL1XUmAId8=", - "owner": "NixOS", - "repo": "nixpkgs", - "rev": "d98ce345cdab58477ca61855540999c86577d19d", - "type": "github" + "lastModified": 1755922037, + "narHash": "sha256-wY1+2JPH0ZZC4BQefoZw/k+3+DowFyfOxv17CN/idKs=", + "rev": "b1b3291469652d5a2edb0becc4ef0246fff97a7c", + "revCount": 808723, + "type": "tarball", + "url": "https://api.flakehub.com/f/pinned/NixOS/nixpkgs/0.2505.808723%2Brev-b1b3291469652d5a2edb0becc4ef0246fff97a7c/0198daf7-011a-7703-95d7-57146e794342/source.tar.gz" }, "original": { - "owner": "NixOS", - "ref": "nixos-25.05-small", - "repo": "nixpkgs", - "type": "github" + "type": "tarball", + "url": "https://flakehub.com/f/NixOS/nixpkgs/0.2505" } }, "nixpkgs-23-11": { @@ -111,7 +104,6 @@ }, "root": { "inputs": { - "flake-compat": "flake-compat", "flake-parts": "flake-parts", "git-hooks-nix": "git-hooks-nix", "nixpkgs": "nixpkgs", diff --git a/flake.nix b/flake.nix index a2bdeb0e594..967f8d8c397 100644 --- a/flake.nix +++ b/flake.nix @@ -1,24 +1,18 @@ { description = "The purely functional package manager"; - inputs.nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05-small"; + inputs.nixpkgs.url = "https://flakehub.com/f/NixOS/nixpkgs/0.2505"; inputs.nixpkgs-regression.url = "github:NixOS/nixpkgs/215d4d0fd80ca5163643b03a33fde804a29cc1e2"; inputs.nixpkgs-23-11.url = "github:NixOS/nixpkgs/a62e6edd6d5e1fa0329b8653c801147986f8d446"; - inputs.flake-compat = { - url = "github:edolstra/flake-compat"; - flake = false; - }; # dev tooling - inputs.flake-parts.url = "github:hercules-ci/flake-parts"; - inputs.git-hooks-nix.url = "github:cachix/git-hooks.nix"; + inputs.flake-parts.url = "https://flakehub.com/f/hercules-ci/flake-parts/0.1"; + inputs.git-hooks-nix.url = "https://flakehub.com/f/cachix/git-hooks.nix/0.1.941"; # work around https://github.com/NixOS/nix/issues/7730 inputs.flake-parts.inputs.nixpkgs-lib.follows = "nixpkgs"; inputs.git-hooks-nix.inputs.nixpkgs.follows = "nixpkgs"; - inputs.git-hooks-nix.inputs.nixpkgs-stable.follows = "nixpkgs"; # work around 7730 and https://github.com/NixOS/nix/issues/7807 - inputs.git-hooks-nix.inputs.flake-compat.follows = ""; inputs.git-hooks-nix.inputs.gitignore.follows = ""; outputs = @@ -34,7 +28,7 @@ officialRelease = true; - linux32BitSystems = [ "i686-linux" ]; + linux32BitSystems = [ ]; linux64BitSystems = [ "x86_64-linux" "aarch64-linux" @@ -47,13 +41,12 @@ systems = linuxSystems ++ darwinSystems; crossSystems = [ - "armv6l-unknown-linux-gnueabihf" - "armv7l-unknown-linux-gnueabihf" - "riscv64-unknown-linux-gnu" + #"armv6l-unknown-linux-gnueabihf" + #"armv7l-unknown-linux-gnueabihf" + #"riscv64-unknown-linux-gnu" # Disabled because of https://github.com/NixOS/nixpkgs/issues/344423 # "x86_64-unknown-netbsd" - "x86_64-unknown-freebsd" - "x86_64-w64-mingw32" + #"x86_64-unknown-freebsd" ]; stdenvs = [ @@ -371,6 +364,40 @@ nix-manual = nixpkgsFor.${system}.native.nixComponents2.nix-manual; nix-internal-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-internal-api-docs; nix-external-api-docs = nixpkgsFor.${system}.native.nixComponents2.nix-external-api-docs; + + fallbackPathsNix = + let + pkgs = nixpkgsFor.${system}.native; + + closures = forAllSystems (system: self.packages.${system}.default.outPath); + + closures_json = + pkgs.runCommand "versions.json" + { + buildInputs = [ pkgs.jq ]; + passAsFile = [ "json" ]; + json = builtins.toJSON closures; + } + '' + cat "$jsonPath" | jq . > $out + ''; + + closures_nix = + pkgs.runCommand "versions.nix" + { + buildInputs = [ pkgs.jq ]; + passAsFile = [ "template" ]; + jsonPath = closures_json; + template = '' + builtins.fromJSON('''@closures@''') + ''; + } + '' + export closures=$(cat "$jsonPath"); + substituteAll "$templatePath" "$out" + ''; + in + closures_nix; } # We need to flatten recursive attribute sets of derivations to pass `flake check`. // @@ -425,8 +452,6 @@ { # These attributes go right into `packages.`. "${pkgName}" = nixpkgsFor.${system}.native.nixComponents2.${pkgName}; - "${pkgName}-static" = nixpkgsFor.${system}.native.pkgsStatic.nixComponents2.${pkgName}; - "${pkgName}-llvm" = nixpkgsFor.${system}.native.pkgsLLVM.nixComponents2.${pkgName}; } // lib.optionalAttrs supportsCross ( flatMapAttrs (lib.genAttrs crossSystems (_: { })) ( @@ -482,32 +507,6 @@ } ) ) - // lib.optionalAttrs (!nixpkgsFor.${system}.native.stdenv.isDarwin) ( - prefixAttrs "static" ( - forAllStdenvs ( - stdenvName: - makeShell { - pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.pkgsStatic; - } - ) - ) - // prefixAttrs "llvm" ( - forAllStdenvs ( - stdenvName: - makeShell { - pkgs = nixpkgsFor.${system}.nativeForStdenv.${stdenvName}.pkgsLLVM; - } - ) - ) - // prefixAttrs "cross" ( - forAllCrossSystems ( - crossSystem: - makeShell { - pkgs = nixpkgsFor.${system}.cross.${crossSystem}; - } - ) - ) - ) // { native = self.devShells.${system}.native-stdenv; default = self.devShells.${system}.native; diff --git a/maintainers/README.md b/maintainers/README.md index 6553cd04815..722a920b72c 100644 --- a/maintainers/README.md +++ b/maintainers/README.md @@ -46,7 +46,7 @@ The team meets twice a week (times are denoted in the [Europe/Amsterdam](https:/ - mark it as draft if it is blocked on the contributor - escalate it back to the team by moving it to To discuss, and leaving a comment as to why the issue needs to be discussed again. -- Work meeting: Mondays 14:00-16:00 Europe/Amsterdam see [calendar](https://calendar.google.com/calendar/u/0/embed?src=b9o52fobqjak8oq8lfkhg3t0qg@group.calendar.google.com). +- Work meeting: Mondays 18:00-20:00 Europe/Amsterdam; see [calendar](https://calendar.google.com/calendar/u/0/embed?src=b9o52fobqjak8oq8lfkhg3t0qg@group.calendar.google.com). 1. Code review on pull requests from [In review](#in-review). 2. Other chores and tasks. diff --git a/maintainers/data/release-credits-email-to-handle.json b/maintainers/data/release-credits-email-to-handle.json index ea37afb9052..0dbbf8fa60e 100644 --- a/maintainers/data/release-credits-email-to-handle.json +++ b/maintainers/data/release-credits-email-to-handle.json @@ -203,5 +203,26 @@ "ConnorBaker01@Gmail.com": "ConnorBaker", "jsoo1@asu.edu": "jsoo1", "hsngrmpf+github@gmail.com": "DavHau", - "matthew@floxdev.com": "mkenigs" + "matthew@floxdev.com": "mkenigs", + "taeer@bar-yam.me": "Radvendii", + "beme@anthropic.com": "lovesegfault", + "osbm@osbm.dev": "osbm", + "jami.kettunen@protonmail.com": "JamiKettunen", + "ephraim.siegfried@hotmail.com": "EphraimSiegfried", + "rszyma.dev@gmail.com": "rszyma", + "tristan.ross@determinate.systems": "RossComputerGuy", + "corngood@gmail.com": "corngood", + "jfroche@pyxel.be": "jfroche", + "848000+eclairevoyant@users.noreply.github.com": "eclairevoyant", + "petersen@redhat.com": "juhp", + "dramforever@live.com": "dramforever", + "me@glenhuang.com": "hgl", + "philip.wilk@fivium.co.uk": "philipwilk", + "me@nycode.dev": "NyCodeGHG", + "14264576+twesterhout@users.noreply.github.com": "twesterhout", + "sinan@sinanmohd.com": "sinanmohd", + "42688647+netadr@users.noreply.github.com": "netadr", + "matej.urbas@gmail.com": "urbas", + "ethanalexevans@gmail.com": "ethanavatar", + "greg.marti@gmail.com": "gmarti" } \ No newline at end of file diff --git a/maintainers/data/release-credits-handle-to-name.json b/maintainers/data/release-credits-handle-to-name.json index e2510548d90..8abffc65caa 100644 --- a/maintainers/data/release-credits-handle-to-name.json +++ b/maintainers/data/release-credits-handle-to-name.json @@ -177,5 +177,24 @@ "avnik": "Alexander V. Nikolaev", "DavHau": null, "aln730": "AGawas", - "vog": "Volker Diels-Grabsch" + "vog": "Volker Diels-Grabsch", + "corngood": "David McFarland", + "twesterhout": "Tom Westerhout", + "JamiKettunen": "Jami Kettunen", + "dramforever": "dram", + "philipwilk": "Philip Wilk", + "netadr": "Clayton", + "NyCodeGHG": "Marie", + "jfroche": "Jean-Fran\u00e7ois Roche", + "urbas": "Matej Urbas", + "osbm": "osman - \u30aa\u30b9\u30de\u30f3", + "rszyma": null, + "eclairevoyant": "\u00e9clairevoyant", + "Radvendii": "Taeer Bar-Yam", + "sinanmohd": "Sinan Mohd", + "ethanavatar": "Ethan Evans", + "gmarti": "Gr\u00e9gory Marti", + "lovesegfault": "Bernardo Meurer", + "EphraimSiegfried": "Ephraim Siegfried", + "hgl": "Glen Huang" } \ No newline at end of file diff --git a/maintainers/flake-module.nix b/maintainers/flake-module.nix index 4815313dd3f..8dcff9c63f0 100644 --- a/maintainers/flake-module.nix +++ b/maintainers/flake-module.nix @@ -104,151 +104,6 @@ }; shellcheck = { enable = true; - excludes = [ - # We haven't linted these files yet - ''^config/install-sh$'' - ''^misc/bash/completion\.sh$'' - ''^misc/fish/completion\.fish$'' - ''^misc/zsh/completion\.zsh$'' - ''^scripts/create-darwin-volume\.sh$'' - ''^scripts/install-darwin-multi-user\.sh$'' - ''^scripts/install-multi-user\.sh$'' - ''^scripts/install-systemd-multi-user\.sh$'' - ''^src/nix/get-env\.sh$'' - ''^tests/functional/ca/build-dry\.sh$'' - ''^tests/functional/ca/build-with-garbage-path\.sh$'' - ''^tests/functional/ca/common\.sh$'' - ''^tests/functional/ca/concurrent-builds\.sh$'' - ''^tests/functional/ca/eval-store\.sh$'' - ''^tests/functional/ca/gc\.sh$'' - ''^tests/functional/ca/import-from-derivation\.sh$'' - ''^tests/functional/ca/new-build-cmd\.sh$'' - ''^tests/functional/ca/nix-shell\.sh$'' - ''^tests/functional/ca/post-hook\.sh$'' - ''^tests/functional/ca/recursive\.sh$'' - ''^tests/functional/ca/repl\.sh$'' - ''^tests/functional/ca/selfref-gc\.sh$'' - ''^tests/functional/ca/why-depends\.sh$'' - ''^tests/functional/characterisation-test-infra\.sh$'' - ''^tests/functional/common/vars-and-functions\.sh$'' - ''^tests/functional/completions\.sh$'' - ''^tests/functional/compute-levels\.sh$'' - ''^tests/functional/config\.sh$'' - ''^tests/functional/db-migration\.sh$'' - ''^tests/functional/debugger\.sh$'' - ''^tests/functional/dependencies\.builder0\.sh$'' - ''^tests/functional/dependencies\.sh$'' - ''^tests/functional/dump-db\.sh$'' - ''^tests/functional/dyn-drv/build-built-drv\.sh$'' - ''^tests/functional/dyn-drv/common\.sh$'' - ''^tests/functional/dyn-drv/dep-built-drv\.sh$'' - ''^tests/functional/dyn-drv/eval-outputOf\.sh$'' - ''^tests/functional/dyn-drv/old-daemon-error-hack\.sh$'' - ''^tests/functional/dyn-drv/recursive-mod-json\.sh$'' - ''^tests/functional/eval-store\.sh$'' - ''^tests/functional/export-graph\.sh$'' - ''^tests/functional/export\.sh$'' - ''^tests/functional/extra-sandbox-profile\.sh$'' - ''^tests/functional/fetchClosure\.sh$'' - ''^tests/functional/fetchGit\.sh$'' - ''^tests/functional/fetchGitRefs\.sh$'' - ''^tests/functional/fetchGitSubmodules\.sh$'' - ''^tests/functional/fetchGitVerification\.sh$'' - ''^tests/functional/fetchMercurial\.sh$'' - ''^tests/functional/fixed\.builder1\.sh$'' - ''^tests/functional/fixed\.builder2\.sh$'' - ''^tests/functional/fixed\.sh$'' - ''^tests/functional/flakes/absolute-paths\.sh$'' - ''^tests/functional/flakes/check\.sh$'' - ''^tests/functional/flakes/config\.sh$'' - ''^tests/functional/flakes/flakes\.sh$'' - ''^tests/functional/flakes/follow-paths\.sh$'' - ''^tests/functional/flakes/prefetch\.sh$'' - ''^tests/functional/flakes/run\.sh$'' - ''^tests/functional/flakes/show\.sh$'' - ''^tests/functional/formatter\.sh$'' - ''^tests/functional/formatter\.simple\.sh$'' - ''^tests/functional/gc-auto\.sh$'' - ''^tests/functional/gc-concurrent\.builder\.sh$'' - ''^tests/functional/gc-concurrent\.sh$'' - ''^tests/functional/gc-concurrent2\.builder\.sh$'' - ''^tests/functional/gc-non-blocking\.sh$'' - ''^tests/functional/hash-convert\.sh$'' - ''^tests/functional/impure-derivations\.sh$'' - ''^tests/functional/impure-eval\.sh$'' - ''^tests/functional/install-darwin\.sh$'' - ''^tests/functional/legacy-ssh-store\.sh$'' - ''^tests/functional/linux-sandbox\.sh$'' - ''^tests/functional/local-overlay-store/add-lower-inner\.sh$'' - ''^tests/functional/local-overlay-store/add-lower\.sh$'' - ''^tests/functional/local-overlay-store/bad-uris\.sh$'' - ''^tests/functional/local-overlay-store/build-inner\.sh$'' - ''^tests/functional/local-overlay-store/build\.sh$'' - ''^tests/functional/local-overlay-store/check-post-init-inner\.sh$'' - ''^tests/functional/local-overlay-store/check-post-init\.sh$'' - ''^tests/functional/local-overlay-store/common\.sh$'' - ''^tests/functional/local-overlay-store/delete-duplicate-inner\.sh$'' - ''^tests/functional/local-overlay-store/delete-duplicate\.sh$'' - ''^tests/functional/local-overlay-store/delete-refs-inner\.sh$'' - ''^tests/functional/local-overlay-store/delete-refs\.sh$'' - ''^tests/functional/local-overlay-store/gc-inner\.sh$'' - ''^tests/functional/local-overlay-store/gc\.sh$'' - ''^tests/functional/local-overlay-store/optimise-inner\.sh$'' - ''^tests/functional/local-overlay-store/optimise\.sh$'' - ''^tests/functional/local-overlay-store/redundant-add-inner\.sh$'' - ''^tests/functional/local-overlay-store/redundant-add\.sh$'' - ''^tests/functional/local-overlay-store/remount\.sh$'' - ''^tests/functional/local-overlay-store/stale-file-handle-inner\.sh$'' - ''^tests/functional/local-overlay-store/stale-file-handle\.sh$'' - ''^tests/functional/local-overlay-store/verify-inner\.sh$'' - ''^tests/functional/local-overlay-store/verify\.sh$'' - ''^tests/functional/logging\.sh$'' - ''^tests/functional/misc\.sh$'' - ''^tests/functional/multiple-outputs\.sh$'' - ''^tests/functional/nested-sandboxing\.sh$'' - ''^tests/functional/nested-sandboxing/command\.sh$'' - ''^tests/functional/nix-build\.sh$'' - ''^tests/functional/nix-channel\.sh$'' - ''^tests/functional/nix-collect-garbage-d\.sh$'' - ''^tests/functional/nix-copy-ssh-common\.sh$'' - ''^tests/functional/nix-copy-ssh-ng\.sh$'' - ''^tests/functional/nix-copy-ssh\.sh$'' - ''^tests/functional/nix-daemon-untrusting\.sh$'' - ''^tests/functional/nix-profile\.sh$'' - ''^tests/functional/nix-shell\.sh$'' - ''^tests/functional/nix_path\.sh$'' - ''^tests/functional/optimise-store\.sh$'' - ''^tests/functional/output-normalization\.sh$'' - ''^tests/functional/parallel\.builder\.sh$'' - ''^tests/functional/parallel\.sh$'' - ''^tests/functional/pass-as-file\.sh$'' - ''^tests/functional/path-from-hash-part\.sh$'' - ''^tests/functional/path-info\.sh$'' - ''^tests/functional/placeholders\.sh$'' - ''^tests/functional/post-hook\.sh$'' - ''^tests/functional/pure-eval\.sh$'' - ''^tests/functional/push-to-store-old\.sh$'' - ''^tests/functional/push-to-store\.sh$'' - ''^tests/functional/read-only-store\.sh$'' - ''^tests/functional/readfile-context\.sh$'' - ''^tests/functional/recursive\.sh$'' - ''^tests/functional/referrers\.sh$'' - ''^tests/functional/remote-store\.sh$'' - ''^tests/functional/repair\.sh$'' - ''^tests/functional/restricted\.sh$'' - ''^tests/functional/search\.sh$'' - ''^tests/functional/secure-drv-outputs\.sh$'' - ''^tests/functional/selfref-gc\.sh$'' - ''^tests/functional/shell\.shebang\.sh$'' - ''^tests/functional/simple\.builder\.sh$'' - ''^tests/functional/supplementary-groups\.sh$'' - ''^tests/functional/toString-path\.sh$'' - ''^tests/functional/user-envs-migration\.sh$'' - ''^tests/functional/user-envs-test-case\.sh$'' - ''^tests/functional/user-envs\.builder\.sh$'' - ''^tests/functional/user-envs\.sh$'' - ''^tests/functional/why-depends\.sh$'' - ]; }; }; }; diff --git a/maintainers/link-headers b/maintainers/link-headers new file mode 100755 index 00000000000..2457a2dc829 --- /dev/null +++ b/maintainers/link-headers @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 + +# This script must be run from the root of the Nix repository. +# +# For include path hygiene, we need to put headers in a separate +# directory than sources. But during development, it is nice to paths +# that are similar for headers and source files, e.g. +# `foo/bar/baz.{cc,hh}`, e.g. for less typing when opening one file, and +# then opening the other file. +# +# This script symlinks the headers next to the source files to +# facilitate such a development workflows. It also updates +# `.git/info/exclude` so that the symlinks are not accidentally committed +# by mistake. + +from pathlib import Path +import subprocess +import os + + +def main() -> None: + # Path to the source directory + GIT_TOPLEVEL = Path( + subprocess.run( + ["git", "rev-parse", "--show-toplevel"], + text=True, + stdout=subprocess.PIPE, + check=True, + ).stdout.strip() + ) + + # Get header files from git + result = subprocess.run( + ["git", "-C", str(GIT_TOPLEVEL), "ls-files", "*/include/nix/**.hh"], + text=True, + stdout=subprocess.PIPE, + check=True, + ) + header_files = result.stdout.strip().split("\n") + header_files.sort() + + links = [] + for file_str in header_files: + project_str, header_str = file_str.split("/include/nix/", 1) + project = Path(project_str) + header = Path(header_str) + + # Reconstruct the full path (relative to SRC_DIR) to the header file. + file = project / "include" / "nix" / header + + # The symlink should be created at "project/header", i.e. next to the project's sources. + link = project / header + + # Compute a relative path from the symlink's parent directory to the actual header file. + relative_source = os.path.relpath( + GIT_TOPLEVEL / file, GIT_TOPLEVEL / link.parent + ) + + # Create the symbolic link. + full_link_path = GIT_TOPLEVEL / link + full_link_path.parent.mkdir(parents=True, exist_ok=True) + if full_link_path.is_symlink(): + full_link_path.unlink() + full_link_path.symlink_to(relative_source) + links.append(link) + + # Generate .gitignore file + gitignore_path = GIT_TOPLEVEL / ".git" / "info" / "exclude" + gitignore_path.parent.mkdir(parents=True, exist_ok=True) + with gitignore_path.open("w") as gitignore: + gitignore.write("# DO NOT EDIT! Autogenerated\n") + gitignore.write( + "# Symlinks for headers to be next to sources for development\n" + ) + gitignore.write('# Run "maintainers/link-headers" to regenerate\n\n') + gitignore.write('# Run "maintainers/link-headers" to regenerate\n\n') + + for link in links: + gitignore.write(f"/{link}\n") + + +if __name__ == "__main__": + main() diff --git a/maintainers/release-notes-todo b/maintainers/release-notes-todo new file mode 100755 index 00000000000..7cadc2a79e2 --- /dev/null +++ b/maintainers/release-notes-todo @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +set -euo pipefail +# debug: +# set -x + +START_REF="${1}" +END_REF="${2:-upstream/master}" + +# Get the merge base +MERGE_BASE=$(git merge-base "$START_REF" "$END_REF") +unset START_REF + +# Get date range +START_DATE=$(git show -s --format=%cI "$MERGE_BASE") +END_DATE=$(git show -s --format=%cI "$END_REF") + +echo "Checking PRs merged between $START_DATE and $END_DATE" >&2 + +# Get all commits between merge base and HEAD +COMMITS=$(git rev-list "$MERGE_BASE..$END_REF") + +# Convert to set for fast lookup +declare -A commit_set +for commit in $COMMITS; do + commit_set["$commit"]=1 +done + +# Get the current changelog +LOG_DONE="$(changelog-d doc/manual/rl-next)" +is_done(){ + local nr="$1" + echo "$LOG_DONE" | grep -E "^- .*/pull/$nr)" +} + +# Query merged PRs in date range +gh pr list \ + --repo NixOS/nix \ + --state merged \ + --limit 1000 \ + --json number,title,author,mergeCommit \ + --search "merged:$START_DATE..$END_DATE" | \ +jq -r '.[] | [.number, .mergeCommit.oid, .title, .author.login] | @tsv' | \ +while IFS=$'\t' read -r pr_num merge_commit _title author; do + # Check if this PR's merge commit is in our branch + if [[ -n "${commit_set[$merge_commit]:-}" ]]; then + # Full detail, not suitable for comment due to mass ping and duplicate title + # echo "- #$pr_num $_title (@$author)" + echo "- #$pr_num ($author)" + if is_done "$pr_num" + then + echo " - [x] has note" + else + echo " - [ ] has note" + fi + echo " - [ ] skip" + fi +done diff --git a/maintainers/release-process.md b/maintainers/release-process.md index fa47bbb1d89..790618b7f4c 100644 --- a/maintainers/release-process.md +++ b/maintainers/release-process.md @@ -24,6 +24,12 @@ release: * In a checkout of the Nix repo, make sure you're on `master` and run `git pull`. +* Compile a release notes to-do list by running + + ```console + $ ./maintainers/release-notes-todo PREV_RELEASE HEAD + ``` + * Compile the release notes by running ```console @@ -127,6 +133,8 @@ release: Commit and push this to the maintenance branch. +* Create a backport label. + * Bump the version of `master`: ```console @@ -134,6 +142,7 @@ release: $ git pull $ NEW_VERSION=2.13.0 $ echo $NEW_VERSION > .version + $ ... edit .mergify.yml to add the previous version ... $ git checkout -b bump-$NEW_VERSION $ git commit -a -m 'Bump version' $ git push --set-upstream origin bump-$NEW_VERSION @@ -141,10 +150,6 @@ release: Make a pull request and auto-merge it. -* Create a backport label. - -* Add the new backport label to `.mergify.yml`. - * Post an [announcement on Discourse](https://discourse.nixos.org/c/announcements/8), including the contents of `rl-$VERSION.md`. diff --git a/meson.build b/meson.build index 5dcf98717f5..73675615721 100644 --- a/meson.build +++ b/meson.build @@ -41,8 +41,10 @@ subproject('libexpr-c') subproject('libflake-c') subproject('libmain-c') +asan_enabled = 'address' in get_option('b_sanitize') + # Language Bindings -if get_option('bindings') and not meson.is_cross_build() +if get_option('bindings') and not meson.is_cross_build() and not asan_enabled subproject('perl') endif diff --git a/misc/bash/completion.sh b/misc/bash/completion.sh index c4ba96cd32c..96f98d6c13d 100644 --- a/misc/bash/completion.sh +++ b/misc/bash/completion.sh @@ -1,3 +1,4 @@ +# shellcheck shell=bash function _complete_nix { local -a words local cword cur diff --git a/misc/fish/completion.fish b/misc/fish/completion.fish index c6b8ef16a8b..b6584963b0d 100644 --- a/misc/fish/completion.fish +++ b/misc/fish/completion.fish @@ -1,3 +1,4 @@ +# shellcheck disable=all function _nix_complete # Get the current command up to a cursor. # - Behaves correctly even with pipes and nested in commands like env. diff --git a/misc/zsh/completion.zsh b/misc/zsh/completion.zsh index f9b3dca7456..eb26a16cb03 100644 --- a/misc/zsh/completion.zsh +++ b/misc/zsh/completion.zsh @@ -1,3 +1,4 @@ +# shellcheck disable=all #compdef nix function _nix() { diff --git a/nix-meson-build-support/asan-options/meson.build b/nix-meson-build-support/asan-options/meson.build new file mode 100644 index 00000000000..17880b0ed25 --- /dev/null +++ b/nix-meson-build-support/asan-options/meson.build @@ -0,0 +1,12 @@ +asan_test_options_env = { + 'ASAN_OPTIONS' : 'abort_on_error=1:print_summary=1:detect_leaks=0', +} + +# Clang gets grumpy about missing libasan symbols if -shared-libasan is not +# passed when building shared libs, at least on Linux +if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option( + 'b_sanitize', +)) + add_project_link_arguments('-shared-libasan', language : 'cpp') +endif + diff --git a/nix-meson-build-support/common/assert-fail/meson.build b/nix-meson-build-support/common/assert-fail/meson.build new file mode 100644 index 00000000000..7539b392132 --- /dev/null +++ b/nix-meson-build-support/common/assert-fail/meson.build @@ -0,0 +1,32 @@ +can_wrap_assert_fail_test_code = ''' +#include +#include + +int main() +{ + assert(0); +} + +extern "C" void * __real___assert_fail(const char *, const char *, unsigned int, const char *); + +extern "C" void * +__wrap___assert_fail(const char *, const char *, unsigned int, const char *) +{ + return __real___assert_fail(nullptr, nullptr, 0, nullptr); +} +''' + +wrap_assert_fail_args = [ '-Wl,--wrap=__assert_fail' ] + +can_wrap_assert_fail = cxx.links( + can_wrap_assert_fail_test_code, + args : wrap_assert_fail_args, + name : 'linker can wrap __assert_fail', +) + +if can_wrap_assert_fail + deps_other += declare_dependency( + sources : 'wrap-assert-fail.cc', + link_args : wrap_assert_fail_args, + ) +endif diff --git a/nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc b/nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc new file mode 100644 index 00000000000..d9e34168bc9 --- /dev/null +++ b/nix-meson-build-support/common/assert-fail/wrap-assert-fail.cc @@ -0,0 +1,17 @@ +#include "nix/util/error.hh" + +#include +#include +#include +#include + +extern "C" [[noreturn]] void __attribute__((weak)) +__wrap___assert_fail(const char * assertion, const char * file, unsigned int line, const char * function) +{ + char buf[512]; + int n = + snprintf(buf, sizeof(buf), "Assertion '%s' failed in %s at %s:%" PRIuLEAST32, assertion, function, file, line); + if (n < 0) + nix::panic("Assertion failed and could not format error message"); + nix::panic(std::string_view(buf, std::min(static_cast(sizeof(buf)), n))); +} diff --git a/nix-meson-build-support/common/meson.build b/nix-meson-build-support/common/meson.build index fd686f1407f..2944a733b05 100644 --- a/nix-meson-build-support/common/meson.build +++ b/nix-meson-build-support/common/meson.build @@ -5,6 +5,15 @@ if not (host_machine.system() == 'windows' and cxx.get_id() == 'gcc') deps_private += dependency('threads') endif +if host_machine.system() == 'cygwin' + # -std=gnu on cygwin defines 'unix', which conflicts with the namespace + add_project_arguments( + '-D_POSIX_C_SOURCE=200809L', + '-D_GNU_SOURCE', + language : 'cpp', + ) +endif + add_project_arguments( '-Wdeprecated-copy', '-Werror=suggest-override', @@ -33,10 +42,7 @@ if cxx.get_id() == 'clang' add_project_arguments('-fpch-instantiate-templates', language : 'cpp') endif -# Clang gets grumpy about missing libasan symbols if -shared-libasan is not -# passed when building shared libs, at least on Linux -if cxx.get_id() == 'clang' and ('address' in get_option('b_sanitize') or 'undefined' in get_option( - 'b_sanitize', -)) - add_project_link_arguments('-shared-libasan', language : 'cpp') -endif +# Darwin ld doesn't like "X.Y.Zpre" +nix_soversion = meson.project_version().split('pre')[0] + +subdir('assert-fail') diff --git a/nix-meson-build-support/libatomic/meson.build b/nix-meson-build-support/libatomic/meson.build index d16d2381764..1c014bee7d9 100644 --- a/nix-meson-build-support/libatomic/meson.build +++ b/nix-meson-build-support/libatomic/meson.build @@ -3,6 +3,6 @@ # This is needed for std::atomic on some platforms # We did not manage to test this reliably on all platforms, so we hardcode # it for now. -if host_machine.cpu_family() == 'arm' +if host_machine.cpu_family() in [ 'arm', 'ppc' ] deps_other += cxx.find_library('atomic') endif diff --git a/packaging/components.nix b/packaging/components.nix index b5fad404343..49249797cca 100644 --- a/packaging/components.nix +++ b/packaging/components.nix @@ -27,7 +27,7 @@ let pkg-config ; - baseVersion = lib.fileContents ../.version; + baseVersion = lib.fileContents ../.version-determinate; versionSuffix = lib.optionalString (!officialRelease) "pre"; @@ -51,15 +51,6 @@ let exts: userFn: stdenv.mkDerivation (lib.extends (lib.composeManyExtensions exts) userFn); setVersionLayer = finalAttrs: prevAttrs: { - preConfigure = - prevAttrs.preConfigure or "" - + - # Update the repo-global .version file. - # Symlink ./.version points there, but by default only workDir is writable. - '' - chmod u+w ./.version - echo ${finalAttrs.version} > ./.version - ''; }; localSourceLayer = @@ -164,6 +155,24 @@ let }; mesonLibraryLayer = finalAttrs: prevAttrs: { + preConfigure = + let + interpositionFlags = [ + "-fno-semantic-interposition" + "-Wl,-Bsymbolic-functions" + ]; + in + # NOTE: By default GCC disables interprocedular optimizations (in particular inlining) for + # position-independent code and thus shared libraries. + # Since LD_PRELOAD tricks aren't worth losing out on optimizations, we disable it for good. + # This is not the case for Clang, where inlining is done by default even without -fno-semantic-interposition. + # https://reviews.llvm.org/D102453 + # https://fedoraproject.org/wiki/Changes/PythonNoSemanticInterpositionSpeedup + prevAttrs.preConfigure or "" + + lib.optionalString stdenv.cc.isGNU '' + export CFLAGS="''${CFLAGS:-} ${toString interpositionFlags}" + export CXXFLAGS="''${CXXFLAGS:-} ${toString interpositionFlags}" + ''; outputs = prevAttrs.outputs or [ "out" ] ++ [ "dev" ]; }; diff --git a/packaging/dependencies.nix b/packaging/dependencies.nix index 3d7da9acb44..0f1471672bf 100644 --- a/packaging/dependencies.nix +++ b/packaging/dependencies.nix @@ -10,27 +10,8 @@ stdenv, }: -let - prevStdenv = stdenv; -in - let inherit (pkgs) lib; - - stdenv = if prevStdenv.isDarwin && prevStdenv.isx86_64 then darwinStdenv else prevStdenv; - - # Fix the following error with the default x86_64-darwin SDK: - # - # error: aligned allocation function of type 'void *(std::size_t, std::align_val_t)' is only available on macOS 10.13 or newer - # - # Despite the use of the 10.13 deployment target here, the aligned - # allocation function Clang uses with this setting actually works - # all the way back to 10.6. - # NOTE: this is not just a version constraint, but a request to make Darwin - # provide this version level of support. Removing this minimum version - # request will regress the above error. - darwinStdenv = pkgs.overrideSDK prevStdenv { darwinMinVersion = "10.13"; }; - in scope: { inherit stdenv; @@ -76,6 +57,21 @@ scope: { prevAttrs.postInstall; }); + # TODO: Remove this when https://github.com/NixOS/nixpkgs/pull/442682 is included in a stable release + toml11 = + if lib.versionAtLeast pkgs.toml11.version "4.4.0" then + pkgs.toml11 + else + pkgs.toml11.overrideAttrs rec { + version = "4.4.0"; + src = pkgs.fetchFromGitHub { + owner = "ToruNiina"; + repo = "toml11"; + tag = "v${version}"; + hash = "sha256-sgWKYxNT22nw376ttGsTdg0AMzOwp8QH3E8mx0BZJTQ="; + }; + }; + # TODO Hack until https://github.com/NixOS/nixpkgs/issues/45462 is fixed. boost = (pkgs.boost.override { @@ -85,6 +81,7 @@ scope: { "--with-coroutine" "--with-iostreams" "--with-url" + "--with-thread" ]; enableIcu = false; }).overrideAttrs diff --git a/packaging/dev-shell.nix b/packaging/dev-shell.nix index 949f7975231..c4fc70511c1 100644 --- a/packaging/dev-shell.nix +++ b/packaging/dev-shell.nix @@ -26,7 +26,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( pname = "shell-for-" + attrs.pname; # Remove the version suffix to avoid unnecessary attempts to substitute in nix develop - version = lib.fileContents ../.version; + version = lib.fileContents ../.version-determinate; name = attrs.pname; installFlags = "sysconfdir=$(out)/etc"; @@ -118,6 +118,7 @@ pkgs.nixComponents2.nix-util.overrideAttrs ( modular.pre-commit.settings.package (pkgs.writeScriptBin "pre-commit-hooks-install" modular.pre-commit.settings.installationScript) pkgs.buildPackages.nixfmt-rfc-style + pkgs.buildPackages.shellcheck pkgs.buildPackages.gdb ] ++ lib.optional (stdenv.cc.isClang && stdenv.hostPlatform == stdenv.buildPlatform) ( diff --git a/packaging/everything.nix b/packaging/everything.nix index f6bdad4907b..3206b8ba423 100644 --- a/packaging/everything.nix +++ b/packaging/everything.nix @@ -75,7 +75,7 @@ let }; devdoc = buildEnv { - name = "nix-${nix-cli.version}-devdoc"; + name = "determinate-nix-${nix-cli.version}-devdoc"; paths = [ nix-internal-api-docs nix-external-api-docs @@ -84,7 +84,7 @@ let in stdenv.mkDerivation (finalAttrs: { - pname = "nix"; + pname = "determinate-nix"; version = nix-cli.version; /** diff --git a/packaging/hydra.nix b/packaging/hydra.nix index 9f9749bde16..d563bff0bb3 100644 --- a/packaging/hydra.nix +++ b/packaging/hydra.nix @@ -119,65 +119,6 @@ in system: self.devShells.${system}.default.inputDerivation )) [ "i686-linux" ]; - buildStatic = forAllPackages ( - pkgName: - lib.genAttrs linux64BitSystems ( - system: nixpkgsFor.${system}.native.pkgsStatic.nixComponents2.${pkgName} - ) - ); - - buildCross = forAllPackages ( - pkgName: - # Hack to avoid non-evaling package - ( - if pkgName == "nix-functional-tests" then - lib.flip builtins.removeAttrs [ "x86_64-w64-mingw32" ] - else - lib.id - ) - ( - forAllCrossSystems ( - crossSystem: - lib.genAttrs [ "x86_64-linux" ] ( - system: nixpkgsFor.${system}.cross.${crossSystem}.nixComponents2.${pkgName} - ) - ) - ) - ); - - buildNoGc = - let - components = forAllSystems ( - system: - nixpkgsFor.${system}.native.nixComponents2.overrideScope ( - self: super: { - nix-expr = super.nix-expr.override { enableGC = false; }; - } - ) - ); - in - forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName})); - - buildNoTests = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-cli); - - # Toggles some settings for better coverage. Windows needs these - # library combinations, and Debian build Nix with GNU readline too. - buildReadlineNoMarkdown = - let - components = forAllSystems ( - system: - nixpkgsFor.${system}.native.nixComponents2.overrideScope ( - self: super: { - nix-cmd = super.nix-cmd.override { - enableMarkdown = false; - readlineFlavor = "readline"; - }; - } - ) - ); - in - forAllPackages (pkgName: forAllSystems (system: components.${system}.${pkgName})); - # Perl bindings for various platforms. perlBindings = forAllSystems (system: nixpkgsFor.${system}.native.nixComponents2.nix-perl-bindings); @@ -188,30 +129,6 @@ in system: nixpkgsFor.${system}.native.callPackage ./binary-tarball.nix { } ); - binaryTarballCross = lib.genAttrs [ "x86_64-linux" ] ( - system: - forAllCrossSystems ( - crossSystem: nixpkgsFor.${system}.cross.${crossSystem}.callPackage ./binary-tarball.nix { } - ) - ); - - # The first half of the installation script. This is uploaded - # to https://nixos.org/nix/install. It downloads the binary - # tarball for the user's system and calls the second half of the - # installation script. - installerScript = installScriptFor [ - # Native - self.hydraJobs.binaryTarball."x86_64-linux" - self.hydraJobs.binaryTarball."i686-linux" - self.hydraJobs.binaryTarball."aarch64-linux" - self.hydraJobs.binaryTarball."x86_64-darwin" - self.hydraJobs.binaryTarball."aarch64-darwin" - # Cross - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv6l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."armv7l-unknown-linux-gnueabihf" - self.hydraJobs.binaryTarballCross."x86_64-linux"."riscv64-unknown-linux-gnu" - ]; - installerScriptForGHA = forAllSystems ( system: nixpkgsFor.${system}.native.callPackage ./installer { @@ -279,6 +196,19 @@ in pkgs = nixpkgsFor.${system}.native; } ); + + nixpkgsLibTestsLazy = forAllSystems ( + system: + lib.overrideDerivation + (import (nixpkgs + "/lib/tests/test-with-nix.nix") { + lib = nixpkgsFor.${system}.native.lib; + nix = self.packages.${system}.nix-cli; + pkgs = nixpkgsFor.${system}.native; + }) + (_: { + "NIX_CONFIG" = "lazy-trees = true"; + }) + ); }; metrics.nixpkgs = import "${nixpkgs-regression}/pkgs/top-level/metrics.nix" { @@ -293,17 +223,12 @@ in in pkgs.runCommand "install-tests" { againstSelf = testNixVersions pkgs pkgs.nix; - againstCurrentLatest = - # FIXME: temporarily disable this on macOS because of #3605. - if system == "x86_64-linux" then testNixVersions pkgs pkgs.nixVersions.latest else null; + #againstCurrentLatest = + # # FIXME: temporarily disable this on macOS because of #3605. + # if system == "x86_64-linux" then testNixVersions pkgs pkgs.nixVersions.latest else null; # Disabled because the latest stable version doesn't handle # `NIX_DAEMON_SOCKET_PATH` which is required for the tests to work # againstLatestStable = testNixVersions pkgs pkgs.nixStable; } "touch $out" ); - - installerTests = import ../tests/installer { - binaryTarballs = self.hydraJobs.binaryTarball; - inherit nixpkgsFor; - }; } diff --git a/packaging/installer/default.nix b/packaging/installer/default.nix index e171f36f99f..a8e344b496c 100644 --- a/packaging/installer/default.nix +++ b/packaging/installer/default.nix @@ -32,7 +32,7 @@ runCommand "installer-script" in '' \ - --replace '@tarballHash_${system}@' $(nix --experimental-features nix-command hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ + --replace '@tarballHash_${system}@' $(nix hash-file --base16 --type sha256 ${tarball}/*.tar.xz) \ --replace '@tarballPath_${system}@' $(tarballPath ${tarball}/*.tar.xz) \ '' ) tarballs diff --git a/scripts/install-multi-user.sh b/scripts/install-multi-user.sh index 477eb1fd682..b013190f97a 100644 --- a/scripts/install-multi-user.sh +++ b/scripts/install-multi-user.sh @@ -55,18 +55,22 @@ readonly NIX_INSTALLED_NIX="@nix@" readonly NIX_INSTALLED_CACERT="@cacert@" #readonly NIX_INSTALLED_NIX="/nix/store/j8dbv5w6jl34caywh2ygdy88knx1mdf7-nix-2.3.6" #readonly NIX_INSTALLED_CACERT="/nix/store/7dxhzymvy330i28ii676fl1pqwcahv2f-nss-cacert-3.49.2" -readonly EXTRACTED_NIX_PATH="$(dirname "$0")" +EXTRACTED_NIX_PATH="$(dirname "$0")" +readonly EXTRACTED_NIX_PATH # allow to override identity change command -readonly NIX_BECOME=${NIX_BECOME:-sudo} +NIX_BECOME=${NIX_BECOME:-sudo} +readonly NIX_BECOME -readonly ROOT_HOME=~root +ROOT_HOME=~root +readonly ROOT_HOME if [ -t 0 ] && [ -z "${NIX_INSTALLER_YES:-}" ]; then - readonly IS_HEADLESS='no' + IS_HEADLESS='no' else - readonly IS_HEADLESS='yes' + IS_HEADLESS='yes' fi +readonly IS_HEADLESS headless() { if [ "$IS_HEADLESS" = "yes" ]; then @@ -156,6 +160,7 @@ EOF } nix_user_for_core() { + # shellcheck disable=SC2059 printf "$NIX_BUILD_USER_NAME_TEMPLATE" "$1" } @@ -381,10 +386,12 @@ _sudo() { # Ensure that $TMPDIR exists if defined. if [[ -n "${TMPDIR:-}" ]] && [[ ! -d "${TMPDIR:-}" ]]; then + # shellcheck disable=SC2174 mkdir -m 0700 -p "${TMPDIR:-}" fi -readonly SCRATCH=$(mktemp -d) +SCRATCH=$(mktemp -d) +readonly SCRATCH finish_cleanup() { rm -rf "$SCRATCH" } @@ -677,7 +684,8 @@ create_directories() { # hiding behind || true, and the general state # should be one the user can repair once they # figure out where chown is... - local get_chr_own="$(PATH="$(getconf PATH 2>/dev/null)" command -vp chown)" + local get_chr_own + get_chr_own="$(PATH="$(getconf PATH 2>/dev/null)" command -vp chown)" if [[ -z "$get_chr_own" ]]; then get_chr_own="$(command -v chown)" fi @@ -915,9 +923,11 @@ configure_shell_profile() { fi if [ -e "$profile_target" ]; then - shell_source_lines \ - | _sudo "extend your $profile_target with nix-daemon settings" \ - tee -a "$profile_target" + { + shell_source_lines + cat "$profile_target" + } | _sudo "extend your $profile_target with nix-daemon settings" \ + tee "$profile_target" fi done @@ -1013,6 +1023,7 @@ main() { # Set profile targets after OS-specific scripts are loaded if command -v poly_configure_default_profile_targets > /dev/null 2>&1; then + # shellcheck disable=SC2207 PROFILE_TARGETS=($(poly_configure_default_profile_targets)) else PROFILE_TARGETS=("/etc/bashrc" "/etc/profile.d/nix.sh" "/etc/zshrc" "/etc/bash.bashrc" "/etc/zsh/zshrc") diff --git a/scripts/install-systemd-multi-user.sh b/scripts/install-systemd-multi-user.sh index dc373f4db3b..8abbb7af4ad 100755 --- a/scripts/install-systemd-multi-user.sh +++ b/scripts/install-systemd-multi-user.sh @@ -39,7 +39,7 @@ create_systemd_proxy_env() { vars="http_proxy https_proxy ftp_proxy all_proxy no_proxy HTTP_PROXY HTTPS_PROXY FTP_PROXY ALL_PROXY NO_PROXY" for v in $vars; do if [ "x${!v:-}" != "x" ]; then - echo "Environment=${v}=$(escape_systemd_env ${!v})" + echo "Environment=${v}=$(escape_systemd_env "${!v}")" fi done } diff --git a/shell.nix b/shell.nix deleted file mode 100644 index 918f4bbd9e9..00000000000 --- a/shell.nix +++ /dev/null @@ -1,3 +0,0 @@ -(import (fetchTarball "https://github.com/edolstra/flake-compat/archive/master.tar.gz") { - src = ./.; -}).shellNix diff --git a/src/external-api-docs/package.nix b/src/external-api-docs/package.nix index b194e16d460..28cde8c09e6 100644 --- a/src/external-api-docs/package.nix +++ b/src/external-api-docs/package.nix @@ -14,7 +14,7 @@ let in mkMesonDerivation (finalAttrs: { - pname = "nix-external-api-docs"; + pname = "determinate-nix-external-api-docs"; inherit version; workDir = ./.; diff --git a/src/internal-api-docs/package.nix b/src/internal-api-docs/package.nix index 6c4f354aee5..636c19653ea 100644 --- a/src/internal-api-docs/package.nix +++ b/src/internal-api-docs/package.nix @@ -14,7 +14,7 @@ let in mkMesonDerivation (finalAttrs: { - pname = "nix-internal-api-docs"; + pname = "determinate-nix-internal-api-docs"; inherit version; workDir = ./.; diff --git a/src/libcmd/built-path.cc b/src/libcmd/built-path.cc index 80d97dc3e9a..4d76dd6da39 100644 --- a/src/libcmd/built-path.cc +++ b/src/libcmd/built-path.cc @@ -83,12 +83,22 @@ nlohmann::json SingleBuiltPath::Built::toJSON(const StoreDirConfig & store) cons nlohmann::json SingleBuiltPath::toJSON(const StoreDirConfig & store) const { - return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw()); + return std::visit( + overloaded{ + [&](const SingleBuiltPath::Opaque & o) -> nlohmann::json { return store.printStorePath(o.path); }, + [&](const SingleBuiltPath::Built & b) { return b.toJSON(store); }, + }, + raw()); } nlohmann::json BuiltPath::toJSON(const StoreDirConfig & store) const { - return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw()); + return std::visit( + overloaded{ + [&](const BuiltPath::Opaque & o) -> nlohmann::json { return store.printStorePath(o.path); }, + [&](const BuiltPath::Built & b) { return b.toJSON(store); }, + }, + raw()); } RealisedPath::Set BuiltPath::toRealisedPaths(Store & store) const diff --git a/src/libcmd/command.cc b/src/libcmd/command.cc index 6b6bbe34585..077381eee43 100644 --- a/src/libcmd/command.cc +++ b/src/libcmd/command.cc @@ -125,6 +125,13 @@ ref EvalCommand::getEvalStore() ref EvalCommand::getEvalState() { if (!evalState) { + if (startReplOnEvalErrors && evalSettings.evalCores != 1U) { + // Disable parallel eval if the debugger is enabled, since + // they're incompatible at the moment. + warn("using the debugger disables multi-threaded evaluation"); + evalSettings.evalCores = 1; + } + evalState = std::allocate_shared( traceable_allocator(), lookupPath, getEvalStore(), fetchSettings, evalSettings, getStore()); diff --git a/src/libcmd/common-eval-args.cc b/src/libcmd/common-eval-args.cc index f7e086c169c..890fe8337b3 100644 --- a/src/libcmd/common-eval-args.cc +++ b/src/libcmd/common-eval-args.cc @@ -19,17 +19,12 @@ namespace nix { -fetchers::Settings fetchSettings; - -static GlobalConfig::Register rFetchSettings(&fetchSettings); - EvalSettings evalSettings{ settings.readOnlyMode, { { "flake", [](EvalState & state, std::string_view rest) { - experimentalFeatureSettings.require(Xp::Flakes); // FIXME `parseFlakeRef` should take a `std::string_view`. auto flakeRef = parseFlakeRef(fetchSettings, std::string{rest}, {}, true, false); debug("fetching flake search path element '%s''", rest); @@ -185,7 +180,6 @@ SourcePath lookupFileArg(EvalState & state, std::string_view s, const Path * bas } else if (hasPrefix(s, "flake:")) { - experimentalFeatureSettings.require(Xp::Flakes); auto flakeRef = parseFlakeRef(fetchSettings, std::string(s.substr(6)), {}, true, false); auto [accessor, lockedRef] = flakeRef.resolve(state.store).lazyFetch(state.store); auto storePath = nix::fetchToStore( diff --git a/src/libcmd/include/nix/cmd/command.hh b/src/libcmd/include/nix/cmd/command.hh index 20cd1abc1c4..bd5786fcdea 100644 --- a/src/libcmd/include/nix/cmd/command.hh +++ b/src/libcmd/include/nix/cmd/command.hh @@ -214,6 +214,8 @@ struct InstallableCommand : virtual Args, SourceExprCommand { InstallableCommand(); + virtual void preRun(ref store); + virtual void run(ref store, ref installable) = 0; void run(ref store) override; @@ -350,6 +352,20 @@ struct MixEnvironment : virtual Args void setEnviron(); }; +struct MixNoCheckSigs : virtual Args +{ + CheckSigsFlag checkSigs = CheckSigs; + + MixNoCheckSigs() + { + addFlag({ + .longName = "no-check-sigs", + .description = "Do not require that paths are signed by trusted keys.", + .handler = {&checkSigs, NoCheckSigs}, + }); + } +}; + void completeFlakeInputAttrPath( AddCompletions & completions, ref evalState, diff --git a/src/libcmd/include/nix/cmd/common-eval-args.hh b/src/libcmd/include/nix/cmd/common-eval-args.hh index 62518ba0e7f..dd8c34c1d9c 100644 --- a/src/libcmd/include/nix/cmd/common-eval-args.hh +++ b/src/libcmd/include/nix/cmd/common-eval-args.hh @@ -25,9 +25,6 @@ namespace flake { struct Settings; } -/** - * @todo Get rid of global settings variables - */ extern fetchers::Settings fetchSettings; /** diff --git a/src/libcmd/installable-attr-path.cc b/src/libcmd/installable-attr-path.cc index 28c3db3fc79..3a80aa384de 100644 --- a/src/libcmd/installable-attr-path.cc +++ b/src/libcmd/installable-attr-path.cc @@ -89,7 +89,8 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() } DerivedPathsWithInfo res; - for (auto & [drvPath, outputs] : byDrvPath) + for (auto & [drvPath, outputs] : byDrvPath) { + state->waitForPath(drvPath); res.push_back({ .path = DerivedPath::Built{ @@ -102,6 +103,7 @@ DerivedPathsWithInfo InstallableAttrPath::toDerivedPaths() so we can fill in this info. */ }), }); + } return res; } diff --git a/src/libcmd/installable-flake.cc b/src/libcmd/installable-flake.cc index 97f7eb645fa..8ac80806235 100644 --- a/src/libcmd/installable-flake.cc +++ b/src/libcmd/installable-flake.cc @@ -102,11 +102,12 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() } auto drvPath = attr->forceDerivation(); + state->waitForPath(drvPath); std::optional priority; - if (attr->maybeGetAttr(state->sOutputSpecified)) { - } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + if (attr->maybeGetAttr(state->s.outputSpecified)) { + } else if (auto aMeta = attr->maybeGetAttr(state->s.meta)) { if (auto aPriority = aMeta->maybeGetAttr("priority")) priority = aPriority->getInt().value; } @@ -119,12 +120,12 @@ DerivedPathsWithInfo InstallableFlake::toDerivedPaths() overloaded{ [&](const ExtendedOutputsSpec::Default & d) -> OutputsSpec { StringSet outputsToInstall; - if (auto aOutputSpecified = attr->maybeGetAttr(state->sOutputSpecified)) { + if (auto aOutputSpecified = attr->maybeGetAttr(state->s.outputSpecified)) { if (aOutputSpecified->getBool()) { if (auto aOutputName = attr->maybeGetAttr("outputName")) outputsToInstall = {aOutputName->getString()}; } - } else if (auto aMeta = attr->maybeGetAttr(state->sMeta)) { + } else if (auto aMeta = attr->maybeGetAttr(state->s.meta)) { if (auto aOutputsToInstall = aMeta->maybeGetAttr("outputsToInstall")) for (auto & s : aOutputsToInstall->getListOfStrings()) outputsToInstall.insert(s); diff --git a/src/libcmd/installable-value.cc b/src/libcmd/installable-value.cc index 3a167af3db4..ec53ee97c89 100644 --- a/src/libcmd/installable-value.cc +++ b/src/libcmd/installable-value.cc @@ -55,7 +55,7 @@ InstallableValue::trySinglePathToDerivedPaths(Value & v, const PosIdx pos, std:: else if (v.type() == nString) { return {{ - .path = DerivedPath::fromSingle(state->coerceToSingleDerivedPath(pos, v, errorCtx)), + .path = DerivedPath::fromSingle(state->devirtualize(state->coerceToSingleDerivedPath(pos, v, errorCtx))), .info = make_ref(), }}; } diff --git a/src/libcmd/installables.cc b/src/libcmd/installables.cc index 0e6a204a7fb..37959431a18 100644 --- a/src/libcmd/installables.cc +++ b/src/libcmd/installables.cc @@ -178,10 +178,16 @@ MixFlakeOptions::MixFlakeOptions() for (auto & [inputName, input] : flake.lockFile.root->inputs) { auto input2 = flake.lockFile.findInput({inputName}); // resolve 'follows' nodes if (auto input3 = std::dynamic_pointer_cast(input2)) { + fetchers::Attrs extraAttrs; + + if (!input3->lockedRef.subdir.empty()) { + extraAttrs["dir"] = input3->lockedRef.subdir; + } + overrideRegistry( fetchers::Input::fromAttrs(fetchSettings, {{"type", "indirect"}, {"id", inputName}}), input3->lockedRef.input, - {}); + extraAttrs); } } }}, @@ -395,9 +401,6 @@ void completeFlakeRefWithFragment( void completeFlakeRef(AddCompletions & completions, ref store, std::string_view prefix) { - if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) - return; - if (prefix == "") completions.add("."); @@ -598,28 +601,28 @@ std::vector Installable::build( static void throwBuildErrors(std::vector & buildResults, const Store & store) { - std::vector failed; + std::vector> failed; for (auto & buildResult : buildResults) { - if (!buildResult.success()) { - failed.push_back(buildResult); + if (auto * failure = buildResult.tryGetFailure()) { + failed.push_back({&buildResult, failure}); } } auto failedResult = failed.begin(); if (failedResult != failed.end()) { if (failed.size() == 1) { - failedResult->rethrow(); + failedResult->second->rethrow(); } else { StringSet failedPaths; for (; failedResult != failed.end(); failedResult++) { - if (!failedResult->errorMsg.empty()) { + if (!failedResult->second->errorMsg.empty()) { logError( ErrorInfo{ .level = lvlError, - .msg = failedResult->errorMsg, + .msg = failedResult->second->errorMsg, }); } - failedPaths.insert(failedResult->path.to_string(store)); + failedPaths.insert(failedResult->first->path.to_string(store)); } throw Error("build of %s failed", concatStringsSep(", ", quoteStrings(failedPaths))); } @@ -689,12 +692,14 @@ std::vector, BuiltPathWithResult>> Installable::build auto buildResults = store->buildPathsWithResults(pathsToBuild, bMode, evalStore); throwBuildErrors(buildResults, *store); for (auto & buildResult : buildResults) { + // If we didn't throw, they must all be sucesses + auto & success = std::get(buildResult.inner); for (auto & aux : backmap[buildResult.path]) { std::visit( overloaded{ [&](const DerivedPath::Built & bfd) { std::map outputs; - for (auto & [outputName, realisation] : buildResult.builtOutputs) + for (auto & [outputName, realisation] : success.builtOutputs) outputs.emplace(outputName, realisation.outPath); res.push_back( {aux.installable, @@ -869,8 +874,11 @@ InstallableCommand::InstallableCommand() }); } +void InstallableCommand::preRun(ref store) {} + void InstallableCommand::run(ref store) { + preRun(store); auto installable = parseInstallable(store, _installable); run(store, std::move(installable)); } diff --git a/src/libcmd/meson.build b/src/libcmd/meson.build index 24e0752462c..3833d7e0a9d 100644 --- a/src/libcmd/meson.build +++ b/src/libcmd/meson.build @@ -67,6 +67,7 @@ config_priv_h = configure_file( ) subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'built-path.cc', @@ -95,6 +96,7 @@ this_library = library( 'nixcmd', sources, config_priv_h, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, diff --git a/src/libcmd/package.nix b/src/libcmd/package.nix index c382f0e5760..21d7586a321 100644 --- a/src/libcmd/package.nix +++ b/src/libcmd/package.nix @@ -35,7 +35,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-cmd"; + pname = "determinate-nix-cmd"; inherit version; workDir = ./.; diff --git a/src/libcmd/repl.cc b/src/libcmd/repl.cc index 01d786debfb..2a6582cd01b 100644 --- a/src/libcmd/repl.cc +++ b/src/libcmd/repl.cc @@ -333,6 +333,7 @@ StorePath NixRepl::getDerivationPath(Value & v) auto drvPath = packageInfo->queryDrvPath(); if (!drvPath) throw Error("expression did not evaluate to a valid derivation (no 'drvPath' attribute)"); + state->waitForPath(*drvPath); if (!state->store->isValidPath(*drvPath)) throw Error("expression evaluated to invalid derivation '%s'", state->store->printStorePath(*drvPath)); return *drvPath; @@ -669,7 +670,7 @@ ProcessLineResult NixRepl::processLine(std::string line) ss << "No documentation found.\n\n"; } - auto markdown = toView(ss); + auto markdown = ss.view(); logger->cout(trim(renderMarkdownToTerminal(markdown))); } else @@ -760,7 +761,7 @@ void NixRepl::loadFlake(const std::string & flakeRefS) void NixRepl::initEnv() { - env = &state->allocEnv(envSize); + env = &state->mem.allocEnv(envSize); env->up = &state->baseEnv; displ = 0; staticEnv->vars.clear(); @@ -869,14 +870,8 @@ void NixRepl::addVarToScope(const Symbol name, Value & v) Expr * NixRepl::parseString(std::string s) { - return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv); -} - -void NixRepl::evalString(std::string s, Value & v) -{ - Expr * e; try { - e = parseString(s); + return state->parseExprFromString(std::move(s), state->rootPath("."), staticEnv); } catch (ParseError & e) { if (e.msg().find("unexpected end of file") != std::string::npos) // For parse errors on incomplete input, we continue waiting for the next line of @@ -885,6 +880,11 @@ void NixRepl::evalString(std::string s, Value & v) else throw; } +} + +void NixRepl::evalString(std::string s, Value & v) +{ + Expr * e = parseString(s); e->eval(*state, *env, v); state->forceValue(v, v.determinePos(noPos)); } diff --git a/src/libexpr-c/meson.build b/src/libexpr-c/meson.build index 7c014d61d37..03cee41a09a 100644 --- a/src/libexpr-c/meson.build +++ b/src/libexpr-c/meson.build @@ -28,6 +28,7 @@ deps_public_maybe_subproject = [ subdir('nix-meson-build-support/subprojects') subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'nix_api_expr.cc', @@ -50,6 +51,7 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nixexprc', sources, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, diff --git a/src/libexpr-c/nix_api_expr.cc b/src/libexpr-c/nix_api_expr.cc index 02e901de9f2..6ba5c07cf89 100644 --- a/src/libexpr-c/nix_api_expr.cc +++ b/src/libexpr-c/nix_api_expr.cc @@ -16,7 +16,7 @@ #include "nix_api_util_internal.h" #if NIX_USE_BOEHMGC -# include +# include #endif /** @@ -40,6 +40,8 @@ static T * unsafe_new_with_self(F && init) return new (p) T(init(static_cast(p))); } +extern "C" { + nix_err nix_libexpr_init(nix_c_context * context) { if (context) @@ -69,6 +71,7 @@ nix_err nix_expr_eval_from_string( nix::Expr * parsedExpr = state->state.parseExprFromString(expr, state->state.rootPath(nix::CanonPath(path))); state->state.eval(parsedExpr, value->value); state->state.forceValue(value->value, nix::noPos); + state->state.waitForAllPaths(); } NIXC_CATCH_ERRS } @@ -80,6 +83,7 @@ nix_err nix_value_call(nix_c_context * context, EvalState * state, Value * fn, n try { state->state.callFunction(fn->value, arg->value, value->value, nix::noPos); state->state.forceValue(value->value, nix::noPos); + state->state.waitForAllPaths(); } NIXC_CATCH_ERRS } @@ -92,6 +96,7 @@ nix_err nix_value_call_multi( try { state->state.callFunction(fn->value, {(nix::Value **) args, nargs}, value->value, nix::noPos); state->state.forceValue(value->value, nix::noPos); + state->state.waitForAllPaths(); } NIXC_CATCH_ERRS } @@ -102,6 +107,7 @@ nix_err nix_value_force(nix_c_context * context, EvalState * state, nix_value * context->last_err_code = NIX_OK; try { state->state.forceValue(value->value, nix::noPos); + state->state.waitForAllPaths(); } NIXC_CATCH_ERRS } @@ -112,6 +118,7 @@ nix_err nix_value_force_deep(nix_c_context * context, EvalState * state, nix_val context->last_err_code = NIX_OK; try { state->state.forceValueDeep(value->value); + state->state.waitForAllPaths(); } NIXC_CATCH_ERRS } @@ -135,7 +142,7 @@ nix_eval_state_builder * nix_eval_state_builder_new(nix_c_context * context, Sto void nix_eval_state_builder_free(nix_eval_state_builder * builder) { - delete builder; + operator delete(builder, static_cast(alignof(nix_eval_state_builder))); } nix_err nix_eval_state_builder_load(nix_c_context * context, nix_eval_state_builder * builder) @@ -201,32 +208,24 @@ EvalState * nix_state_create(nix_c_context * context, const char ** lookupPath_c void nix_state_free(EvalState * state) { - delete state; + operator delete(state, static_cast(alignof(EvalState))); } #if NIX_USE_BOEHMGC -std::unordered_map< +boost::concurrent_flat_map< const void *, unsigned int, std::hash, std::equal_to, traceable_allocator>> - nix_refcounts; - -std::mutex nix_refcount_lock; + nix_refcounts{}; nix_err nix_gc_incref(nix_c_context * context, const void * p) { if (context) context->last_err_code = NIX_OK; try { - std::scoped_lock lock(nix_refcount_lock); - auto f = nix_refcounts.find(p); - if (f != nix_refcounts.end()) { - f->second++; - } else { - nix_refcounts[p] = 1; - } + nix_refcounts.insert_or_visit({p, 1}, [](auto & kv) { kv.second++; }); } NIXC_CATCH_ERRS } @@ -237,12 +236,12 @@ nix_err nix_gc_decref(nix_c_context * context, const void * p) if (context) context->last_err_code = NIX_OK; try { - std::scoped_lock lock(nix_refcount_lock); - auto f = nix_refcounts.find(p); - if (f != nix_refcounts.end()) { - if (--f->second == 0) - nix_refcounts.erase(f); - } else + bool fail = true; + nix_refcounts.erase_if(p, [&](auto & kv) { + fail = false; + return !--kv.second; + }); + if (fail) throw std::runtime_error("nix_gc_decref: object was not referenced"); } NIXC_CATCH_ERRS @@ -287,3 +286,5 @@ void nix_gc_register_finalizer(void * obj, void * cd, void (*finalizer)(void * o GC_REGISTER_FINALIZER(obj, finalizer, cd, 0, 0); #endif } + +} // extern "C" diff --git a/src/libexpr-c/nix_api_expr_internal.h b/src/libexpr-c/nix_api_expr_internal.h index a26595cec5d..3aa1d993225 100644 --- a/src/libexpr-c/nix_api_expr_internal.h +++ b/src/libexpr-c/nix_api_expr_internal.h @@ -8,6 +8,8 @@ #include "nix_api_value.h" #include "nix/expr/search-path.hh" +extern "C" { + struct nix_eval_state_builder { nix::ref store; @@ -61,4 +63,6 @@ struct nix_realised_string std::vector storePaths; }; +} // extern "C" + #endif // NIX_API_EXPR_INTERNAL_H diff --git a/src/libexpr-c/nix_api_external.cc b/src/libexpr-c/nix_api_external.cc index ecb67cfb495..ff2950448c6 100644 --- a/src/libexpr-c/nix_api_external.cc +++ b/src/libexpr-c/nix_api_external.cc @@ -14,6 +14,8 @@ #include +extern "C" { + void nix_set_string_return(nix_string_return * str, const char * c) { str->str = c; @@ -40,6 +42,8 @@ nix_err nix_external_add_string_context(nix_c_context * context, nix_string_cont NIXC_CATCH_ERRS } +} // extern "C" + class NixCExternalValue : public nix::ExternalValueBase { NixCExternalValueDesc & desc; @@ -170,6 +174,8 @@ class NixCExternalValue : public nix::ExternalValueBase virtual ~NixCExternalValue() override {}; }; +extern "C" { + ExternalValue * nix_create_external_value(nix_c_context * context, NixCExternalValueDesc * desc, void * v) { if (context) @@ -198,3 +204,5 @@ void * nix_get_external_value_content(nix_c_context * context, ExternalValue * b } NIXC_CATCH_ERRS_NULL } + +} // extern "C" diff --git a/src/libexpr-c/nix_api_value.cc b/src/libexpr-c/nix_api_value.cc index fb90e2872e6..5ad85128e1e 100644 --- a/src/libexpr-c/nix_api_value.cc +++ b/src/libexpr-c/nix_api_value.cc @@ -111,6 +111,8 @@ static void nix_c_primop_wrapper( v = vTmp; } +extern "C" { + PrimOp * nix_alloc_primop( nix_c_context * context, PrimOpFun fun, @@ -175,6 +177,8 @@ ValueType nix_get_type(nix_c_context * context, const nix_value * value) switch (v.type()) { case nThunk: return NIX_TYPE_THUNK; + case nFailed: + return NIX_TYPE_FAILED; case nInt: return NIX_TYPE_INT; case nFloat: @@ -324,6 +328,10 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, try { auto & v = check_value_in(value); assert(v.type() == nix::nList); + if (ix >= v.listSize()) { + nix_set_err_msg(context, NIX_ERR_KEY, "list index out of bounds"); + return nullptr; + } auto * p = v.listView()[ix]; nix_gc_incref(nullptr, p); if (p != nullptr) @@ -333,6 +341,26 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, NIXC_CATCH_ERRS_NULL } +nix_value * +nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + assert(v.type() == nix::nList); + if (ix >= v.listSize()) { + nix_set_err_msg(context, NIX_ERR_KEY, "list index out of bounds"); + return nullptr; + } + auto * p = v.listView()[ix]; + nix_gc_incref(nullptr, p); + // Note: intentionally NOT calling forceValue() to keep the element lazy + return as_nix_value_ptr(p); + } + NIXC_CATCH_ERRS_NULL +} + nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name) { if (context) @@ -345,6 +373,28 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value if (attr) { nix_gc_incref(nullptr, attr->value); state->state.forceValue(*attr->value, nix::noPos); + state->state.waitForAllPaths(); + return as_nix_value_ptr(attr->value); + } + nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute"); + return nullptr; + } + NIXC_CATCH_ERRS_NULL +} + +nix_value * +nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalState * state, const char * name) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + assert(v.type() == nix::nAttrs); + nix::Symbol s = state->state.symbols.create(name); + auto attr = v.attrs()->get(s); + if (attr) { + nix_gc_incref(nullptr, attr->value); + // Note: intentionally NOT calling forceValue() to keep the attribute lazy return as_nix_value_ptr(attr->value); } nix_set_err_msg(context, NIX_ERR_KEY, "missing attribute"); @@ -369,13 +419,28 @@ bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalS NIXC_CATCH_ERRS_RES(false); } -nix_value * nix_get_attr_byidx( - nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name) +static void collapse_attrset_layer_chain_if_needed(nix::Value & v, EvalState * state) +{ + auto & attrs = *v.attrs(); + if (attrs.isLayered()) { + auto bindings = state->state.buildBindings(attrs.size()); + std::ranges::copy(attrs, std::back_inserter(bindings)); + v.mkAttrs(bindings); + } +} + +nix_value * +nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name) { if (context) context->last_err_code = NIX_OK; try { auto & v = check_value_in(value); + collapse_attrset_layer_chain_if_needed(v, state); + if (i >= v.attrs()->size()) { + nix_set_err_msg(context, NIX_ERR_KEY, "attribute index out of bounds"); + return nullptr; + } const nix::Attr & a = (*v.attrs())[i]; *name = state->state.symbols[a.name].c_str(); nix_gc_incref(nullptr, a.value); @@ -385,13 +450,38 @@ nix_value * nix_get_attr_byidx( NIXC_CATCH_ERRS_NULL } -const char * -nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i) +nix_value * nix_get_attr_byidx_lazy( + nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name) { if (context) context->last_err_code = NIX_OK; try { auto & v = check_value_in(value); + collapse_attrset_layer_chain_if_needed(v, state); + if (i >= v.attrs()->size()) { + nix_set_err_msg(context, NIX_ERR_KEY, "attribute index out of bounds (Nix C API contract violation)"); + return nullptr; + } + const nix::Attr & a = (*v.attrs())[i]; + *name = state->state.symbols[a.name].c_str(); + nix_gc_incref(nullptr, a.value); + // Note: intentionally NOT calling forceValue() to keep the attribute lazy + return as_nix_value_ptr(a.value); + } + NIXC_CATCH_ERRS_NULL +} + +const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto & v = check_value_in(value); + collapse_attrset_layer_chain_if_needed(v, state); + if (i >= v.attrs()->size()) { + nix_set_err_msg(context, NIX_ERR_KEY, "attribute index out of bounds (Nix C API contract violation)"); + return nullptr; + } const nix::Attr & a = (*v.attrs())[i]; return state->state.symbols[a.name].c_str(); } @@ -592,7 +682,7 @@ nix_err nix_bindings_builder_insert(nix_c_context * context, BindingsBuilder * b context->last_err_code = NIX_OK; try { auto & v = check_value_not_null(value); - nix::Symbol s = bb->builder.state.symbols.create(name); + nix::Symbol s = bb->builder.symbols.get().create(name); bb->builder.insert(s, &v); } NIXC_CATCH_ERRS @@ -651,3 +741,5 @@ const StorePath * nix_realised_string_get_store_path(nix_realised_string * s, si { return &s->storePaths[i]; } + +} // extern "C" diff --git a/src/libexpr-c/nix_api_value.h b/src/libexpr-c/nix_api_value.h index 7cd6ad18087..7540ba77d22 100644 --- a/src/libexpr-c/nix_api_value.h +++ b/src/libexpr-c/nix_api_value.h @@ -32,7 +32,8 @@ typedef enum { NIX_TYPE_ATTRS, NIX_TYPE_LIST, NIX_TYPE_FUNCTION, - NIX_TYPE_EXTERNAL + NIX_TYPE_EXTERNAL, + NIX_TYPE_FAILED, } ValueType; // forward declarations @@ -265,10 +266,25 @@ ExternalValue * nix_get_external(nix_c_context * context, nix_value * value); */ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix); -/** @brief Get an attr by name +/** @brief Get the ix'th element of a list without forcing evaluation of the element + * + * Returns the list element without forcing its evaluation, allowing access to lazy values. + * The list value itself must already be evaluated. * * Owned by the GC. Use nix_gc_decref when you're done with the pointer * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect (must be an evaluated list) + * @param[in] state nix evaluator state + * @param[in] ix list element to get + * @return value, NULL in case of errors + */ +nix_value * +nix_get_list_byidx_lazy(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int ix); + +/** @brief Get an attr by name + * + * Use nix_gc_decref when you're done with the pointer + * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @param[in] state nix evaluator state * @param[in] name attribute name @@ -276,6 +292,21 @@ nix_value * nix_get_list_byidx(nix_c_context * context, const nix_value * value, */ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); +/** @brief Get an attribute value by attribute name, without forcing evaluation of the attribute's value + * + * Returns the attribute value without forcing its evaluation, allowing access to lazy values. + * The attribute set value itself must already be evaluated. + * + * Use nix_gc_decref when you're done with the pointer + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect (must be an evaluated attribute set) + * @param[in] state nix evaluator state + * @param[in] name attribute name + * @return value, NULL in case of errors + */ +nix_value * +nix_get_attr_byname_lazy(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); + /** @brief Check if an attribute name exists on a value * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect @@ -285,11 +316,21 @@ nix_value * nix_get_attr_byname(nix_c_context * context, const nix_value * value */ bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalState * state, const char * name); -/** @brief Get an attribute by index in the sorted bindings +/** @brief Get an attribute by index * * Also gives you the name. * - * Owned by the GC. Use nix_gc_decref when you're done with the pointer + * Attributes are returned in an unspecified order which is NOT suitable for + * reproducible operations. In Nix's domain, reproducibility is paramount. The caller + * is responsible for sorting the attributes or storing them in an ordered map to + * ensure deterministic behavior in your application. + * + * @note When Nix does sort attributes, which it does for virtually all intermediate + * operations and outputs, it uses byte-wise lexicographic order (equivalent to + * lexicographic order by Unicode scalar value for valid UTF-8). We recommend + * applying this same ordering for consistency. + * + * Use nix_gc_decref when you're done with the pointer * @param[out] context Optional, stores error information * @param[in] value Nix value to inspect * @param[in] state nix evaluator state @@ -297,12 +338,50 @@ bool nix_has_attr_byname(nix_c_context * context, const nix_value * value, EvalS * @param[out] name will store a pointer to the attribute name * @return value, NULL in case of errors */ -nix_value * nix_get_attr_byidx( - nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i, const char ** name); +nix_value * +nix_get_attr_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name); + +/** @brief Get an attribute by index, without forcing evaluation of the attribute's value + * + * Also gives you the name. + * + * Returns the attribute value without forcing its evaluation, allowing access to lazy values. + * The attribute set value itself must already have been evaluated. + * + * Attributes are returned in an unspecified order which is NOT suitable for + * reproducible operations. In Nix's domain, reproducibility is paramount. The caller + * is responsible for sorting the attributes or storing them in an ordered map to + * ensure deterministic behavior in your application. + * + * @note When Nix does sort attributes, which it does for virtually all intermediate + * operations and outputs, it uses byte-wise lexicographic order (equivalent to + * lexicographic order by Unicode scalar value for valid UTF-8). We recommend + * applying this same ordering for consistency. + * + * Use nix_gc_decref when you're done with the pointer + * @param[out] context Optional, stores error information + * @param[in] value Nix value to inspect (must be an evaluated attribute set) + * @param[in] state nix evaluator state + * @param[in] i attribute index + * @param[out] name will store a pointer to the attribute name + * @return value, NULL in case of errors + */ +nix_value * nix_get_attr_byidx_lazy( + nix_c_context * context, nix_value * value, EvalState * state, unsigned int i, const char ** name); -/** @brief Get an attribute name by index in the sorted bindings +/** @brief Get an attribute name by index + * + * Returns the attribute name without forcing evaluation of the attribute's value. + * + * Attributes are returned in an unspecified order which is NOT suitable for + * reproducible operations. In Nix's domain, reproducibility is paramount. The caller + * is responsible for sorting the attributes or storing them in an ordered map to + * ensure deterministic behavior in your application. * - * Useful when you want the name but want to avoid evaluation. + * @note When Nix does sort attributes, which it does for virtually all intermediate + * operations and outputs, it uses byte-wise lexicographic order (equivalent to + * lexicographic order by Unicode scalar value for valid UTF-8). We recommend + * applying this same ordering for consistency. * * Owned by the nix EvalState * @param[out] context Optional, stores error information @@ -311,8 +390,7 @@ nix_value * nix_get_attr_byidx( * @param[in] i attribute index * @return name, NULL in case of errors */ -const char * -nix_get_attr_name_byidx(nix_c_context * context, const nix_value * value, EvalState * state, unsigned int i); +const char * nix_get_attr_name_byidx(nix_c_context * context, nix_value * value, EvalState * state, unsigned int i); /**@}*/ /** @name Initializers diff --git a/src/libexpr-c/package.nix b/src/libexpr-c/package.nix index 694fbc1fe78..ec92ecce105 100644 --- a/src/libexpr-c/package.nix +++ b/src/libexpr-c/package.nix @@ -15,7 +15,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-expr-c"; + pname = "determinate-nix-expr-c"; inherit version; workDir = ./.; diff --git a/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh b/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh index 4cf985e1534..a1320e14a25 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/libexpr.hh @@ -26,11 +26,20 @@ public: } protected: - LibExprTest() + LibExprTest(ref store, auto && makeEvalSettings) : LibStoreTest() + , evalSettings(makeEvalSettings(readOnlyMode)) , state({}, store, fetchSettings, evalSettings, nullptr) { - evalSettings.nixPath = {}; + } + + LibExprTest() + : LibExprTest(openStore("dummy://"), [](bool & readOnlyMode) { + EvalSettings settings{readOnlyMode}; + settings.nixPath = {}; + return settings; + }) + { } Value eval(std::string input, bool forceValue = true) diff --git a/src/libexpr-test-support/include/nix/expr/tests/value/context.hh b/src/libexpr-test-support/include/nix/expr/tests/value/context.hh index 68a0b8dea7d..2311f3941c1 100644 --- a/src/libexpr-test-support/include/nix/expr/tests/value/context.hh +++ b/src/libexpr-test-support/include/nix/expr/tests/value/context.hh @@ -26,6 +26,12 @@ struct Arbitrary static Gen arbitrary(); }; +template<> +struct Arbitrary +{ + static Gen arbitrary(); +}; + template<> struct Arbitrary { diff --git a/src/libexpr-test-support/meson.build b/src/libexpr-test-support/meson.build index d762eb85e32..01a3f3bcbbf 100644 --- a/src/libexpr-test-support/meson.build +++ b/src/libexpr-test-support/meson.build @@ -31,6 +31,7 @@ rapidcheck = dependency('rapidcheck') deps_public += rapidcheck subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'tests/value/context.cc', @@ -44,6 +45,7 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-expr-test-support', sources, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 diff --git a/src/libexpr-test-support/package.nix b/src/libexpr-test-support/package.nix index 5cb4adaa8c4..1879a571608 100644 --- a/src/libexpr-test-support/package.nix +++ b/src/libexpr-test-support/package.nix @@ -18,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-util-test-support"; + pname = "determinate-nix-util-test-support"; inherit version; workDir = ./.; diff --git a/src/libexpr-test-support/tests/value/context.cc b/src/libexpr-test-support/tests/value/context.cc index d6036601a94..8ce84fb51f5 100644 --- a/src/libexpr-test-support/tests/value/context.cc +++ b/src/libexpr-test-support/tests/value/context.cc @@ -16,6 +16,15 @@ Gen Arbitrary::arb }); } +Gen Arbitrary::arbitrary() +{ + return gen::map(gen::arbitrary(), [](StorePath storePath) { + return NixStringContextElem::Path{ + .storePath = storePath, + }; + }); +} + Gen Arbitrary::arbitrary() { return gen::mapcat( @@ -31,6 +40,8 @@ Gen Arbitrary::arbitrary() case 2: return gen::map( gen::arbitrary(), [](NixStringContextElem a) { return a; }); + case 3: + return gen::map(gen::arbitrary(), [](NixStringContextElem a) { return a; }); default: assert(false); } diff --git a/src/libexpr-tests/eval.cc b/src/libexpr-tests/eval.cc index ad70ea5b8d2..7562a9da21a 100644 --- a/src/libexpr-tests/eval.cc +++ b/src/libexpr-tests/eval.cc @@ -3,6 +3,7 @@ #include "nix/expr/eval.hh" #include "nix/expr/tests/libexpr.hh" +#include "nix/util/memory-source-accessor.hh" namespace nix { @@ -174,4 +175,41 @@ TEST_F(EvalStateTest, getBuiltin_fail) ASSERT_THROW(state.getBuiltin("nonexistent"), EvalError); } +class PureEvalTest : public LibExprTest +{ +public: + PureEvalTest() + : LibExprTest(openStore("dummy://", {{"read-only", "false"}}), [](bool & readOnlyMode) { + EvalSettings settings{readOnlyMode}; + settings.pureEval = true; + settings.restrictEval = true; + return settings; + }) + { + } +}; + +TEST_F(PureEvalTest, pathExists) +{ + ASSERT_THAT(eval("builtins.pathExists /."), IsFalse()); + ASSERT_THAT(eval("builtins.pathExists /nix"), IsFalse()); + ASSERT_THAT(eval("builtins.pathExists /nix/store"), IsFalse()); + + { + std::string contents = "Lorem ipsum"; + + StringSource s{contents}; + auto path = state.store->addToStoreFromDump( + s, "source", FileSerialisationMethod::Flat, ContentAddressMethod::Raw::Text, HashAlgorithm::SHA256); + auto printed = store->printStorePath(path); + + ASSERT_THROW(eval(fmt("builtins.readFile %s", printed)), RestrictedPathError); + ASSERT_THAT(eval(fmt("builtins.pathExists %s", printed)), IsFalse()); + + ASSERT_THROW(eval("builtins.readDir /."), RestrictedPathError); + state.allowPath(path); // FIXME: This shouldn't behave this way. + ASSERT_THAT(eval("builtins.readDir /."), IsAttrsOfSize(0)); + } +} + } // namespace nix diff --git a/src/libexpr-tests/json.cc b/src/libexpr-tests/json.cc index c090ac5d7c7..8b1bd7d96d9 100644 --- a/src/libexpr-tests/json.cc +++ b/src/libexpr-tests/json.cc @@ -54,7 +54,7 @@ TEST_F(JSONValueTest, IntNegative) TEST_F(JSONValueTest, String) { Value v; - v.mkString("test"); + v.mkStringNoCopy("test"); ASSERT_EQ(getJSONValue(v), "\"test\""); } @@ -62,7 +62,7 @@ TEST_F(JSONValueTest, StringQuotes) { Value v; - v.mkString("test\""); + v.mkStringNoCopy("test\""); ASSERT_EQ(getJSONValue(v), "\"test\\\"\""); } diff --git a/src/libexpr-tests/main.cc b/src/libexpr-tests/main.cc index 61b40e8349f..88a9d6684d5 100644 --- a/src/libexpr-tests/main.cc +++ b/src/libexpr-tests/main.cc @@ -1,40 +1,15 @@ #include -#include -#include "nix/store/globals.hh" -#include "nix/util/logging.hh" + +#include "nix/store/tests/test-main.hh" +#include "nix/util/config-global.hh" using namespace nix; int main(int argc, char ** argv) { - if (argc > 1 && std::string_view(argv[1]) == "__build-remote") { - printError("test-build-remote: not supported in libexpr unit tests"); - return 1; - } - - // Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook. - settings.buildHook = {}; - -#ifdef __linux__ // should match the conditional around sandboxBuildDir declaration. - - // When building and testing nix within the host's Nix sandbox, our store dir will be located in the host's - // sandboxBuildDir, e.g.: Host - // storeDir = /nix/store - // sandboxBuildDir = /build - // This process - // storeDir = /build/foo/bar/store - // sandboxBuildDir = /build - // However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different - // sandboxBuildDir. - settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir"; -#endif - -#ifdef __APPLE__ - // Avoid this error, when already running in a sandbox: - // sandbox-exec: sandbox_apply: Operation not permitted - settings.sandboxMode = smDisabled; - setEnv("_NIX_TEST_NO_SANDBOX", "1"); -#endif + auto res = testMainForBuidingPre(argc, argv); + if (res) + return res; // For pipe operator tests in trivial.cc experimentalFeatureSettings.set("experimental-features", "pipe-operators"); diff --git a/src/libexpr-tests/meson.build b/src/libexpr-tests/meson.build index a876e970550..7f7c08955c0 100644 --- a/src/libexpr-tests/meson.build +++ b/src/libexpr-tests/meson.build @@ -45,6 +45,7 @@ config_priv_h = configure_file( ) subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'derived-path.cc', @@ -55,6 +56,7 @@ sources = files( 'nix_api_expr.cc', 'nix_api_external.cc', 'nix_api_value.cc', + 'nix_api_value_internal.cc', 'primops.cc', 'search-path.cc', 'trivial.cc', @@ -81,7 +83,7 @@ this_exe = executable( test( meson.project_name(), this_exe, - env : { + env : asan_test_options_env + { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', }, protocol : 'gtest', diff --git a/src/libexpr-tests/nix_api_expr.cc b/src/libexpr-tests/nix_api_expr.cc index 529c2f5845b..de508b4e40b 100644 --- a/src/libexpr-tests/nix_api_expr.cc +++ b/src/libexpr-tests/nix_api_expr.cc @@ -1,7 +1,5 @@ #include "nix_api_store.h" -#include "nix_api_store_internal.h" #include "nix_api_util.h" -#include "nix_api_util_internal.h" #include "nix_api_expr.h" #include "nix_api_value.h" @@ -151,8 +149,8 @@ TEST_F(nix_api_expr_test, nix_expr_realise_context_bad_value) assert_ctx_ok(); auto r = nix_string_realise(ctx, state, value, false); ASSERT_EQ(nullptr, r); - ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); - ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("cannot coerce"))); + ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR); + ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("cannot coerce")); } TEST_F(nix_api_expr_test, nix_expr_realise_context_bad_build) @@ -168,8 +166,8 @@ TEST_F(nix_api_expr_test, nix_expr_realise_context_bad_build) assert_ctx_ok(); auto r = nix_string_realise(ctx, state, value, false); ASSERT_EQ(nullptr, r); - ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); - ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("failed with exit code 1"))); + ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR); + ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("failed with exit code 1")); } TEST_F(nix_api_expr_test, nix_expr_realise_context) @@ -381,12 +379,11 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_no_return) nix_value * result = nix_alloc_value(ctx, state); assert_ctx_ok(); nix_value_call(ctx, state, primopValue, three, result); - ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR); ASSERT_THAT( - ctx->last_err, - testing::Optional( - testing::HasSubstr("Implementation error in custom function: return value was not initialized"))); - ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("badNoReturn"))); + nix_err_msg(nullptr, ctx, nullptr), + testing::HasSubstr("Implementation error in custom function: return value was not initialized")); + ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("badNoReturn")); } static void primop_bad_return_thunk( @@ -419,12 +416,60 @@ TEST_F(nix_api_expr_test, nix_expr_primop_bad_return_thunk) assert_ctx_ok(); NIX_VALUE_CALL(ctx, state, result, primopValue, toString, four); - ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR); ASSERT_THAT( - ctx->last_err, - testing::Optional( - testing::HasSubstr("Implementation error in custom function: return value must not be a thunk"))); - ASSERT_THAT(ctx->last_err, testing::Optional(testing::HasSubstr("badReturnThunk"))); + nix_err_msg(nullptr, ctx, nullptr), + testing::HasSubstr("Implementation error in custom function: return value must not be a thunk")); + ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("badReturnThunk")); +} + +static void primop_with_nix_err_key( + void * user_data, nix_c_context * context, EvalState * state, nix_value ** args, nix_value * ret) +{ + nix_set_err_msg(context, NIX_ERR_KEY, "Test error from primop"); +} + +TEST_F(nix_api_expr_test, nix_expr_primop_nix_err_key_conversion) +{ + // Test that NIX_ERR_KEY from a custom primop gets converted to a generic EvalError + // + // RATIONALE: NIX_ERR_KEY must not be propagated from custom primops because it would + // create semantic confusion. NIX_ERR_KEY indicates missing keys/indices in C API functions + // (like nix_get_attr_byname, nix_get_list_byidx). If custom primops could return NIX_ERR_KEY, + // an evaluation error would be indistinguishable from an actual missing attribute. + // + // For example, if nix_get_attr_byname returned NIX_ERR_KEY when the attribute is present + // but the value evaluation fails, callers expecting NIX_ERR_KEY to mean "missing attribute" + // would incorrectly handle evaluation failures as missing attributes. In places where + // missing attributes are tolerated (like optional attributes), this would cause the + // program to continue after swallowing the error, leading to silent failures. + PrimOp * primop = nix_alloc_primop( + ctx, primop_with_nix_err_key, 1, "testErrorPrimop", nullptr, "a test primop that sets NIX_ERR_KEY", nullptr); + assert_ctx_ok(); + nix_value * primopValue = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_primop(ctx, primopValue, primop); + assert_ctx_ok(); + + nix_value * arg = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_init_int(ctx, arg, 42); + assert_ctx_ok(); + + nix_value * result = nix_alloc_value(ctx, state); + assert_ctx_ok(); + nix_value_call(ctx, state, primopValue, arg, result); + + // Verify that NIX_ERR_KEY gets converted to NIX_ERR_NIX_ERROR (generic evaluation error) + ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR); + ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("Error from custom function")); + ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("Test error from primop")); + ASSERT_THAT(nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("testErrorPrimop")); + + // Clean up + nix_gc_decref(ctx, primopValue); + nix_gc_decref(ctx, arg); + nix_gc_decref(ctx, result); } TEST_F(nix_api_expr_test, nix_value_call_multi_no_args) @@ -441,4 +486,31 @@ TEST_F(nix_api_expr_test, nix_value_call_multi_no_args) assert_ctx_ok(); ASSERT_EQ(3, rInt); } + +TEST_F(nix_api_expr_test, nix_expr_attrset_update) +{ + nix_expr_eval_from_string(ctx, state, "{ a = 0; b = 2; } // { a = 1; b = 3; } // { a = 2; }", ".", value); + assert_ctx_ok(); + + ASSERT_EQ(nix_get_attrs_size(ctx, value), 2); + assert_ctx_ok(); + std::array, 2> values; + for (unsigned int i = 0; i < 2; ++i) { + const char * name; + values[i].second = nix_get_attr_byidx(ctx, value, state, i, &name); + assert_ctx_ok(); + values[i].first = name; + } + std::sort(values.begin(), values.end(), [](const auto & lhs, const auto & rhs) { return lhs.first < rhs.first; }); + + nix_value * a = values[0].second; + ASSERT_EQ("a", values[0].first); + ASSERT_EQ(nix_get_int(ctx, a), 2); + assert_ctx_ok(); + nix_value * b = values[1].second; + ASSERT_EQ("b", values[1].first); + ASSERT_EQ(nix_get_int(ctx, b), 3); + assert_ctx_ok(); +} + } // namespace nixC diff --git a/src/libexpr-tests/nix_api_external.cc b/src/libexpr-tests/nix_api_external.cc index 93da3ca393c..ec19f1212e9 100644 --- a/src/libexpr-tests/nix_api_external.cc +++ b/src/libexpr-tests/nix_api_external.cc @@ -1,9 +1,6 @@ #include "nix_api_store.h" -#include "nix_api_store_internal.h" #include "nix_api_util.h" -#include "nix_api_util_internal.h" #include "nix_api_expr.h" -#include "nix_api_expr_internal.h" #include "nix_api_value.h" #include "nix_api_external.h" @@ -39,7 +36,7 @@ class MyExternalValueDesc : public NixCExternalValueDesc std::string type_string = "nix-external_x); type_string += " )>"; - res->str = &*type_string.begin(); + nix_set_string_return(res, &*type_string.begin()); } }; diff --git a/src/libexpr-tests/nix_api_value.cc b/src/libexpr-tests/nix_api_value.cc index 5d85ed68d4b..830637f3ec5 100644 --- a/src/libexpr-tests/nix_api_value.cc +++ b/src/libexpr-tests/nix_api_value.cc @@ -1,10 +1,7 @@ #include "nix_api_store.h" -#include "nix_api_store_internal.h" #include "nix_api_util.h" -#include "nix_api_util_internal.h" #include "nix_api_expr.h" #include "nix_api_value.h" -#include "nix_api_expr_internal.h" #include "nix/expr/tests/nix_api_expr.hh" #include "nix/util/tests/string_callback.hh" @@ -16,14 +13,6 @@ namespace nixC { -TEST_F(nix_api_expr_test, as_nix_value_ptr) -{ - // nix_alloc_value casts nix::Value to nix_value - // It should be obvious from the decl that that works, but if it doesn't, - // the whole implementation would be utterly broken. - ASSERT_EQ(sizeof(nix::Value), sizeof(nix_value)); -} - TEST_F(nix_api_expr_test, nix_value_get_int_invalid) { ASSERT_EQ(0, nix_get_int(ctx, nullptr)); @@ -173,6 +162,114 @@ TEST_F(nix_api_expr_test, nix_build_and_init_list) nix_gc_decref(ctx, intValue); } +TEST_F(nix_api_expr_test, nix_get_list_byidx_large_indices) +{ + // Create a small list to test extremely large out-of-bounds access + ListBuilder * builder = nix_make_list_builder(ctx, state, 2); + nix_value * intValue = nix_alloc_value(ctx, state); + nix_init_int(ctx, intValue, 42); + nix_list_builder_insert(ctx, builder, 0, intValue); + nix_list_builder_insert(ctx, builder, 1, intValue); + nix_make_list(ctx, builder, value); + nix_list_builder_free(builder); + + // Test extremely large indices that would definitely crash without bounds checking + ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, 1000000)); + ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx)); + ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, UINT_MAX / 2)); + ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx)); + ASSERT_EQ(nullptr, nix_get_list_byidx(ctx, value, state, UINT_MAX / 2 + 1000000)); + ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx)); + + // Clean up + nix_gc_decref(ctx, intValue); +} + +TEST_F(nix_api_expr_test, nix_get_list_byidx_lazy) +{ + // Create a list with a throwing lazy element, an already-evaluated int, and a lazy function call + + // 1. Throwing lazy element - create a function application thunk that will throw when forced + nix_value * throwingFn = nix_alloc_value(ctx, state); + nix_value * throwingValue = nix_alloc_value(ctx, state); + + nix_expr_eval_from_string( + ctx, + state, + R"( + _: throw "This should not be evaluated by the lazy accessor" + )", + "", + throwingFn); + assert_ctx_ok(); + + nix_init_apply(ctx, throwingValue, throwingFn, throwingFn); + assert_ctx_ok(); + + // 2. Already evaluated int (not lazy) + nix_value * intValue = nix_alloc_value(ctx, state); + nix_init_int(ctx, intValue, 42); + assert_ctx_ok(); + + // 3. Lazy function application that would compute increment 5 = 6 + nix_value * lazyApply = nix_alloc_value(ctx, state); + nix_value * incrementFn = nix_alloc_value(ctx, state); + nix_value * argFive = nix_alloc_value(ctx, state); + + nix_expr_eval_from_string(ctx, state, "x: x + 1", "", incrementFn); + assert_ctx_ok(); + nix_init_int(ctx, argFive, 5); + + // Create a lazy application: (x: x + 1) 5 + nix_init_apply(ctx, lazyApply, incrementFn, argFive); + assert_ctx_ok(); + + ListBuilder * builder = nix_make_list_builder(ctx, state, 3); + nix_list_builder_insert(ctx, builder, 0, throwingValue); + nix_list_builder_insert(ctx, builder, 1, intValue); + nix_list_builder_insert(ctx, builder, 2, lazyApply); + nix_make_list(ctx, builder, value); + nix_list_builder_free(builder); + + // Test 1: Lazy accessor should return the throwing element without forcing evaluation + nix_value * lazyThrowingElement = nix_get_list_byidx_lazy(ctx, value, state, 0); + assert_ctx_ok(); + ASSERT_NE(nullptr, lazyThrowingElement); + + // Verify the element is still lazy by checking that forcing it throws + nix_value_force(ctx, state, lazyThrowingElement); + assert_ctx_err(); + ASSERT_THAT( + nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("This should not be evaluated by the lazy accessor")); + + // Test 2: Lazy accessor should return the already-evaluated int + nix_value * intElement = nix_get_list_byidx_lazy(ctx, value, state, 1); + assert_ctx_ok(); + ASSERT_NE(nullptr, intElement); + ASSERT_EQ(42, nix_get_int(ctx, intElement)); + + // Test 3: Lazy accessor should return the lazy function application without forcing + nix_value * lazyFunctionElement = nix_get_list_byidx_lazy(ctx, value, state, 2); + assert_ctx_ok(); + ASSERT_NE(nullptr, lazyFunctionElement); + + // Force the lazy function application - should compute 5 + 1 = 6 + nix_value_force(ctx, state, lazyFunctionElement); + assert_ctx_ok(); + ASSERT_EQ(6, nix_get_int(ctx, lazyFunctionElement)); + + // Clean up + nix_gc_decref(ctx, throwingFn); + nix_gc_decref(ctx, throwingValue); + nix_gc_decref(ctx, intValue); + nix_gc_decref(ctx, lazyApply); + nix_gc_decref(ctx, incrementFn); + nix_gc_decref(ctx, argFive); + nix_gc_decref(ctx, lazyThrowingElement); + nix_gc_decref(ctx, intElement); + nix_gc_decref(ctx, lazyFunctionElement); +} + TEST_F(nix_api_expr_test, nix_build_and_init_attr_invalid) { ASSERT_EQ(nullptr, nix_get_attr_byname(ctx, nullptr, state, 0)); @@ -255,6 +352,225 @@ TEST_F(nix_api_expr_test, nix_build_and_init_attr) free(out_name); } +TEST_F(nix_api_expr_test, nix_get_attr_byidx_large_indices) +{ + // Create a small attribute set to test extremely large out-of-bounds access + const char ** out_name = (const char **) malloc(sizeof(char *)); + BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, 2); + nix_value * intValue = nix_alloc_value(ctx, state); + nix_init_int(ctx, intValue, 42); + nix_bindings_builder_insert(ctx, builder, "test", intValue); + nix_make_attrs(ctx, value, builder); + nix_bindings_builder_free(builder); + + // Test extremely large indices that would definitely crash without bounds checking + ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, value, state, 1000000, out_name)); + ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx)); + ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, value, state, UINT_MAX / 2, out_name)); + ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx)); + ASSERT_EQ(nullptr, nix_get_attr_byidx(ctx, value, state, UINT_MAX / 2 + 1000000, out_name)); + ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx)); + + // Test nix_get_attr_name_byidx with large indices too + ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, 1000000)); + ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx)); + ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, UINT_MAX / 2)); + ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx)); + ASSERT_EQ(nullptr, nix_get_attr_name_byidx(ctx, value, state, UINT_MAX / 2 + 1000000)); + ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx)); + + // Clean up + nix_gc_decref(ctx, intValue); + free(out_name); +} + +TEST_F(nix_api_expr_test, nix_get_attr_byname_lazy) +{ + // Create an attribute set with a throwing lazy attribute, an already-evaluated int, and a lazy function call + + // 1. Throwing lazy element - create a function application thunk that will throw when forced + nix_value * throwingFn = nix_alloc_value(ctx, state); + nix_value * throwingValue = nix_alloc_value(ctx, state); + + nix_expr_eval_from_string( + ctx, + state, + R"( + _: throw "This should not be evaluated by the lazy accessor" + )", + "", + throwingFn); + assert_ctx_ok(); + + nix_init_apply(ctx, throwingValue, throwingFn, throwingFn); + assert_ctx_ok(); + + // 2. Already evaluated int (not lazy) + nix_value * intValue = nix_alloc_value(ctx, state); + nix_init_int(ctx, intValue, 42); + assert_ctx_ok(); + + // 3. Lazy function application that would compute increment 7 = 8 + nix_value * lazyApply = nix_alloc_value(ctx, state); + nix_value * incrementFn = nix_alloc_value(ctx, state); + nix_value * argSeven = nix_alloc_value(ctx, state); + + nix_expr_eval_from_string(ctx, state, "x: x + 1", "", incrementFn); + assert_ctx_ok(); + nix_init_int(ctx, argSeven, 7); + + // Create a lazy application: (x: x + 1) 7 + nix_init_apply(ctx, lazyApply, incrementFn, argSeven); + assert_ctx_ok(); + + BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, 3); + nix_bindings_builder_insert(ctx, builder, "throwing", throwingValue); + nix_bindings_builder_insert(ctx, builder, "normal", intValue); + nix_bindings_builder_insert(ctx, builder, "lazy", lazyApply); + nix_make_attrs(ctx, value, builder); + nix_bindings_builder_free(builder); + + // Test 1: Lazy accessor should return the throwing attribute without forcing evaluation + nix_value * lazyThrowingAttr = nix_get_attr_byname_lazy(ctx, value, state, "throwing"); + assert_ctx_ok(); + ASSERT_NE(nullptr, lazyThrowingAttr); + + // Verify the attribute is still lazy by checking that forcing it throws + nix_value_force(ctx, state, lazyThrowingAttr); + assert_ctx_err(); + ASSERT_THAT( + nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("This should not be evaluated by the lazy accessor")); + + // Test 2: Lazy accessor should return the already-evaluated int + nix_value * intAttr = nix_get_attr_byname_lazy(ctx, value, state, "normal"); + assert_ctx_ok(); + ASSERT_NE(nullptr, intAttr); + ASSERT_EQ(42, nix_get_int(ctx, intAttr)); + + // Test 3: Lazy accessor should return the lazy function application without forcing + nix_value * lazyFunctionAttr = nix_get_attr_byname_lazy(ctx, value, state, "lazy"); + assert_ctx_ok(); + ASSERT_NE(nullptr, lazyFunctionAttr); + + // Force the lazy function application - should compute 7 + 1 = 8 + nix_value_force(ctx, state, lazyFunctionAttr); + assert_ctx_ok(); + ASSERT_EQ(8, nix_get_int(ctx, lazyFunctionAttr)); + + // Test 4: Missing attribute should return NULL with NIX_ERR_KEY + nix_value * missingAttr = nix_get_attr_byname_lazy(ctx, value, state, "nonexistent"); + ASSERT_EQ(nullptr, missingAttr); + ASSERT_EQ(NIX_ERR_KEY, nix_err_code(ctx)); + + // Clean up + nix_gc_decref(ctx, throwingFn); + nix_gc_decref(ctx, throwingValue); + nix_gc_decref(ctx, intValue); + nix_gc_decref(ctx, lazyApply); + nix_gc_decref(ctx, incrementFn); + nix_gc_decref(ctx, argSeven); + nix_gc_decref(ctx, lazyThrowingAttr); + nix_gc_decref(ctx, intAttr); + nix_gc_decref(ctx, lazyFunctionAttr); +} + +TEST_F(nix_api_expr_test, nix_get_attr_byidx_lazy) +{ + // Create an attribute set with a throwing lazy attribute, an already-evaluated int, and a lazy function call + + // 1. Throwing lazy element - create a function application thunk that will throw when forced + nix_value * throwingFn = nix_alloc_value(ctx, state); + nix_value * throwingValue = nix_alloc_value(ctx, state); + + nix_expr_eval_from_string( + ctx, + state, + R"( + _: throw "This should not be evaluated by the lazy accessor" + )", + "", + throwingFn); + assert_ctx_ok(); + + nix_init_apply(ctx, throwingValue, throwingFn, throwingFn); + assert_ctx_ok(); + + // 2. Already evaluated int (not lazy) + nix_value * intValue = nix_alloc_value(ctx, state); + nix_init_int(ctx, intValue, 99); + assert_ctx_ok(); + + // 3. Lazy function application that would compute increment 10 = 11 + nix_value * lazyApply = nix_alloc_value(ctx, state); + nix_value * incrementFn = nix_alloc_value(ctx, state); + nix_value * argTen = nix_alloc_value(ctx, state); + + nix_expr_eval_from_string(ctx, state, "x: x + 1", "", incrementFn); + assert_ctx_ok(); + nix_init_int(ctx, argTen, 10); + + // Create a lazy application: (x: x + 1) 10 + nix_init_apply(ctx, lazyApply, incrementFn, argTen); + assert_ctx_ok(); + + BindingsBuilder * builder = nix_make_bindings_builder(ctx, state, 3); + nix_bindings_builder_insert(ctx, builder, "a_throwing", throwingValue); + nix_bindings_builder_insert(ctx, builder, "b_normal", intValue); + nix_bindings_builder_insert(ctx, builder, "c_lazy", lazyApply); + nix_make_attrs(ctx, value, builder); + nix_bindings_builder_free(builder); + + // Proper usage: first get the size and gather all attributes into a map + unsigned int attrCount = nix_get_attrs_size(ctx, value); + assert_ctx_ok(); + ASSERT_EQ(3u, attrCount); + + // Gather all attributes into a map (proper contract usage) + std::map attrMap; + const char * name; + + for (unsigned int i = 0; i < attrCount; i++) { + nix_value * attr = nix_get_attr_byidx_lazy(ctx, value, state, i, &name); + assert_ctx_ok(); + ASSERT_NE(nullptr, attr); + attrMap[std::string(name)] = attr; + } + + // Now test the gathered attributes + ASSERT_EQ(3u, attrMap.size()); + ASSERT_TRUE(attrMap.count("a_throwing")); + ASSERT_TRUE(attrMap.count("b_normal")); + ASSERT_TRUE(attrMap.count("c_lazy")); + + // Test 1: Throwing attribute should be lazy + nix_value * throwingAttr = attrMap["a_throwing"]; + nix_value_force(ctx, state, throwingAttr); + assert_ctx_err(); + ASSERT_THAT( + nix_err_msg(nullptr, ctx, nullptr), testing::HasSubstr("This should not be evaluated by the lazy accessor")); + + // Test 2: Normal attribute should be already evaluated + nix_value * normalAttr = attrMap["b_normal"]; + ASSERT_EQ(99, nix_get_int(ctx, normalAttr)); + + // Test 3: Lazy function should compute when forced + nix_value * lazyAttr = attrMap["c_lazy"]; + nix_value_force(ctx, state, lazyAttr); + assert_ctx_ok(); + ASSERT_EQ(11, nix_get_int(ctx, lazyAttr)); + + // Clean up + nix_gc_decref(ctx, throwingFn); + nix_gc_decref(ctx, throwingValue); + nix_gc_decref(ctx, intValue); + nix_gc_decref(ctx, lazyApply); + nix_gc_decref(ctx, incrementFn); + nix_gc_decref(ctx, argTen); + for (auto & pair : attrMap) { + nix_gc_decref(ctx, pair.second); + } +} + TEST_F(nix_api_expr_test, nix_value_init) { // Setup @@ -320,8 +636,10 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_error) // Evaluate it nix_value_force(ctx, state, v); - ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); - ASSERT_THAT(ctx->last_err.value(), testing::HasSubstr("attempt to call something which is not a function but")); + ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR); + ASSERT_THAT( + nix_err_msg(nullptr, ctx, nullptr), + testing::HasSubstr("attempt to call something which is not a function but")); // Clean up nix_gc_decref(ctx, some_string); @@ -380,7 +698,9 @@ TEST_F(nix_api_expr_test, nix_value_init_apply_lazy_arg) // nix_get_attr_byname isn't lazy (it could have been) so it will throw the exception nix_value * foo = nix_get_attr_byname(ctx, r, state, "foo"); ASSERT_EQ(nullptr, foo); - ASSERT_THAT(ctx->last_err.value(), testing::HasSubstr("error message for test case nix_value_init_apply_lazy_arg")); + ASSERT_THAT( + nix_err_msg(nullptr, ctx, nullptr), + testing::HasSubstr("error message for test case nix_value_init_apply_lazy_arg")); // Clean up nix_gc_decref(ctx, f); diff --git a/src/libexpr-tests/nix_api_value_internal.cc b/src/libexpr-tests/nix_api_value_internal.cc new file mode 100644 index 00000000000..34db6ac81c8 --- /dev/null +++ b/src/libexpr-tests/nix_api_value_internal.cc @@ -0,0 +1,25 @@ +#include "nix_api_store.h" +#include "nix_api_util.h" +#include "nix_api_expr.h" +#include "nix_api_value.h" +#include "nix_api_expr_internal.h" + +#include "nix/expr/tests/nix_api_expr.hh" +#include "nix/util/tests/string_callback.hh" + +#include +#include +#include +#include + +namespace nixC { + +TEST_F(nix_api_expr_test, as_nix_value_ptr) +{ + // nix_alloc_value casts nix::Value to nix_value + // It should be obvious from the decl that that works, but if it doesn't, + // the whole implementation would be utterly broken. + ASSERT_EQ(sizeof(nix::Value), sizeof(nix_value)); +} + +} // namespace nixC diff --git a/src/libexpr-tests/package.nix b/src/libexpr-tests/package.nix index 51d52e935bf..c36aa2dc725 100644 --- a/src/libexpr-tests/package.nix +++ b/src/libexpr-tests/package.nix @@ -62,6 +62,7 @@ mkMesonExecutable (finalAttrs: { mkdir -p "$HOME" '' + '' + export ASAN_OPTIONS=abort_on_error=1:print_summary=1:detect_leaks=0 export _NIX_TEST_UNIT_DATA=${resolvePath ./data} ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out diff --git a/src/libexpr-tests/primops.cc b/src/libexpr-tests/primops.cc index f3f7de8d970..74d676844b7 100644 --- a/src/libexpr-tests/primops.cc +++ b/src/libexpr-tests/primops.cc @@ -195,18 +195,18 @@ TEST_F(PrimOpTest, unsafeGetAttrPos) auto v = eval(expr); ASSERT_THAT(v, IsAttrsOfSize(3)); - auto file = v.attrs()->find(createSymbol("file")); + auto file = v.attrs()->get(createSymbol("file")); ASSERT_NE(file, nullptr); ASSERT_THAT(*file->value, IsString()); auto s = baseNameOf(file->value->string_view()); ASSERT_EQ(s, "foo.nix"); - auto line = v.attrs()->find(createSymbol("line")); + auto line = v.attrs()->get(createSymbol("line")); ASSERT_NE(line, nullptr); state.forceValue(*line->value, noPos); ASSERT_THAT(*line->value, IsIntEq(4)); - auto column = v.attrs()->find(createSymbol("column")); + auto column = v.attrs()->get(createSymbol("column")); ASSERT_NE(column, nullptr); state.forceValue(*column->value, noPos); ASSERT_THAT(*column->value, IsIntEq(3)); @@ -246,7 +246,7 @@ TEST_F(PrimOpTest, removeAttrsRetains) { auto v = eval("builtins.removeAttrs { x = 1; y = 2; } [\"x\"]"); ASSERT_THAT(v, IsAttrsOfSize(1)); - ASSERT_NE(v.attrs()->find(createSymbol("y")), nullptr); + ASSERT_NE(v.attrs()->get(createSymbol("y")), nullptr); } TEST_F(PrimOpTest, listToAttrsEmptyList) @@ -266,7 +266,7 @@ TEST_F(PrimOpTest, listToAttrs) { auto v = eval("builtins.listToAttrs [ { name = \"key\"; value = 123; } ]"); ASSERT_THAT(v, IsAttrsOfSize(1)); - auto key = v.attrs()->find(createSymbol("key")); + auto key = v.attrs()->get(createSymbol("key")); ASSERT_NE(key, nullptr); ASSERT_THAT(*key->value, IsIntEq(123)); } @@ -275,7 +275,7 @@ TEST_F(PrimOpTest, intersectAttrs) { auto v = eval("builtins.intersectAttrs { a = 1; b = 2; } { b = 3; c = 4; }"); ASSERT_THAT(v, IsAttrsOfSize(1)); - auto b = v.attrs()->find(createSymbol("b")); + auto b = v.attrs()->get(createSymbol("b")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsIntEq(3)); } @@ -293,11 +293,11 @@ TEST_F(PrimOpTest, functionArgs) auto v = eval("builtins.functionArgs ({ x, y ? 123}: 1)"); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto x = v.attrs()->find(createSymbol("x")); + auto x = v.attrs()->get(createSymbol("x")); ASSERT_NE(x, nullptr); ASSERT_THAT(*x->value, IsFalse()); - auto y = v.attrs()->find(createSymbol("y")); + auto y = v.attrs()->get(createSymbol("y")); ASSERT_NE(y, nullptr); ASSERT_THAT(*y->value, IsTrue()); } @@ -307,13 +307,13 @@ TEST_F(PrimOpTest, mapAttrs) auto v = eval("builtins.mapAttrs (name: value: value * 10) { a = 1; b = 2; }"); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto a = v.attrs()->find(createSymbol("a")); + auto a = v.attrs()->get(createSymbol("a")); ASSERT_NE(a, nullptr); ASSERT_THAT(*a->value, IsThunk()); state.forceValue(*a->value, noPos); ASSERT_THAT(*a->value, IsIntEq(10)); - auto b = v.attrs()->find(createSymbol("b")); + auto b = v.attrs()->get(createSymbol("b")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsThunk()); state.forceValue(*b->value, noPos); @@ -642,7 +642,7 @@ class ToStringPrimOpTest : public PrimOpTest, TEST_P(ToStringPrimOpTest, toString) { - const auto [input, output] = GetParam(); + const auto & [input, output] = GetParam(); auto v = eval(input); ASSERT_THAT(v, IsStringEq(output)); } @@ -798,7 +798,7 @@ class CompareVersionsPrimOpTest : public PrimOpTest, TEST_P(CompareVersionsPrimOpTest, compareVersions) { - auto [expression, expectation] = GetParam(); + const auto & [expression, expectation] = GetParam(); auto v = eval(expression); ASSERT_THAT(v, IsIntEq(expectation)); } @@ -834,16 +834,16 @@ class ParseDrvNamePrimOpTest TEST_P(ParseDrvNamePrimOpTest, parseDrvName) { - auto [input, expectedName, expectedVersion] = GetParam(); + const auto & [input, expectedName, expectedVersion] = GetParam(); const auto expr = fmt("builtins.parseDrvName \"%1%\"", input); auto v = eval(expr); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto name = v.attrs()->find(createSymbol("name")); + auto name = v.attrs()->get(createSymbol("name")); ASSERT_TRUE(name); ASSERT_THAT(*name->value, IsStringEq(expectedName)); - auto version = v.attrs()->find(createSymbol("version")); + auto version = v.attrs()->get(createSymbol("version")); ASSERT_TRUE(version); ASSERT_THAT(*version->value, IsStringEq(expectedVersion)); } diff --git a/src/libexpr-tests/trivial.cc b/src/libexpr-tests/trivial.cc index 02433234e4c..a287ce4d185 100644 --- a/src/libexpr-tests/trivial.cc +++ b/src/libexpr-tests/trivial.cc @@ -75,11 +75,11 @@ TEST_F(TrivialExpressionTest, updateAttrs) { auto v = eval("{ a = 1; } // { b = 2; a = 3; }"); ASSERT_THAT(v, IsAttrsOfSize(2)); - auto a = v.attrs()->find(createSymbol("a")); + auto a = v.attrs()->get(createSymbol("a")); ASSERT_NE(a, nullptr); ASSERT_THAT(*a->value, IsIntEq(3)); - auto b = v.attrs()->find(createSymbol("b")); + auto b = v.attrs()->get(createSymbol("b")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsIntEq(2)); } @@ -176,7 +176,7 @@ TEST_P(AttrSetMergeTrvialExpressionTest, attrsetMergeLazy) auto v = eval(expr); ASSERT_THAT(v, IsAttrsOfSize(1)); - auto a = v.attrs()->find(createSymbol("a")); + auto a = v.attrs()->get(createSymbol("a")); ASSERT_NE(a, nullptr); ASSERT_THAT(*a->value, IsThunk()); @@ -184,11 +184,11 @@ TEST_P(AttrSetMergeTrvialExpressionTest, attrsetMergeLazy) ASSERT_THAT(*a->value, IsAttrsOfSize(2)); - auto b = a->value->attrs()->find(createSymbol("b")); + auto b = a->value->attrs()->get(createSymbol("b")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsIntEq(1)); - auto c = a->value->attrs()->find(createSymbol("c")); + auto c = a->value->attrs()->get(createSymbol("c")); ASSERT_NE(c, nullptr); ASSERT_THAT(*c->value, IsIntEq(2)); } @@ -330,7 +330,7 @@ TEST_F(TrivialExpressionTest, bindOr) { auto v = eval("{ or = 1; }"); ASSERT_THAT(v, IsAttrsOfSize(1)); - auto b = v.attrs()->find(createSymbol("or")); + auto b = v.attrs()->get(createSymbol("or")); ASSERT_NE(b, nullptr); ASSERT_THAT(*b->value, IsIntEq(1)); } diff --git a/src/libexpr-tests/value/print.cc b/src/libexpr-tests/value/print.cc index 7647cd334d7..6cadbc70ac1 100644 --- a/src/libexpr-tests/value/print.cc +++ b/src/libexpr-tests/value/print.cc @@ -10,7 +10,7 @@ using namespace testing; struct ValuePrintingTests : LibExprTest { template - void test(Value v, std::string_view expected, A... args) + void test(Value & v, std::string_view expected, A... args) { std::stringstream out; v.print(state, out, args...); @@ -35,14 +35,14 @@ TEST_F(ValuePrintingTests, tBool) TEST_F(ValuePrintingTests, tString) { Value vString; - vString.mkString("some-string"); + vString.mkStringNoCopy("some-string"); test(vString, "\"some-string\""); } TEST_F(ValuePrintingTests, tPath) { Value vPath; - vPath.mkString("/foo"); + vPath.mkStringNoCopy("/foo"); test(vPath, "\"/foo\""); } @@ -61,7 +61,7 @@ TEST_F(ValuePrintingTests, tAttrs) Value vTwo; vTwo.mkInt(2); - BindingsBuilder builder(state, state.allocBindings(10)); + BindingsBuilder builder = state.buildBindings(10); builder.insert(state.symbols.create("one"), &vOne); builder.insert(state.symbols.create("two"), &vTwo); @@ -196,11 +196,11 @@ TEST_F(ValuePrintingTests, depthAttrs) Value vTwo; vTwo.mkInt(2); - BindingsBuilder builderEmpty(state, state.allocBindings(0)); + BindingsBuilder builderEmpty = state.buildBindings(0); Value vAttrsEmpty; vAttrsEmpty.mkAttrs(builderEmpty.finish()); - BindingsBuilder builder(state, state.allocBindings(10)); + BindingsBuilder builder = state.buildBindings(10); builder.insert(state.symbols.create("one"), &vOne); builder.insert(state.symbols.create("two"), &vTwo); builder.insert(state.symbols.create("nested"), &vAttrsEmpty); @@ -208,7 +208,7 @@ TEST_F(ValuePrintingTests, depthAttrs) Value vAttrs; vAttrs.mkAttrs(builder.finish()); - BindingsBuilder builder2(state, state.allocBindings(10)); + BindingsBuilder builder2 = state.buildBindings(10); builder2.insert(state.symbols.create("one"), &vOne); builder2.insert(state.symbols.create("two"), &vTwo); builder2.insert(state.symbols.create("nested"), &vAttrs); @@ -233,14 +233,14 @@ TEST_F(ValuePrintingTests, depthList) Value vTwo; vTwo.mkInt(2); - BindingsBuilder builder(state, state.allocBindings(10)); + BindingsBuilder builder = state.buildBindings(10); builder.insert(state.symbols.create("one"), &vOne); builder.insert(state.symbols.create("two"), &vTwo); Value vAttrs; vAttrs.mkAttrs(builder.finish()); - BindingsBuilder builder2(state, state.allocBindings(10)); + BindingsBuilder builder2 = state.buildBindings(10); builder2.insert(state.symbols.create("one"), &vOne); builder2.insert(state.symbols.create("two"), &vTwo); builder2.insert(state.symbols.create("nested"), &vAttrs); @@ -290,12 +290,12 @@ TEST_F(StringPrintingTests, maxLengthTruncation) TEST_F(ValuePrintingTests, attrsTypeFirst) { Value vType; - vType.mkString("puppy"); + vType.mkStringNoCopy("puppy"); Value vApple; - vApple.mkString("apple"); + vApple.mkStringNoCopy("apple"); - BindingsBuilder builder(state, state.allocBindings(10)); + BindingsBuilder builder = state.buildBindings(10); builder.insert(state.symbols.create("type"), &vType); builder.insert(state.symbols.create("apple"), &vApple); @@ -334,7 +334,7 @@ TEST_F(ValuePrintingTests, ansiColorsBool) TEST_F(ValuePrintingTests, ansiColorsString) { Value v; - v.mkString("puppy"); + v.mkStringNoCopy("puppy"); test(v, ANSI_MAGENTA "\"puppy\"" ANSI_NORMAL, PrintOptions{.ansiColors = true}); } @@ -342,7 +342,7 @@ TEST_F(ValuePrintingTests, ansiColorsString) TEST_F(ValuePrintingTests, ansiColorsStringElided) { Value v; - v.mkString("puppy"); + v.mkStringNoCopy("puppy"); test( v, @@ -374,7 +374,7 @@ TEST_F(ValuePrintingTests, ansiColorsAttrs) Value vTwo; vTwo.mkInt(2); - BindingsBuilder builder(state, state.allocBindings(10)); + BindingsBuilder builder = state.buildBindings(10); builder.insert(state.symbols.create("one"), &vOne); builder.insert(state.symbols.create("two"), &vTwo); @@ -390,10 +390,10 @@ TEST_F(ValuePrintingTests, ansiColorsAttrs) TEST_F(ValuePrintingTests, ansiColorsDerivation) { Value vDerivation; - vDerivation.mkString("derivation"); + vDerivation.mkStringNoCopy("derivation"); - BindingsBuilder builder(state, state.allocBindings(10)); - builder.insert(state.sType, &vDerivation); + BindingsBuilder builder = state.buildBindings(10); + builder.insert(state.s.type, &vDerivation); Value vAttrs; vAttrs.mkAttrs(builder.finish()); @@ -413,7 +413,7 @@ TEST_F(ValuePrintingTests, ansiColorsError) { Value throw_ = state.getBuiltin("throw"); Value message; - message.mkString("uh oh!"); + message.mkStringNoCopy("uh oh!"); Value vError; vError.mkApp(&throw_, &message); @@ -430,16 +430,16 @@ TEST_F(ValuePrintingTests, ansiColorsDerivationError) { Value throw_ = state.getBuiltin("throw"); Value message; - message.mkString("uh oh!"); + message.mkStringNoCopy("uh oh!"); Value vError; vError.mkApp(&throw_, &message); Value vDerivation; - vDerivation.mkString("derivation"); + vDerivation.mkStringNoCopy("derivation"); - BindingsBuilder builder(state, state.allocBindings(10)); - builder.insert(state.sType, &vDerivation); - builder.insert(state.sDrvPath, &vError); + BindingsBuilder builder = state.buildBindings(10); + builder.insert(state.s.type, &vDerivation); + builder.insert(state.s.drvPath, &vError); Value vAttrs; vAttrs.mkAttrs(builder.finish()); @@ -553,12 +553,12 @@ TEST_F(ValuePrintingTests, ansiColorsBlackhole) TEST_F(ValuePrintingTests, ansiColorsAttrsRepeated) { - BindingsBuilder emptyBuilder(state, state.allocBindings(1)); + BindingsBuilder emptyBuilder = state.buildBindings(1); Value vEmpty; vEmpty.mkAttrs(emptyBuilder.finish()); - BindingsBuilder builder(state, state.allocBindings(10)); + BindingsBuilder builder = state.buildBindings(10); builder.insert(state.symbols.create("a"), &vEmpty); builder.insert(state.symbols.create("b"), &vEmpty); @@ -570,7 +570,7 @@ TEST_F(ValuePrintingTests, ansiColorsAttrsRepeated) TEST_F(ValuePrintingTests, ansiColorsListRepeated) { - BindingsBuilder emptyBuilder(state, state.allocBindings(1)); + BindingsBuilder emptyBuilder = state.buildBindings(1); Value vEmpty; vEmpty.mkAttrs(emptyBuilder.finish()); @@ -586,7 +586,7 @@ TEST_F(ValuePrintingTests, ansiColorsListRepeated) TEST_F(ValuePrintingTests, listRepeated) { - BindingsBuilder emptyBuilder(state, state.allocBindings(1)); + BindingsBuilder emptyBuilder = state.buildBindings(1); Value vEmpty; vEmpty.mkAttrs(emptyBuilder.finish()); @@ -609,7 +609,7 @@ TEST_F(ValuePrintingTests, ansiColorsAttrsElided) Value vTwo; vTwo.mkInt(2); - BindingsBuilder builder(state, state.allocBindings(10)); + BindingsBuilder builder = state.buildBindings(10); builder.insert(state.symbols.create("one"), &vOne); builder.insert(state.symbols.create("two"), &vTwo); @@ -625,18 +625,17 @@ TEST_F(ValuePrintingTests, ansiColorsAttrsElided) vThree.mkInt(3); builder.insert(state.symbols.create("three"), &vThree); - vAttrs.mkAttrs(builder.finish()); + Value vAttrs2; + vAttrs2.mkAttrs(builder.finish()); test( - vAttrs, + vAttrs2, "{ one = " ANSI_CYAN "1" ANSI_NORMAL "; " ANSI_FAINT "«2 attributes elided»" ANSI_NORMAL " }", PrintOptions{.ansiColors = true, .maxAttrs = 1}); } TEST_F(ValuePrintingTests, ansiColorsListElided) { - BindingsBuilder emptyBuilder(state, state.allocBindings(1)); - Value vOne; vOne.mkInt(1); diff --git a/src/libexpr-tests/value/value.cc b/src/libexpr-tests/value/value.cc index 63501dd4995..c6349436fb7 100644 --- a/src/libexpr-tests/value/value.cc +++ b/src/libexpr-tests/value/value.cc @@ -11,7 +11,6 @@ TEST_F(ValueTest, unsetValue) { Value unsetValue; ASSERT_EQ(false, unsetValue.isValid()); - ASSERT_EQ(nThunk, unsetValue.type(true)); ASSERT_DEATH(unsetValue.type(), ""); } diff --git a/src/libexpr/attr-path.cc b/src/libexpr/attr-path.cc index b02b08db4ee..58705bfa1bd 100644 --- a/src/libexpr/attr-path.cc +++ b/src/libexpr/attr-path.cc @@ -110,8 +110,8 @@ std::pair findPackageFilename(EvalState & state, Value & v { Value * v2; try { - auto dummyArgs = state.allocBindings(0); - v2 = findAlongAttrPath(state, "meta.position", *dummyArgs, v).first; + auto & dummyArgs = Bindings::emptyBindings; + v2 = findAlongAttrPath(state, "meta.position", dummyArgs, v).first; } catch (Error &) { throw NoPositionInfo("package '%s' has no source location information", what); } diff --git a/src/libexpr/attr-set.cc b/src/libexpr/attr-set.cc index 3a06441e981..92b67f6ad25 100644 --- a/src/libexpr/attr-set.cc +++ b/src/libexpr/attr-set.cc @@ -5,36 +5,37 @@ namespace nix { +Bindings Bindings::emptyBindings; + /* Allocate a new array of attributes for an attribute set with a specific capacity. The space is implicitly reserved after the Bindings structure. */ -Bindings * EvalState::allocBindings(size_t capacity) +Bindings * EvalMemory::allocBindings(size_t capacity) { if (capacity == 0) - return &emptyBindings; - if (capacity > std::numeric_limits::max()) + return &Bindings::emptyBindings; + if (capacity > std::numeric_limits::max()) throw Error("attribute set of size %d is too big", capacity); - nrAttrsets++; - nrAttrsInAttrsets += capacity; - return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings((Bindings::size_t) capacity); + stats.nrAttrsets++; + stats.nrAttrsInAttrsets += capacity; + return new (allocBytes(sizeof(Bindings) + sizeof(Attr) * capacity)) Bindings(); } Value & BindingsBuilder::alloc(Symbol name, PosIdx pos) { - auto value = state.allocValue(); + auto value = mem.get().allocValue(); bindings->push_back(Attr(name, value, pos)); return *value; } Value & BindingsBuilder::alloc(std::string_view name, PosIdx pos) { - return alloc(state.symbols.create(name), pos); + return alloc(symbols.get().create(name), pos); } void Bindings::sort() { - if (size_) - std::sort(begin(), end()); + std::sort(attrs, attrs + numAttrs); } Value & Value::mkAttrs(BindingsBuilder & bindings) diff --git a/src/libexpr/eval-cache.cc b/src/libexpr/eval-cache.cc index 292d76e025d..46a4bdaeacd 100644 --- a/src/libexpr/eval-cache.cc +++ b/src/libexpr/eval-cache.cc @@ -330,7 +330,7 @@ AttrCursor::AttrCursor( AttrKey AttrCursor::getKey() { if (!parent) - return {0, root->state.sEpsilon}; + return {0, root->state.s.epsilon}; if (!parent->first->cachedValue) { parent->first->cachedValue = root->db->getAttr(parent->first->getKey()); assert(parent->first->cachedValue); @@ -552,16 +552,17 @@ string_t AttrCursor::getStringWithContext() if (auto s = std::get_if(&cachedValue->second)) { bool valid = true; for (auto & c : s->second) { - const StorePath & path = std::visit( + const StorePath * path = std::visit( overloaded{ - [&](const NixStringContextElem::DrvDeep & d) -> const StorePath & { return d.drvPath; }, - [&](const NixStringContextElem::Built & b) -> const StorePath & { - return b.drvPath->getBaseStorePath(); + [&](const NixStringContextElem::DrvDeep & d) -> const StorePath * { return &d.drvPath; }, + [&](const NixStringContextElem::Built & b) -> const StorePath * { + return &b.drvPath->getBaseStorePath(); }, - [&](const NixStringContextElem::Opaque & o) -> const StorePath & { return o.path; }, + [&](const NixStringContextElem::Opaque & o) -> const StorePath * { return &o.path; }, + [&](const NixStringContextElem::Path & p) -> const StorePath * { return nullptr; }, }, c.raw); - if (!root->state.store->isValidPath(path)) { + if (!path || !root->state.store->isValidPath(*path)) { valid = false; break; } @@ -702,13 +703,14 @@ bool AttrCursor::isDerivation() StorePath AttrCursor::forceDerivation() { - auto aDrvPath = getAttr(root->state.sDrvPath); + auto aDrvPath = getAttr(root->state.s.drvPath); auto drvPath = root->state.store->parseStorePath(aDrvPath->getString()); drvPath.requireDerivation(); if (!root->state.store->isValidPath(drvPath) && !settings.readOnlyMode) { /* The eval cache contains 'drvPath', but the actual path has been garbage-collected. So force it to be regenerated. */ aDrvPath->forceValue(); + root->state.waitForPath(drvPath); if (!root->state.store->isValidPath(drvPath)) throw Error( "don't know how to recreate store derivation '%s'!", root->state.store->printStorePath(drvPath)); diff --git a/src/libexpr/eval-gc.cc b/src/libexpr/eval-gc.cc index b17336a901a..c1e974e053b 100644 --- a/src/libexpr/eval-gc.cc +++ b/src/libexpr/eval-gc.cc @@ -16,6 +16,7 @@ # endif # include +# include // For GC_GRANULE_BYTES # include # include @@ -26,6 +27,18 @@ namespace nix { #if NIX_USE_BOEHMGC + +/* + * Ensure that Boehm satisfies our alignment requirements. This is the default configuration [^] + * and this assertion should never break for any platform. Let's assert it just in case. + * + * This alignment is particularly useful to be able to use aligned + * load/store instructions for loading/writing Values. + * + * [^]: https://github.com/bdwgc/bdwgc/blob/54ac18ccbc5a833dd7edaff94a10ab9b65044d61/include/gc/gc_tiny_fl.h#L31-L33 + */ +static_assert(sizeof(void *) * 2 == GC_GRANULE_BYTES, "Boehm GC must use GC_GRANULE_WORDS = 2"); + /* Called when the Boehm GC runs out of memory. */ static void * oomHandler(size_t requested) { @@ -33,6 +46,88 @@ static void * oomHandler(size_t requested) throw std::bad_alloc(); } +static size_t getFreeMem() +{ + /* On Linux, use the `MemAvailable` or `MemFree` fields from + /proc/cpuinfo. */ +# ifdef __linux__ + { + std::unordered_map fields; + for (auto & line : + tokenizeString>(readFile(std::filesystem::path("/proc/meminfo")), "\n")) { + auto colon = line.find(':'); + if (colon == line.npos) + continue; + fields.emplace(line.substr(0, colon), trim(line.substr(colon + 1))); + } + + auto i = fields.find("MemAvailable"); + if (i == fields.end()) + i = fields.find("MemFree"); + if (i != fields.end()) { + auto kb = tokenizeString>(i->second, " "); + if (kb.size() == 2 && kb[1] == "kB") + return string2Int(kb[0]).value_or(0) * 1024; + } + } +# endif + + /* On non-Linux systems, conservatively assume that 25% of memory is free. */ + long pageSize = sysconf(_SC_PAGESIZE); + long pages = sysconf(_SC_PHYS_PAGES); + if (pageSize > 0 && pages > 0) + return (static_cast(pageSize) * static_cast(pages)) / 4; + return 0; +} + +/** + * When a thread goes into a coroutine, we lose its original sp until + * control flow returns to the thread. This causes Boehm GC to crash + * since it will scan memory between the coroutine's sp and the + * original stack base of the thread. Therefore, we detect when the + * current sp is outside of the original thread stack and push the + * entire thread stack instead, as an approximation. + * + * This is not optimal, because it causes the stack below sp to be + * scanned. However, we usually we don't have active coroutines during + * evaluation, so this is acceptable. + * + * Note that we don't scan coroutine stacks. It's currently assumed + * that we don't have GC roots in coroutines. + */ +void fixupBoehmStackPointer(void ** sp_ptr, void * _pthread_id) +{ + void *& sp = *sp_ptr; + auto pthread_id = reinterpret_cast(_pthread_id); + size_t osStackSize; + char * osStackHi; + char * osStackLo; + +# ifdef __APPLE__ + osStackSize = pthread_get_stacksize_np(pthread_id); + osStackHi = (char *) pthread_get_stackaddr_np(pthread_id); + osStackLo = osStackHi - osStackSize; +# else + pthread_attr_t pattr; + if (pthread_attr_init(&pattr)) + throw Error("fixupBoehmStackPointer: pthread_attr_init failed"); +# ifdef HAVE_PTHREAD_GETATTR_NP + if (pthread_getattr_np(pthread_id, &pattr)) + throw Error("fixupBoehmStackPointer: pthread_getattr_np failed"); +# else +# error "Need `pthread_attr_get_np`" +# endif + if (pthread_attr_getstack(&pattr, (void **) &osStackLo, &osStackSize)) + throw Error("fixupBoehmStackPointer: pthread_attr_getstack failed"); + if (pthread_attr_destroy(&pattr)) + throw Error("fixupBoehmStackPointer: pthread_attr_destroy failed"); + osStackHi = osStackLo + osStackSize; +# endif + + if (sp >= osStackHi || sp < osStackLo) // sp is outside the os stack + sp = osStackLo; +} + static inline void initGCReal() { /* Initialise the Boehm garbage collector. */ @@ -63,8 +158,11 @@ static inline void initGCReal() GC_set_oom_fn(oomHandler); - /* Set the initial heap size to something fairly big (25% of - physical RAM, up to a maximum of 384 MiB) so that in most cases + GC_set_sp_corrector(&fixupBoehmStackPointer); + assert(GC_get_sp_corrector()); + + /* Set the initial heap size to something fairly big (80% of + free RAM, up to a maximum of 4 GiB) so that in most cases we don't need to garbage collect at all. (Collection has a fairly significant overhead.) The heap size can be overridden through libgc's GC_INITIAL_HEAP_SIZE environment variable. We @@ -75,15 +173,10 @@ static inline void initGCReal() if (!getEnv("GC_INITIAL_HEAP_SIZE")) { size_t size = 32 * 1024 * 1024; # if HAVE_SYSCONF && defined(_SC_PAGESIZE) && defined(_SC_PHYS_PAGES) - size_t maxSize = 384 * 1024 * 1024; - long pageSize = sysconf(_SC_PAGESIZE); - long pages = sysconf(_SC_PHYS_PAGES); - if (pageSize != -1) - size = (pageSize * pages) / 4; // 25% of RAM - if (size > maxSize) - size = maxSize; + size_t maxSize = 4ULL * 1024 * 1024 * 1024; + auto free = getFreeMem(); + size = std::max(size, std::min((size_t) (free * 0.5), maxSize)); # endif - debug("setting initial heap size to %1% bytes", size); GC_expand_hp(size); } } diff --git a/src/libexpr/eval-profiler.cc b/src/libexpr/eval-profiler.cc index 7769d47d59e..e9dc1e021ea 100644 --- a/src/libexpr/eval-profiler.cc +++ b/src/libexpr/eval-profiler.cc @@ -185,7 +185,7 @@ FrameInfo SampleStack::getPrimOpFrameInfo(const PrimOp & primOp, std::spanattrs(); - auto nameAttr = state.getAttr(state.sName, attrs, ""); + auto nameAttr = state.getAttr(state.s.name, attrs, ""); auto drvName = std::string(state.forceStringNoCtx(*nameAttr->value, pos, "")); return DerivationStrictFrameInfo{.callPos = pos, .drvName = std::move(drvName)}; } catch (...) { @@ -211,7 +211,7 @@ FrameInfo SampleStack::getFrameInfoFromValueAndPos(const Value & v, std::spanget(state.sFunctor); + const auto functor = v.attrs()->get(state.s.functor); if (auto pos_ = posCache.lookup(pos); std::holds_alternative(pos_.origin)) /* HACK: In case callsite position is unresolved. */ return FunctorFrameInfo{.pos = functor->pos}; @@ -324,7 +324,7 @@ void SampleStack::saveProfile() std::visit([&](auto && info) { info.symbolize(state, os, posCache); }, pos); } os << " " << count; - writeLine(profileFd.get(), std::move(os).str()); + writeLine(profileFd.get(), os.str()); /* Clear ostringstream. */ os.str(""); os.clear(); diff --git a/src/libexpr/eval-settings.cc b/src/libexpr/eval-settings.cc index 93db5aebbdc..c9e271b952f 100644 --- a/src/libexpr/eval-settings.cc +++ b/src/libexpr/eval-settings.cc @@ -91,9 +91,19 @@ bool EvalSettings::isPseudoUrl(std::string_view s) std::string EvalSettings::resolvePseudoUrl(std::string_view url) { - if (hasPrefix(url, "channel:")) - return "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; - else + if (hasPrefix(url, "channel:")) { + auto realUrl = "https://nixos.org/channels/" + std::string(url.substr(8)) + "/nixexprs.tar.xz"; + static bool haveWarned = false; + warnOnce( + haveWarned, + "Channels are deprecated in favor of flakes in Determinate Nix. " + "Instead of '%s', use '%s'. " + "See https://zero-to-nix.com for a guide to Nix flakes. " + "For details and to offer feedback on the deprecation process, see: https://github.com/DeterminateSystems/nix-src/issues/34.", + url, + realUrl); + return realUrl; + } else return std::string(url); } @@ -108,4 +118,4 @@ Path getNixDefExpr() return settings.useXDGBaseDirectories ? getStateDir() + "/defexpr" : getHome() + "/.nix-defexpr"; } -} // namespace nix \ No newline at end of file +} // namespace nix diff --git a/src/libexpr/eval.cc b/src/libexpr/eval.cc index f0b19994661..cf29e189567 100644 --- a/src/libexpr/eval.cc +++ b/src/libexpr/eval.cc @@ -17,11 +17,15 @@ #include "nix/expr/print.hh" #include "nix/fetchers/filtering-source-accessor.hh" #include "nix/util/memory-source-accessor.hh" +#include "nix/util/mounted-source-accessor.hh" #include "nix/expr/gc-small-vector.hh" #include "nix/util/url.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/tarball.hh" #include "nix/fetchers/input-cache.hh" +#include "nix/util/current-process.hh" +#include "nix/store/async-path-writer.hh" +#include "nix/expr/parallel-eval.hh" #include "parser-tab.hh" @@ -37,6 +41,8 @@ #include #include +#include +#include #ifndef _WIN32 // TODO use portable implementation # include @@ -127,6 +133,8 @@ std::string_view showType(ValueType type, bool withArticle) return WA("a", "float"); case nThunk: return WA("a", "thunk"); + case nFailed: + return WA("a", "failure"); } unreachable(); } @@ -173,12 +181,25 @@ PosIdx Value::determinePos(const PosIdx pos) const #pragma GCC diagnostic pop } -bool Value::isTrivial() const +template<> +bool ValueStorage::isTrivial() const { - return !isa() - && (!isa() - || (dynamic_cast(thunk().expr) && ((ExprAttrs *) thunk().expr)->dynamicAttrs.empty()) - || dynamic_cast(thunk().expr) || dynamic_cast(thunk().expr)); + auto p1_ = p1; // must acquire before reading p0, since thunks can change + auto p0_ = p0.load(std::memory_order_acquire); + + auto pd = static_cast(p0_ & discriminatorMask); + + if (pd == pdThunk || pd == pdPending || pd == pdAwaited) { + bool isApp = p1_ & discriminatorMask; + if (isApp) + return false; + auto expr = untagPointer(p1_); + return (dynamic_cast(expr) && ((ExprAttrs *) expr)->dynamicAttrs.empty()) + || dynamic_cast(expr) || dynamic_cast(expr); + } + + else + return true; } static Symbol getName(const AttrName & name, EvalState & state, Env & env) @@ -195,6 +216,15 @@ static Symbol getName(const AttrName & name, EvalState & state, Env & env) static constexpr size_t BASE_ENV_SIZE = 128; +EvalMemory::EvalMemory() +#if NIX_USE_BOEHMGC + : valueAllocCache(std::allocate_shared(traceable_allocator(), nullptr)) + , env1AllocCache(std::allocate_shared(traceable_allocator(), nullptr)) +#endif +{ + assertGCInitialized(); +} + EvalState::EvalState( const LookupPath & lookupPathFromArguments, ref store, @@ -203,158 +233,87 @@ EvalState::EvalState( std::shared_ptr buildStore) : fetchSettings{fetchSettings} , settings{settings} - , sWith(symbols.create("")) - , sOutPath(symbols.create("outPath")) - , sDrvPath(symbols.create("drvPath")) - , sType(symbols.create("type")) - , sMeta(symbols.create("meta")) - , sName(symbols.create("name")) - , sValue(symbols.create("value")) - , sSystem(symbols.create("system")) - , sOverrides(symbols.create("__overrides")) - , sOutputs(symbols.create("outputs")) - , sOutputName(symbols.create("outputName")) - , sIgnoreNulls(symbols.create("__ignoreNulls")) - , sFile(symbols.create("file")) - , sLine(symbols.create("line")) - , sColumn(symbols.create("column")) - , sFunctor(symbols.create("__functor")) - , sToString(symbols.create("__toString")) - , sRight(symbols.create("right")) - , sWrong(symbols.create("wrong")) - , sStructuredAttrs(symbols.create("__structuredAttrs")) - , sJson(symbols.create("__json")) - , sAllowedReferences(symbols.create("allowedReferences")) - , sAllowedRequisites(symbols.create("allowedRequisites")) - , sDisallowedReferences(symbols.create("disallowedReferences")) - , sDisallowedRequisites(symbols.create("disallowedRequisites")) - , sMaxSize(symbols.create("maxSize")) - , sMaxClosureSize(symbols.create("maxClosureSize")) - , sBuilder(symbols.create("builder")) - , sArgs(symbols.create("args")) - , sContentAddressed(symbols.create("__contentAddressed")) - , sImpure(symbols.create("__impure")) - , sOutputHash(symbols.create("outputHash")) - , sOutputHashAlgo(symbols.create("outputHashAlgo")) - , sOutputHashMode(symbols.create("outputHashMode")) - , sRecurseForDerivations(symbols.create("recurseForDerivations")) - , sDescription(symbols.create("description")) - , sSelf(symbols.create("self")) - , sEpsilon(symbols.create("")) - , sStartSet(symbols.create("startSet")) - , sOperator(symbols.create("operator")) - , sKey(symbols.create("key")) - , sPath(symbols.create("path")) - , sPrefix(symbols.create("prefix")) - , sOutputSpecified(symbols.create("outputSpecified")) - , exprSymbols{ - .sub = symbols.create("__sub"), - .lessThan = symbols.create("__lessThan"), - .mul = symbols.create("__mul"), - .div = symbols.create("__div"), - .or_ = symbols.create("or"), - .findFile = symbols.create("__findFile"), - .nixPath = symbols.create("__nixPath"), - .body = symbols.create("body"), - } + , symbols(StaticEvalSymbols::staticSymbolTable()) , repair(NoRepair) - , emptyBindings(0) - , storeFS( - makeMountedSourceAccessor( - { - {CanonPath::root, makeEmptySourceAccessor()}, - /* In the pure eval case, we can simply require - valid paths. However, in the *impure* eval - case this gets in the way of the union - mechanism, because an invalid access in the - upper layer will *not* be caught by the union - source accessor, but instead abort the entire - lookup. - - This happens when the store dir in the - ambient file system has a path (e.g. because - another Nix store there), but the relocated - store does not. - - TODO make the various source accessors doing - access control all throw the same type of - exception, and make union source accessor - catch it, so we don't need to do this hack. - */ - {CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)}, - })) - , rootFS( - ({ - /* In pure eval mode, we provide a filesystem that only - contains the Nix store. - - If we have a chroot store and pure eval is not enabled, - use a union accessor to make the chroot store available - at its logical location while still having the - underlying directory available. This is necessary for - instance if we're evaluating a file from the physical - /nix/store while using a chroot store. */ - auto accessor = getFSSourceAccessor(); - - auto realStoreDir = dirOf(store->toRealPath(StorePath::dummy)); - if (settings.pureEval || store->storeDir != realStoreDir) { - accessor = settings.pureEval - ? storeFS - : makeUnionSourceAccessor({accessor, storeFS}); - } - - /* Apply access control if needed. */ - if (settings.restrictEval || settings.pureEval) - accessor = AllowListSourceAccessor::create(accessor, {}, {}, - [&settings](const CanonPath & path) -> RestrictedPathError { - auto modeInformation = settings.pureEval - ? "in pure evaluation mode (use '--impure' to override)" - : "in restricted mode"; - throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); - }); - - accessor; - })) + , storeFS(makeMountedSourceAccessor({ + {CanonPath::root, makeEmptySourceAccessor()}, + /* In the pure eval case, we can simply require + valid paths. However, in the *impure* eval + case this gets in the way of the union + mechanism, because an invalid access in the + upper layer will *not* be caught by the union + source accessor, but instead abort the entire + lookup. + + This happens when the store dir in the + ambient file system has a path (e.g. because + another Nix store there), but the relocated + store does not. + + TODO make the various source accessors doing + access control all throw the same type of + exception, and make union source accessor + catch it, so we don't need to do this hack. + */ + {CanonPath(store->storeDir), store->getFSAccessor(settings.pureEval)}, + })) + , rootFS([&] { + /* In pure eval mode, we provide a filesystem that only + contains the Nix store. + + Otherwise, use a union accessor to make the augmented store + available at its logical location while still having the + underlying directory available. This is necessary for + instance if we're evaluating a file from the physical + /nix/store while using a chroot store, and also for lazy + mounted fetchTree. */ + auto accessor = settings.pureEval ? storeFS.cast() + : makeUnionSourceAccessor({getFSSourceAccessor(), storeFS}); + + /* Apply access control if needed. */ + if (settings.restrictEval || settings.pureEval) + accessor = AllowListSourceAccessor::create( + accessor, {}, {}, [&settings](const CanonPath & path) -> RestrictedPathError { + auto modeInformation = settings.pureEval ? "in pure evaluation mode (use '--impure' to override)" + : "in restricted mode"; + throw RestrictedPathError("access to absolute path '%1%' is forbidden %2%", path, modeInformation); + }); + + return accessor; + }()) , corepkgsFS(make_ref()) , internalFS(make_ref()) - , derivationInternal{corepkgsFS->addFile( - CanonPath("derivation-internal.nix"), + , derivationInternal{internalFS->addFile( + CanonPath("derivation-internal.nix"), #include "primops/derivation.nix.gen.hh" - )} + )} , store(store) , buildStore(buildStore ? buildStore : store) , inputCache(fetchers::InputCache::create()) , debugRepl(nullptr) , debugStop(false) , trylevel(0) + , asyncPathWriter(AsyncPathWriter::make(store)) + , srcToStore(make_ref()) + , importResolutionCache(make_ref()) + , fileEvalCache(make_ref()) , regexCache(makeRegexCache()) #if NIX_USE_BOEHMGC - , valueAllocCache(std::allocate_shared(traceable_allocator(), nullptr)) - , env1AllocCache(std::allocate_shared(traceable_allocator(), nullptr)) - , baseEnvP(std::allocate_shared(traceable_allocator(), &allocEnv(BASE_ENV_SIZE))) + , baseEnvP(std::allocate_shared(traceable_allocator(), &mem.allocEnv(BASE_ENV_SIZE))) , baseEnv(**baseEnvP) #else - , baseEnv(allocEnv(BASE_ENV_SIZE)) + , baseEnv(mem.allocEnv(BASE_ENV_SIZE)) #endif , staticBaseEnv{std::make_shared(nullptr, nullptr)} + , executor{make_ref(settings)} { corepkgsFS->setPathDisplay(""); internalFS->setPathDisplay("«nix-internal»", ""); countCalls = getEnv("NIX_COUNT_CALLS").value_or("0") != "0"; - assertGCInitialized(); - static_assert(sizeof(Env) <= 16, "environment must be <= 16 bytes"); - - vEmptyList.mkList(buildList(0)); - vNull.mkNull(); - vTrue.mkBool(true); - vFalse.mkBool(false); - vStringRegular.mkString("regular"); - vStringDirectory.mkString("directory"); - vStringSymlink.mkString("symlink"); - vStringUnknown.mkString("unknown"); + static_assert(sizeof(Counter) == 64, "counters must be 64 bytes"); /* Construct the Nix expression search path. */ assert(lookupPath.elements.empty()); @@ -401,7 +360,7 @@ EvalState::EvalState( EvalState::~EvalState() {} -void EvalState::allowPath(const Path & path) +void EvalState::allowPathLegacy(const Path & path) { if (auto rootFS2 = rootFS.dynamic_pointer_cast()) rootFS2->allowPrefix(CanonPath(path)); @@ -487,7 +446,8 @@ void EvalState::checkURI(const std::string & uri) Value * EvalState::addConstant(const std::string & name, Value & v, Constant info) { Value * v2 = allocValue(); - *v2 = v; + // Do a raw copy since `operator =` barfs on thunks. + memcpy((char *) v2, (char *) &v, sizeof(Value)); addConstant(name, v2, info); return v2; } @@ -503,8 +463,10 @@ void EvalState::addConstant(const std::string & name, Value * v, Constant info) We might know the type of a thunk in advance, so be allowed to just write it down in that case. */ - if (auto gotType = v->type(true); gotType != nThunk) - assert(info.type == gotType); + if (v->isFinished()) { + if (auto gotType = v->type(); gotType != nThunk) + assert(info.type == gotType); + } /* Install value the base environment. */ staticBaseEnv->vars.emplace_back(symbols.create(name), baseEnvDispl); @@ -649,12 +611,12 @@ std::optional EvalState::getDoc(Value & v) .name = name, .arity = 0, // FIXME: figure out how deep by syntax only? It's not semantically useful though... .args = {}, - .doc = makeImmutableString(toView(s)), // NOTE: memory leak when compiled without GC + .doc = makeImmutableString(s.view()), // NOTE: memory leak when compiled without GC }; } if (isFunctor(v)) { try { - Value & functor = *v.attrs()->find(sFunctor)->value; + Value & functor = *v.attrs()->get(s.functor)->value; Value * vp[] = {&v}; Value partiallyApplied; // The first parameter is not user-provided, and may be @@ -686,7 +648,7 @@ void printStaticEnvBindings(const SymbolTable & st, const StaticEnv & se) // just for the current level of Env, not the whole chain. void printWithBindings(const SymbolTable & st, const Env & env) { - if (!env.values[0]->isThunk()) { + if (env.values[0]->isFinished()) { std::cout << "with: "; std::cout << ANSI_MAGENTA; auto j = env.values[0]->attrs()->begin(); @@ -741,7 +703,7 @@ void mapStaticEnvBindings(const SymbolTable & st, const StaticEnv & se, const En if (env.up && se.up) { mapStaticEnvBindings(st, *se.up, *env.up, vm); - if (se.isWith && !env.values[0]->isThunk()) { + if (se.isWith && env.values[0]->isFinished()) { // add 'with' bindings. for (auto & j : *env.values[0]->attrs()) vm.insert_or_assign(std::string(st[j.name]), j.value); @@ -883,7 +845,7 @@ DebugTraceStacker::DebugTraceStacker(EvalState & evalState, DebugTrace t) void Value::mkString(std::string_view s) { - mkString(makeImmutableString(s)); + mkStringNoCopy(makeImmutableString(s)); } static const char ** encodeContext(const NixStringContext & context) @@ -902,12 +864,12 @@ static const char ** encodeContext(const NixStringContext & context) void Value::mkString(std::string_view s, const NixStringContext & context) { - mkString(makeImmutableString(s), encodeContext(context)); + mkStringNoCopy(makeImmutableString(s), encodeContext(context)); } void Value::mkStringMove(const char * s, const NixStringContext & context) { - mkString(s, encodeContext(context)); + mkStringNoCopy(s, encodeContext(context)); } void Value::mkPath(const SourcePath & path) @@ -948,19 +910,18 @@ inline Value * EvalState::lookupVar(Env * env, const ExprVar & var, bool noEval) } } -ListBuilder::ListBuilder(EvalState & state, size_t size) +ListBuilder::ListBuilder(size_t size) : size(size) , elems(size <= 2 ? inlineElems : (Value **) allocBytes(size * sizeof(Value *))) { - state.nrListElems += size; } Value * EvalState::getBool(bool b) { - return b ? &vTrue : &vFalse; + return b ? &Value::vTrue : &Value::vFalse; } -unsigned long nrThunks = 0; +static Counter nrThunks; static inline void mkThunk(Value & v, Env & env, Expr * expr) { @@ -978,8 +939,14 @@ void EvalState::mkPos(Value & v, PosIdx p) auto origin = positions.originOf(p); if (auto path = std::get_if(&origin)) { auto attrs = buildBindings(3); - attrs.alloc(sFile).mkString(path->path.abs()); - makePositionThunks(*this, p, attrs.alloc(sLine), attrs.alloc(sColumn)); + if (path->accessor == rootFS && store->isInStore(path->path.abs())) + // FIXME: only do this for virtual store paths? + attrs.alloc(s.file).mkString( + path->path.abs(), + {NixStringContextElem::Path{.storePath = store->toStorePath(path->path.abs()).first}}); + else + attrs.alloc(s.file).mkString(path->path.abs()); + makePositionThunks(*this, p, attrs.alloc(s.line), attrs.alloc(s.column)); v.mkAttrs(attrs); } else v.mkNull(); @@ -1025,6 +992,7 @@ std::string EvalState::mkSingleDerivedPathStringRaw(const SingleDerivedPath & p) auto optStaticOutputPath = std::visit( overloaded{ [&](const SingleDerivedPath::Opaque & o) { + waitForPath(o.path); auto drv = store->readDerivation(o.path); auto i = drv.outputs.find(b.output); if (i == drv.outputs.end()) @@ -1051,10 +1019,6 @@ void EvalState::mkSingleDerivedPathString(const SingleDerivedPath & p, Value & v }); } -/* Create a thunk for the delayed computation of the given expression - in the given environment. But if the expression is a variable, - then look it up right away. This significantly reduces the number - of thunks allocated. */ Value * Expr::maybeThunk(EvalState & state, Env & env) { Value * v = state.allocValue(); @@ -1098,61 +1062,85 @@ Value * ExprPath::maybeThunk(EvalState & state, Env & env) return &v; } -void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) +/** + * A helper `Expr` class to lets us parse and evaluate Nix expressions + * from a thunk, ensuring that every file is parsed/evaluated only + * once (via the thunk stored in `EvalState::fileEvalCache`). + */ +struct ExprParseFile : Expr { - FileEvalCache::iterator i; - if ((i = fileEvalCache.find(path)) != fileEvalCache.end()) { - v = i->second; - return; - } + SourcePath & path; + bool mustBeTrivial; - auto resolvedPath = resolveExprPath(path); - if ((i = fileEvalCache.find(resolvedPath)) != fileEvalCache.end()) { - v = i->second; - return; + ExprParseFile(SourcePath & path, bool mustBeTrivial) + : path(path) + , mustBeTrivial(mustBeTrivial) + { } - printTalkative("evaluating file '%1%'", resolvedPath); - Expr * e = nullptr; + void eval(EvalState & state, Env & env, Value & v) override + { + printTalkative("evaluating file '%s'", path); - auto j = fileParseCache.find(resolvedPath); - if (j != fileParseCache.end()) - e = j->second; + auto e = state.parseExprFromFile(path); - if (!e) - e = parseExprFromFile(resolvedPath); + try { + auto dts = + state.debugRepl + ? makeDebugTraceStacker( + state, *e, state.baseEnv, e->getPos(), "while evaluating the file '%s':", path.to_string()) + : nullptr; + + // Enforce that 'flake.nix' is a direct attrset, not a + // computation. + if (mustBeTrivial && !(dynamic_cast(e))) + state.error("file '%s' must be an attribute set", path).debugThrow(); + + state.eval(e, v); + } catch (Error & e) { + state.addErrorTrace(e, "while evaluating the file '%s':", path.to_string()); + throw; + } + } +}; - fileParseCache.emplace(resolvedPath, e); +void EvalState::evalFile(const SourcePath & path, Value & v, bool mustBeTrivial) +{ + auto resolvedPath = getConcurrent(*importResolutionCache, path); - try { - auto dts = debugRepl ? makeDebugTraceStacker( - *this, - *e, - this->baseEnv, - e->getPos(), - "while evaluating the file '%1%':", - resolvedPath.to_string()) - : nullptr; - - // Enforce that 'flake.nix' is a direct attrset, not a - // computation. - if (mustBeTrivial && !(dynamic_cast(e))) - error("file '%s' must be an attribute set", path).debugThrow(); - eval(e, v); - } catch (Error & e) { - addErrorTrace(e, "while evaluating the file '%1%':", resolvedPath.to_string()); - throw; + if (!resolvedPath) { + resolvedPath = resolveExprPath(path); + importResolutionCache->emplace(path, *resolvedPath); } - fileEvalCache.emplace(resolvedPath, v); - if (path != resolvedPath) - fileEvalCache.emplace(path, v); + if (auto v2 = getConcurrent(*fileEvalCache, *resolvedPath)) { + forceValue(**v2, noPos); + v = **v2; + return; + } + + Value * vExpr; + ExprParseFile expr{*resolvedPath, mustBeTrivial}; + + fileEvalCache->try_emplace_and_cvisit( + *resolvedPath, + nullptr, + [&](auto & i) { + vExpr = allocValue(); + vExpr->mkThunk(&baseEnv, &expr); + i.second = vExpr; + }, + [&](auto & i) { vExpr = i.second; }); + + forceValue(*vExpr, noPos); + + v = *vExpr; } void EvalState::resetFileCache() { - fileEvalCache.clear(); - fileParseCache.clear(); + importResolutionCache->clear(); + fileEvalCache->clear(); inputCache->clear(); } @@ -1221,7 +1209,7 @@ void ExprPath::eval(EvalState & state, Env & env, Value & v) Env * ExprAttrs::buildInheritFromEnv(EvalState & state, Env & up) { - Env & inheritEnv = state.allocEnv(inheritFromExprs->size()); + Env & inheritEnv = state.mem.allocEnv(inheritFromExprs->size()); inheritEnv.up = &up; Displacement displ = 0; @@ -1240,12 +1228,12 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) if (recursive) { /* Create a new environment that contains the attributes in this `rec'. */ - Env & env2(state.allocEnv(attrs.size())); + Env & env2(state.mem.allocEnv(attrs.size())); env2.up = &env; dynamicEnv = &env2; Env * inheritEnv = inheritFromExprs ? buildInheritFromEnv(state, env2) : nullptr; - AttrDefs::iterator overrides = attrs.find(state.sOverrides); + AttrDefs::iterator overrides = attrs.find(state.s.overrides); bool hasOverrides = overrides != attrs.end(); /* The recursive attributes are evaluated in the new @@ -1277,7 +1265,7 @@ void ExprAttrs::eval(EvalState & state, Env & env, Value & v) *vOverrides, [&]() { return vOverrides->determinePos(noPos); }, "while evaluating the `__overrides` attribute"); - bindings.grow(state.allocBindings(bindings.capacity() + vOverrides->attrs()->size())); + bindings.grow(state.buildBindings(bindings.capacity() + vOverrides->attrs()->size())); for (auto & i : *vOverrides->attrs()) { AttrDefs::iterator j = attrs.find(i.name); if (j != attrs.end()) { @@ -1332,7 +1320,7 @@ void ExprLet::eval(EvalState & state, Env & env, Value & v) { /* Create a new environment that contains the attributes in this `let'. */ - Env & env2(state.allocEnv(attrs->attrs.size())); + Env & env2(state.mem.allocEnv(attrs->attrs.size())); env2.up = &env; Env * inheritEnv = attrs->inheritFromExprs ? attrs->buildInheritFromEnv(state, env2) : nullptr; @@ -1363,7 +1351,7 @@ void ExprList::eval(EvalState & state, Env & env, Value & v) Value * ExprList::maybeThunk(EvalState & state, Env & env) { if (elems.empty()) { - return &state.vEmptyList; + return &Value::vEmptyList; } return Expr::maybeThunk(state, env); } @@ -1375,7 +1363,7 @@ void ExprVar::eval(EvalState & state, Env & env, Value & v) v = *v2; } -static std::string showAttrPath(EvalState & state, Env & env, const AttrPath & attrPath) +static std::string showAttrPath(EvalState & state, Env & env, std::span attrPath) { std::ostringstream out; bool first = true; @@ -1411,10 +1399,10 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) env, getPos(), "while evaluating the attribute '%1%'", - showAttrPath(state, env, attrPath)) + showAttrPath(state, env, getAttrPath())) : nullptr; - for (auto & i : attrPath) { + for (auto & i : getAttrPath()) { state.nrLookups++; const Attr * j; auto name = getName(i, state, env); @@ -1444,7 +1432,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) state.attrSelects[pos2]++; } - state.forceValue(*vAttrs, (pos2 ? pos2 : this->pos)); + state.forceValue(*vAttrs, pos2 ? pos2 : this->pos); } catch (Error & e) { if (pos2) { @@ -1452,7 +1440,7 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) auto origin = std::get_if(&pos2r.origin); if (!(origin && *origin == state.derivationInternal)) state.addErrorTrace( - e, pos2, "while evaluating the attribute '%1%'", showAttrPath(state, env, attrPath)); + e, pos2, "while evaluating the attribute '%1%'", showAttrPath(state, env, getAttrPath())); } throw; } @@ -1463,13 +1451,13 @@ void ExprSelect::eval(EvalState & state, Env & env, Value & v) Symbol ExprSelect::evalExceptFinalSelect(EvalState & state, Env & env, Value & attrs) { Value vTmp; - Symbol name = getName(attrPath[attrPath.size() - 1], state, env); + Symbol name = getName(attrPathStart[nAttrPath - 1], state, env); - if (attrPath.size() == 1) { + if (nAttrPath == 1) { e->eval(state, env, vTmp); } else { ExprSelect init(*this); - init.attrPath.pop_back(); + init.nAttrPath--; init.eval(state, env, vTmp); } attrs = vTmp; @@ -1503,6 +1491,8 @@ void ExprLambda::eval(EvalState & state, Env & env, Value & v) v.mkLambda(&env, this); } +thread_local size_t EvalState::callDepth = 0; + void EvalState::callFunction(Value & fun, std::span args, Value & vRes, const PosIdx pos) { auto _level = addCallDepth(pos); @@ -1518,15 +1508,16 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, forceValue(fun, pos); - Value vCur(fun); + Value vCur = fun; auto makeAppChain = [&]() { - vRes = vCur; for (auto arg : args) { auto fun2 = allocValue(); - *fun2 = vRes; - vRes.mkPrimOpApp(fun2, arg); + *fun2 = vCur; + vCur.reset(); + vCur.mkPrimOpApp(fun2, arg); } + vRes = vCur; }; const Attr * functor; @@ -1538,7 +1529,7 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, ExprLambda & lambda(*vCur.lambda().fun); auto size = (!lambda.arg ? 0 : 1) + (lambda.hasFormals() ? lambda.formals->formals.size() : 0); - Env & env2(allocEnv(size)); + Env & env2(mem.allocEnv(size)); env2.up = vCur.lambda().env; Displacement displ = 0; @@ -1622,6 +1613,7 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, lambda.name ? concatStrings("'", symbols[lambda.name], "'") : "anonymous lambda") : nullptr; + vCur.reset(); lambda.body->eval(*this, env2, vCur); } catch (Error & e) { if (loggerSettings.showTrace.get()) { @@ -1656,7 +1648,9 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, primOpCalls[fn->name]++; try { - fn->fun(*this, vCur.determinePos(noPos), args.data(), vCur); + auto pos = vCur.determinePos(noPos); + vCur.reset(); + fn->fun(*this, pos, args.data(), vCur); } catch (Error & e) { if (fn->addTrace) addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); @@ -1678,6 +1672,7 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, assert(primOp->isPrimOp()); auto arity = primOp->primOp()->arity; auto argsLeft = arity - argsDone; + assert(argsLeft); if (args.size() < argsLeft) { /* We still don't have enough arguments, so extend the tPrimOpApp chain. */ @@ -1706,7 +1701,9 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, // 2. Create a fake env (arg1, arg2, etc.) and a fake expr (arg1: arg2: etc: builtins.name arg1 arg2 // etc) // so the debugger allows to inspect the wrong parameters passed to the builtin. - fn->fun(*this, vCur.determinePos(noPos), vArgs, vCur); + auto pos = vCur.determinePos(noPos); + vCur.reset(); + fn->fun(*this, pos, vArgs, vCur); } catch (Error & e) { if (fn->addTrace) addErrorTrace(e, pos, "while calling the '%1%' builtin", fn->name); @@ -1717,12 +1714,13 @@ void EvalState::callFunction(Value & fun, std::span args, Value & vRes, } } - else if (vCur.type() == nAttrs && (functor = vCur.attrs()->get(sFunctor))) { + else if (vCur.type() == nAttrs && (functor = vCur.attrs()->get(s.functor))) { /* 'vCur' may be allocated on the stack of the calling function, but for functors we may keep a reference, so heap-allocate a copy and use that instead. */ Value * args2[] = {allocValue(), args[0]}; *args2[0] = vCur; + vCur.reset(); try { callFunction(*functor->value, args2, vCur, functor->pos); } catch (Error & e) { @@ -1779,8 +1777,8 @@ void EvalState::autoCallFunction(const Bindings & args, Value & fun, Value & res forceValue(fun, pos); if (fun.type() == nAttrs) { - auto found = fun.attrs()->find(sFunctor); - if (found != fun.attrs()->end()) { + auto found = fun.attrs()->get(s.functor); + if (found) { Value * v = allocValue(); callFunction(*found->value, fun, *v, pos); forceValue(*v, pos); @@ -1827,7 +1825,7 @@ values, or passed explicitly with '--arg' or '--argstr'. See void ExprWith::eval(EvalState & state, Env & env, Value & v) { - Env & env2(state.allocEnv(1)); + Env & env2(state.mem.allocEnv(1)); env2.up = &env; env2.values[0] = attrs->maybeThunk(state, env); @@ -1845,7 +1843,7 @@ void ExprAssert::eval(EvalState & state, Env & env, Value & v) if (!state.evalBool(env, cond, pos, "in the condition of the assert statement")) { std::ostringstream out; cond->show(state.symbols, out); - auto exprStr = toView(out); + auto exprStr = out.view(); if (auto eq = dynamic_cast(cond)) { try { @@ -1917,37 +1915,71 @@ void ExprOpUpdate::eval(EvalState & state, Env & env, Value & v) state.nrOpUpdates++; - if (v1.attrs()->size() == 0) { + const Bindings & bindings1 = *v1.attrs(); + if (bindings1.empty()) { v = v2; return; } - if (v2.attrs()->size() == 0) { + + const Bindings & bindings2 = *v2.attrs(); + if (bindings2.empty()) { v = v1; return; } - auto attrs = state.buildBindings(v1.attrs()->size() + v2.attrs()->size()); + /* Simple heuristic for determining whether attrs2 should be "layered" on top of + attrs1 instead of copying to a new Bindings. */ + bool shouldLayer = [&]() -> bool { + if (bindings1.isLayerListFull()) + return false; + + if (bindings2.size() > state.settings.bindingsUpdateLayerRhsSizeThreshold) + return false; + + return true; + }(); + + if (shouldLayer) { + auto attrs = state.buildBindings(bindings2.size()); + attrs.layerOnTopOf(bindings1); + + std::ranges::copy(bindings2, std::back_inserter(attrs)); + v.mkAttrs(attrs.alreadySorted()); + + state.nrOpUpdateValuesCopied += bindings2.size(); + return; + } + + auto attrs = state.buildBindings(bindings1.size() + bindings2.size()); /* Merge the sets, preferring values from the second set. Make sure to keep the resulting vector in sorted order. */ - auto i = v1.attrs()->begin(); - auto j = v2.attrs()->begin(); + auto i = bindings1.begin(); + auto j = bindings2.begin(); - while (i != v1.attrs()->end() && j != v2.attrs()->end()) { + while (i != bindings1.end() && j != bindings2.end()) { if (i->name == j->name) { attrs.insert(*j); ++i; ++j; - } else if (i->name < j->name) - attrs.insert(*i++); - else - attrs.insert(*j++); + } else if (i->name < j->name) { + attrs.insert(*i); + ++i; + } else { + attrs.insert(*j); + ++j; + } } - while (i != v1.attrs()->end()) - attrs.insert(*i++); - while (j != v2.attrs()->end()) - attrs.insert(*j++); + while (i != bindings1.end()) { + attrs.insert(*i); + ++i; + } + + while (j != bindings2.end()) { + attrs.insert(*j); + ++j; + } v.mkAttrs(attrs.alreadySorted()); @@ -2094,7 +2126,7 @@ void ExprConcatStrings::eval(EvalState & state, Env & env, Value & v) else if (firstType == nFloat) v.mkFloat(nf); else if (firstType == nPath) { - if (!context.empty()) + if (hasContext(context)) state.error("a string that refers to a store path cannot be appended to a path") .atPos(pos) .withFrame(env, *this) @@ -2109,16 +2141,6 @@ void ExprPos::eval(EvalState & state, Env & env, Value & v) state.mkPos(v, pos); } -void ExprBlackHole::eval(EvalState & state, [[maybe_unused]] Env & env, Value & v) -{ - throwInfiniteRecursionError(state, v); -} - -[[gnu::noinline]] [[noreturn]] void ExprBlackHole::throwInfiniteRecursionError(EvalState & state, Value & v) -{ - state.error("infinite recursion encountered").atPos(v.determinePos(noPos)).debugThrow(); -} - // always force this to be separate, otherwise forceValue may inline it and take // a massive perf hit [[gnu::noinline]] @@ -2151,6 +2173,7 @@ void EvalState::forceValueDeep(Value & v) for (auto & i : *v.attrs()) try { // If the value is a thunk, we're evaling. Otherwise no trace necessary. + // FIXME: race, thunk might be updated by another thread auto dts = debugRepl && i.value->isThunk() ? makeDebugTraceStacker( *this, *i.value->thunk().expr, @@ -2230,10 +2253,10 @@ bool EvalState::forceBool(Value & v, const PosIdx pos, std::string_view errorCtx return v.boolean(); } -Bindings::const_iterator EvalState::getAttr(Symbol attrSym, const Bindings * attrSet, std::string_view errorCtx) +const Attr * EvalState::getAttr(Symbol attrSym, const Bindings * attrSet, std::string_view errorCtx) { - auto value = attrSet->find(attrSym); - if (value == attrSet->end()) { + auto value = attrSet->get(attrSym); + if (!value) { error("attribute '%s' missing", symbols[attrSym]).withTrace(noPos, errorCtx).debugThrow(); } return value; @@ -2241,7 +2264,7 @@ Bindings::const_iterator EvalState::getAttr(Symbol attrSym, const Bindings * att bool EvalState::isFunctor(const Value & fun) const { - return fun.type() == nAttrs && fun.attrs()->find(sFunctor) != fun.attrs()->end(); + return fun.type() == nAttrs && fun.attrs()->get(s.functor); } void EvalState::forceFunction(Value & v, const PosIdx pos, std::string_view errorCtx) @@ -2298,10 +2321,15 @@ std::string_view EvalState::forceStringNoCtx(Value & v, const PosIdx pos, std::s { auto s = forceString(v, pos, errorCtx); if (v.context()) { - error( - "the string '%1%' is not allowed to refer to a store path (such as '%2%')", v.string_view(), v.context()[0]) - .withTrace(pos, errorCtx) - .debugThrow(); + NixStringContext context; + copyContext(v, context); + if (hasContext(context)) + error( + "the string '%1%' is not allowed to refer to a store path (such as '%2%')", + v.string_view(), + v.context()[0]) + .withTrace(pos, errorCtx) + .debugThrow(); } return s; } @@ -2310,7 +2338,7 @@ bool EvalState::isDerivation(Value & v) { if (v.type() != nAttrs) return false; - auto i = v.attrs()->get(sType); + auto i = v.attrs()->get(s.type); if (!i) return false; forceValue(*i->value, i->pos); @@ -2322,8 +2350,8 @@ bool EvalState::isDerivation(Value & v) std::optional EvalState::tryAttrsToString(const PosIdx pos, Value & v, NixStringContext & context, bool coerceMore, bool copyToStore) { - auto i = v.attrs()->find(sToString); - if (i != v.attrs()->end()) { + auto i = v.attrs()->get(s.toString); + if (i) { Value v1; callFunction(*i->value, v, v1, pos); return coerceToString( @@ -2356,20 +2384,29 @@ BackedStringView EvalState::coerceToString( } if (v.type() == nPath) { + // FIXME: instead of copying the path to the store, we could + // return a virtual store path that lazily copies the path to + // the store in devirtualize(). return !canonicalizePath && !copyToStore ? // FIXME: hack to preserve path literals that end in a // slash, as in /foo/${x}. v.pathStr() - : copyToStore ? store->printStorePath(copyPathToStore(context, v.path())) - : std::string(v.path().path.abs()); + : copyToStore ? store->printStorePath(copyPathToStore(context, v.path(), v.determinePos(pos))) : ({ + auto path = v.path(); + if (path.accessor == rootFS && store->isInStore(path.path.abs())) { + context.insert( + NixStringContextElem::Path{.storePath = store->toStorePath(path.path.abs()).first}); + } + std::string(path.path.abs()); + }); } if (v.type() == nAttrs) { auto maybeString = tryAttrsToString(pos, v, context, coerceMore, copyToStore); if (maybeString) return std::move(*maybeString); - auto i = v.attrs()->find(sOutPath); - if (i == v.attrs()->end()) { + auto i = v.attrs()->get(s.outPath); + if (!i) { error( "cannot coerce %1% to a string: %2%", showType(v), ValuePrinter(*this, v, errorPrintOptions)) .withTrace(pos, errorCtx) @@ -2432,12 +2469,12 @@ BackedStringView EvalState::coerceToString( .debugThrow(); } -StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path) +StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePath & path, PosIdx pos) { if (nix::isDerivation(path.path.abs())) error("file names are not allowed to end in '%1%'", drvExtension).debugThrow(); - auto dstPathCached = get(*srcToStore.lock(), path); + auto dstPathCached = getConcurrent(*srcToStore, path); auto dstPath = dstPathCached ? *dstPathCached : [&]() { auto dstPath = fetchToStore( @@ -2445,12 +2482,12 @@ StorePath EvalState::copyPathToStore(NixStringContext & context, const SourcePat *store, path.resolveSymlinks(SymlinkResolution::Ancestors), settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, - path.baseName(), + computeBaseName(path, pos), ContentAddressMethod::Raw::NixArchive, nullptr, repair); allowPath(dstPath); - srcToStore.lock()->try_emplace(path, dstPath); + srcToStore->try_emplace(path, dstPath); printMsg(lvlChatty, "copied source '%1%' -> '%2%'", path, store->printStorePath(dstPath)); return dstPath; }(); @@ -2475,8 +2512,8 @@ SourcePath EvalState::coerceToPath(const PosIdx pos, Value & v, NixStringContext /* Similarly, handle __toString where the result may be a path value. */ if (v.type() == nAttrs) { - auto i = v.attrs()->find(sToString); - if (i != v.attrs()->end()) { + auto i = v.attrs()->get(s.toString); + if (i) { Value v1; callFunction(*i->value, v, v1, pos); return coerceToPath(pos, v1, context, errorCtx); @@ -2497,7 +2534,9 @@ EvalState::coerceToStorePath(const PosIdx pos, Value & v, NixStringContext & con auto path = coerceToString(pos, v, context, errorCtx, false, false, true).toOwned(); if (auto storePath = store->maybeParseStorePath(path)) return *storePath; - error("path '%1%' is not in the Nix store", path).withTrace(pos, errorCtx).debugThrow(); + error("cannot coerce '%s' to a store path because it is not a subpath of the Nix store", path) + .withTrace(pos, errorCtx) + .debugThrow(); } std::pair EvalState::coerceToSingleDerivedPathUnchecked( @@ -2521,6 +2560,9 @@ std::pair EvalState::coerceToSingleDerivedP .debugThrow(); }, [&](NixStringContextElem::Built && b) -> SingleDerivedPath { return std::move(b); }, + [&](NixStringContextElem::Path && p) -> SingleDerivedPath { + error("string '%s' has no context", s).withTrace(pos, errorCtx).debugThrow(); + }, }, ((NixStringContextElem &&) *context.begin()).raw); return { @@ -2665,8 +2707,8 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st case nAttrs: { if (isDerivation(v1) && isDerivation(v2)) { - auto i = v1.attrs()->get(sOutPath); - auto j = v2.attrs()->get(sOutPath); + auto i = v1.attrs()->get(s.outPath); + auto j = v2.attrs()->get(s.outPath); if (i && j) { try { assertEqValues(*i->value, *j->value, pos, errorCtx); @@ -2755,8 +2797,11 @@ void EvalState::assertEqValues(Value & v1, Value & v2, const PosIdx pos, std::st } return; - case nThunk: // Must not be left by forceValue - assert(false); + // Cannot be returned by forceValue(). + case nThunk: + case nFailed: + unreachable(); + default: // Note that we pass compiler flags that should make `default:` unreachable. // Also note that this probably ran after `eqValues`, which implements // the same logic more efficiently (without having to unwind stacks), @@ -2819,8 +2864,8 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v /* If both sets denote a derivation (type = "derivation"), then compare their outPaths. */ if (isDerivation(v1) && isDerivation(v2)) { - auto i = v1.attrs()->get(sOutPath); - auto j = v2.attrs()->get(sOutPath); + auto i = v1.attrs()->get(s.outPath); + auto j = v2.attrs()->get(s.outPath); if (i && j) return eqValues(*i->value, *j->value, pos, errorCtx); } @@ -2848,8 +2893,11 @@ bool EvalState::eqValues(Value & v1, Value & v2, const PosIdx pos, std::string_v // !!! return v1.fpoint() == v2.fpoint(); - case nThunk: // Must not be left by forceValue - assert(false); + // Cannot be returned by forceValue(). + case nThunk: + case nFailed: + unreachable(); + default: // Note that we pass compiler flags that should make `default:` unreachable. error("eqValues: cannot compare %1% with %2%", showType(v1), showType(v2)) .withTrace(pos, errorCtx) @@ -2872,11 +2920,11 @@ bool EvalState::fullGC() #endif } +bool Counter::enabled = getEnv("NIX_SHOW_STATS").value_or("0") != "0"; + void EvalState::maybePrintStats() { - bool showStats = getEnv("NIX_SHOW_STATS").value_or("0") != "0"; - - if (showStats) { + if (Counter::enabled) { // Make the final heap size more deterministic. #if NIX_USE_BOEHMGC if (!fullGC()) { @@ -2889,16 +2937,15 @@ void EvalState::maybePrintStats() void EvalState::printStatistics() { -#ifndef _WIN32 // TODO use portable implementation - struct rusage buf; - getrusage(RUSAGE_SELF, &buf); - float cpuTime = buf.ru_utime.tv_sec + ((float) buf.ru_utime.tv_usec / 1000000); -#endif + std::chrono::microseconds cpuTimeDuration = getCpuUserTime(); + float cpuTime = std::chrono::duration_cast>(cpuTimeDuration).count(); - uint64_t bEnvs = nrEnvs * sizeof(Env) + nrValuesInEnvs * sizeof(Value *); - uint64_t bLists = nrListElems * sizeof(Value *); - uint64_t bValues = nrValues * sizeof(Value); - uint64_t bAttrsets = nrAttrsets * sizeof(Bindings) + nrAttrsInAttrsets * sizeof(Attr); + auto & memstats = mem.getStats(); + + uint64_t bEnvs = memstats.nrEnvs * sizeof(Env) + memstats.nrValuesInEnvs * sizeof(Value *); + uint64_t bLists = memstats.nrListElems * sizeof(Value *); + uint64_t bValues = memstats.nrValues * sizeof(Value); + uint64_t bAttrsets = memstats.nrAttrsets * sizeof(Bindings) + memstats.nrAttrsInAttrsets * sizeof(Attr); #if NIX_USE_BOEHMGC GC_word heapSize, totalBytes; @@ -2915,33 +2962,27 @@ void EvalState::printStatistics() if (outPath != "-") fs.open(outPath, std::fstream::out); json topObj = json::object(); -#ifndef _WIN32 // TODO implement topObj["cpuTime"] = cpuTime; -#endif topObj["time"] = { -#ifndef _WIN32 // TODO implement {"cpu", cpuTime}, -#endif #if NIX_USE_BOEHMGC {GC_is_incremental_mode() ? "gcNonIncremental" : "gc", gcFullOnlyTime}, -# ifndef _WIN32 // TODO implement {GC_is_incremental_mode() ? "gcNonIncrementalFraction" : "gcFraction", gcFullOnlyTime / cpuTime}, -# endif #endif }; topObj["envs"] = { - {"number", nrEnvs}, - {"elements", nrValuesInEnvs}, + {"number", memstats.nrEnvs.load()}, + {"elements", memstats.nrValuesInEnvs.load()}, {"bytes", bEnvs}, }; - topObj["nrExprs"] = Expr::nrExprs; + topObj["nrExprs"] = Expr::nrExprs.load(); topObj["list"] = { - {"elements", nrListElems}, + {"elements", memstats.nrListElems.load()}, {"bytes", bLists}, - {"concats", nrListConcats}, + {"concats", nrListConcats.load()}, }; topObj["values"] = { - {"number", nrValues}, + {"number", memstats.nrValues.load()}, {"bytes", bValues}, }; topObj["symbols"] = { @@ -2949,9 +2990,9 @@ void EvalState::printStatistics() {"bytes", symbols.totalSize()}, }; topObj["sets"] = { - {"number", nrAttrsets}, + {"number", memstats.nrAttrsets.load()}, {"bytes", bAttrsets}, - {"elements", nrAttrsInAttrsets}, + {"elements", memstats.nrAttrsInAttrsets.load()}, }; topObj["sizes"] = { {"Env", sizeof(Env)}, @@ -2959,13 +3000,18 @@ void EvalState::printStatistics() {"Bindings", sizeof(Bindings)}, {"Attr", sizeof(Attr)}, }; - topObj["nrOpUpdates"] = nrOpUpdates; - topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied; - topObj["nrThunks"] = nrThunks; - topObj["nrAvoided"] = nrAvoided; - topObj["nrLookups"] = nrLookups; - topObj["nrPrimOpCalls"] = nrPrimOpCalls; - topObj["nrFunctionCalls"] = nrFunctionCalls; + topObj["nrOpUpdates"] = nrOpUpdates.load(); + topObj["nrOpUpdateValuesCopied"] = nrOpUpdateValuesCopied.load(); + topObj["nrThunks"] = nrThunks.load(); + topObj["nrThunksAwaited"] = nrThunksAwaited.load(); + topObj["nrThunksAwaitedSlow"] = nrThunksAwaitedSlow.load(); + topObj["nrSpuriousWakeups"] = nrSpuriousWakeups.load(); + topObj["maxWaiting"] = maxWaiting.load(); + topObj["waitingTime"] = microsecondsWaiting / (double) 1000000; + topObj["nrAvoided"] = nrAvoided.load(); + topObj["nrLookups"] = nrLookups.load(); + topObj["nrPrimOpCalls"] = nrPrimOpCalls.load(); + topObj["nrFunctionCalls"] = nrFunctionCalls.load(); #if NIX_USE_BOEHMGC topObj["gc"] = { {"heapSize", heapSize}, @@ -3013,10 +3059,10 @@ void EvalState::printStatistics() } if (getEnv("NIX_SHOW_SYMBOLS").value_or("0") != "0") { + auto list = json::array(); + symbols.dump([&](std::string_view s) { list.emplace_back(std::string(s)); }); // XXX: overrides earlier assignment - topObj["symbols"] = json::array(); - auto & list = topObj["symbols"]; - symbols.dump([&](std::string_view s) { list.emplace_back(s); }); + topObj["symbols"] = std::move(list); } if (outPath == "-") { std::cerr << topObj.dump(2) << std::endl; @@ -3112,6 +3158,11 @@ SourcePath EvalState::findFile(const LookupPath & lookupPath, const std::string_ auto res = (r / CanonPath(suffix)).resolveSymlinks(); if (res.pathExists()) return res; + + // Backward compatibility hack: throw an exception if access + // to this path is not allowed. + if (auto accessor = res.accessor.dynamic_pointer_cast()) + accessor->checkAccess(res.path); } if (hasPrefix(path, "nix/")) @@ -3166,7 +3217,7 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat /* Allow access to paths in the search path. */ if (initAccessControl) { - allowPath(path.path.abs()); + allowPathLegacy(path.path.abs()); if (store->isInStore(path.path.abs())) { try { allowClosure(store->toStorePath(path.path.abs()).first); @@ -3178,6 +3229,11 @@ std::optional EvalState::resolveLookupPathPath(const LookupPath::Pat if (path.resolveSymlinks().pathExists()) return finish(std::move(path)); else { + // Backward compatibility hack: throw an exception if access + // to this path is not allowed. + if (auto accessor = path.accessor.dynamic_pointer_cast()) + accessor->checkAccess(path.path); + logWarning({.msg = HintFmt("Nix search path entry '%1%' does not exist, ignoring", value)}); } } @@ -3189,15 +3245,15 @@ Expr * EvalState::parse( char * text, size_t length, Pos::Origin origin, const SourcePath & basePath, std::shared_ptr & staticEnv) { DocCommentMap tmpDocComments; // Only used when not origin is not a SourcePath - DocCommentMap * docComments = &tmpDocComments; + auto * docComments = &tmpDocComments; if (auto sourcePath = std::get_if(&origin)) { - auto [it, _] = positionToDocComment.try_emplace(*sourcePath); - docComments = &it->second; + auto [it, _] = positionToDocComment.lock()->try_emplace(*sourcePath, make_ref()); + docComments = &*it->second; } auto result = parseExprFromBuf( - text, length, origin, basePath, symbols, settings, positions, *docComments, rootFS, exprSymbols); + text, length, origin, basePath, mem.exprs.alloc, symbols, settings, positions, *docComments, rootFS); result->bindVars(*this, staticEnv); @@ -3211,12 +3267,14 @@ DocComment EvalState::getDocCommentForPos(PosIdx pos) if (!path) return {}; - auto table = positionToDocComment.find(*path); - if (table == positionToDocComment.end()) + auto positionToDocComment_ = positionToDocComment.readLock(); + + auto table = positionToDocComment_->find(*path); + if (table == positionToDocComment_->end()) return {}; - auto it = table->second.find(pos); - if (it == table->second.end()) + auto it = table->second->find(pos); + if (it == table->second->end()) return {}; return it->second; } @@ -3250,4 +3308,24 @@ void forceNoNullByte(std::string_view s, std::function pos) } } +void EvalState::waitForPath(const StorePath & path) +{ + asyncPathWriter->waitForPath(path); +} + +void EvalState::waitForPath(const SingleDerivedPath & path) +{ + std::visit( + overloaded{ + [&](const DerivedPathOpaque & p) { waitForPath(p.path); }, + [&](const SingleDerivedPathBuilt & p) { waitForPath(*p.drvPath); }, + }, + path.raw()); +} + +void EvalState::waitForAllPaths() +{ + asyncPathWriter->waitForAllPaths(); +} + } // namespace nix diff --git a/src/libexpr/get-drvs.cc b/src/libexpr/get-drvs.cc index a1c3e56113e..5a7281b2b82 100644 --- a/src/libexpr/get-drvs.cc +++ b/src/libexpr/get-drvs.cc @@ -45,8 +45,8 @@ PackageInfo::PackageInfo(EvalState & state, ref store, const std::string std::string PackageInfo::queryName() const { if (name == "" && attrs) { - auto i = attrs->find(state->sName); - if (i == attrs->end()) + auto i = attrs->get(state->s.name); + if (!i) state->error("derivation name missing").debugThrow(); name = state->forceStringNoCtx(*i->value, noPos, "while evaluating the 'name' attribute of a derivation"); } @@ -56,11 +56,10 @@ std::string PackageInfo::queryName() const std::string PackageInfo::querySystem() const { if (system == "" && attrs) { - auto i = attrs->find(state->sSystem); + auto i = attrs->get(state->s.system); system = - i == attrs->end() - ? "unknown" - : state->forceStringNoCtx(*i->value, i->pos, "while evaluating the 'system' attribute of a derivation"); + !i ? "unknown" + : state->forceStringNoCtx(*i->value, i->pos, "while evaluating the 'system' attribute of a derivation"); } return system; } @@ -68,7 +67,7 @@ std::string PackageInfo::querySystem() const std::optional PackageInfo::queryDrvPath() const { if (!drvPath && attrs) { - if (auto i = attrs->get(state->sDrvPath)) { + if (auto i = attrs->get(state->s.drvPath)) { NixStringContext context; auto found = state->coerceToStorePath( i->pos, *i->value, context, "while evaluating the 'drvPath' attribute of a derivation"); @@ -95,9 +94,9 @@ StorePath PackageInfo::requireDrvPath() const StorePath PackageInfo::queryOutPath() const { if (!outPath && attrs) { - auto i = attrs->find(state->sOutPath); + auto i = attrs->get(state->s.outPath); NixStringContext context; - if (i != attrs->end()) + if (i) outPath = state->coerceToStorePath( i->pos, *i->value, context, "while evaluating the output path of a derivation"); } @@ -111,7 +110,7 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT if (outputs.empty()) { /* Get the ‘outputs’ list. */ const Attr * i; - if (attrs && (i = attrs->get(state->sOutputs))) { + if (attrs && (i = attrs->get(state->s.outputs))) { state->forceList(*i->value, i->pos, "while evaluating the 'outputs' attribute of a derivation"); /* For each output... */ @@ -127,7 +126,7 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT state->forceAttrs(*out->value, i->pos, "while evaluating an output of a derivation"); /* And evaluate its ‘outPath’ attribute. */ - auto outPath = out->value->attrs()->get(state->sOutPath); + auto outPath = out->value->attrs()->get(state->s.outPath); if (!outPath) continue; // FIXME: throw error? NixStringContext context; @@ -146,7 +145,7 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT return outputs; const Attr * i; - if (attrs && (i = attrs->get(state->sOutputSpecified)) + if (attrs && (i = attrs->get(state->s.outputSpecified)) && state->forceBool(*i->value, i->pos, "while evaluating the 'outputSpecified' attribute of a derivation")) { Outputs result; auto out = outputs.find(queryOutputName()); @@ -181,7 +180,7 @@ PackageInfo::Outputs PackageInfo::queryOutputs(bool withPaths, bool onlyOutputsT std::string PackageInfo::queryOutputName() const { if (outputName == "" && attrs) { - auto i = attrs->get(state->sOutputName); + auto i = attrs->get(state->s.outputName); outputName = i ? state->forceStringNoCtx(*i->value, noPos, "while evaluating the output name of a derivation") : ""; } @@ -194,7 +193,7 @@ const Bindings * PackageInfo::getMeta() return meta; if (!attrs) return 0; - auto a = attrs->get(state->sMeta); + auto a = attrs->get(state->s.meta); if (!a) return 0; state->forceAttrs(*a->value, a->pos, "while evaluating the 'meta' attribute of a derivation"); @@ -221,7 +220,7 @@ bool PackageInfo::checkMeta(Value & v) return false; return true; } else if (v.type() == nAttrs) { - if (v.attrs()->get(state->sOutPath)) + if (v.attrs()->get(state->s.outPath)) return false; for (auto & i : *v.attrs()) if (!checkMeta(*i.value)) @@ -411,7 +410,7 @@ static void getDerivations( should we recurse into it? => Only if it has a `recurseForDerivations = true' attribute. */ if (i->value->type() == nAttrs) { - auto j = i->value->attrs()->get(state.sRecurseForDerivations); + auto j = i->value->attrs()->get(state.s.recurseForDerivations); if (j && state.forceBool( *j->value, j->pos, "while evaluating the attribute `recurseForDerivations`")) diff --git a/src/libexpr/include/nix/expr/attr-set.hh b/src/libexpr/include/nix/expr/attr-set.hh index 85bba1099ba..46eecd9bd0d 100644 --- a/src/libexpr/include/nix/expr/attr-set.hh +++ b/src/libexpr/include/nix/expr/attr-set.hh @@ -4,11 +4,16 @@ #include "nix/expr/nixexpr.hh" #include "nix/expr/symbol-table.hh" +#include + #include +#include +#include +#include namespace nix { -class EvalState; +class EvalMemory; struct Value; /** @@ -46,141 +51,430 @@ static_assert( * by its size and its capacity, the capacity being the number of Attr * elements allocated after this structure, while the size corresponds to * the number of elements already inserted in this structure. + * + * Bindings can be efficiently `//`-composed into an intrusive linked list of "layers" + * that saves on copies and allocations. Each lookup (@see Bindings::get) traverses + * this linked list until a matching attribute is found (thus overlays earlier in + * the list take precedence). For iteration over the whole Bindings, an on-the-fly + * k-way merge is performed by Bindings::iterator class. */ class Bindings { public: - typedef uint32_t size_t; + using size_type = uint32_t; + PosIdx pos; + /** + * An instance of bindings objects with 0 attributes. + * This object must never be modified. + */ + static Bindings emptyBindings; + private: - size_t size_, capacity_; + /** + * Number of attributes in the attrs FAM (Flexible Array Member). + */ + size_type numAttrs = 0; + + /** + * Number of attributes with unique names in the layer chain. + * + * This is the *real* user-facing size of bindings, whereas @ref numAttrs is + * an implementation detail of the data structure. + */ + size_type numAttrsInChain = 0; + + /** + * Length of the layers list. + */ + uint32_t numLayers = 1; + + /** + * Bindings that this attrset is "layered" on top of. + */ + const Bindings * baseLayer = nullptr; + + /** + * Flexible array member of attributes. + */ Attr attrs[0]; - Bindings(size_t capacity) - : size_(0) - , capacity_(capacity) - { - } + Bindings() = default; + Bindings(const Bindings &) = delete; + Bindings(Bindings &&) = delete; + Bindings & operator=(const Bindings &) = delete; + Bindings & operator=(Bindings &&) = delete; - Bindings(const Bindings & bindings) = delete; + friend class BindingsBuilder; + + /** + * Maximum length of the Bindings layer chains. + */ + static constexpr unsigned maxLayers = 8; public: - size_t size() const + size_type size() const { - return size_; + return numAttrsInChain; } bool empty() const { - return !size_; + return size() == 0; } - typedef Attr * iterator; - - typedef const Attr * const_iterator; + class iterator + { + public: + using value_type = Attr; + using pointer = const value_type *; + using reference = const value_type &; + using difference_type = std::ptrdiff_t; + using iterator_category = std::forward_iterator_tag; + + friend class Bindings; + + private: + struct BindingsCursor + { + /** + * Attr that the cursor currently points to. + */ + pointer current; + + /** + * One past the end pointer to the contiguous buffer of Attrs. + */ + pointer end; + + /** + * Priority of the value. Lesser values have more priority (i.e. they override + * attributes that appear later in the linked list of Bindings). + */ + uint32_t priority; + + pointer operator->() const noexcept + { + return current; + } + + reference get() const noexcept + { + return *current; + } + + bool empty() const noexcept + { + return current == end; + } + + void increment() noexcept + { + ++current; + } + + void consume(Symbol name) noexcept + { + while (!empty() && current->name <= name) + ++current; + } + + GENERATE_CMP(BindingsCursor, me->current->name, me->priority) + }; + + using QueueStorageType = boost::container::static_vector; + + /** + * Comparator implementing the override priority / name ordering + * for BindingsCursor. + */ + static constexpr auto comp = std::greater(); + + /** + * A priority queue used to implement an on-the-fly k-way merge. + */ + QueueStorageType cursorHeap; + + /** + * The attribute the iterator currently points to. + */ + pointer current = nullptr; + + /** + * Whether iterating over a single attribute and not a merge chain. + */ + bool doMerge = true; + + void push(BindingsCursor cursor) noexcept + { + cursorHeap.push_back(cursor); + std::ranges::make_heap(cursorHeap, comp); + } + + [[nodiscard]] BindingsCursor pop() noexcept + { + std::ranges::pop_heap(cursorHeap, comp); + auto cursor = cursorHeap.back(); + cursorHeap.pop_back(); + return cursor; + } + + iterator & finished() noexcept + { + current = nullptr; + return *this; + } + + void next(BindingsCursor cursor) noexcept + { + current = &cursor.get(); + cursor.increment(); + + if (!cursor.empty()) + push(cursor); + } + + std::optional consumeAllUntilCurrentName() noexcept + { + auto cursor = pop(); + Symbol lastHandledName = current->name; + + while (cursor->name <= lastHandledName) { + cursor.consume(lastHandledName); + if (!cursor.empty()) + push(cursor); + + if (cursorHeap.empty()) + return std::nullopt; + + cursor = pop(); + } + + return cursor; + } + + explicit iterator(const Bindings & attrs) noexcept + : doMerge(attrs.baseLayer) + { + auto pushBindings = [this, priority = unsigned{0}](const Bindings & layer) mutable { + auto first = layer.attrs; + push( + BindingsCursor{ + .current = first, + .end = first + layer.numAttrs, + .priority = priority++, + }); + }; + + if (!doMerge) { + if (attrs.empty()) + return; + + current = attrs.attrs; + pushBindings(attrs); + + return; + } + + const Bindings * layer = &attrs; + while (layer) { + if (layer->numAttrs != 0) + pushBindings(*layer); + layer = layer->baseLayer; + } + + if (cursorHeap.empty()) + return; + + next(pop()); + } + + public: + iterator() = default; + + reference operator*() const noexcept + { + return *current; + } + + pointer operator->() const noexcept + { + return current; + } + + iterator & operator++() noexcept + { + if (!doMerge) { + ++current; + if (current == cursorHeap.front().end) + return finished(); + return *this; + } + + if (cursorHeap.empty()) + return finished(); + + auto cursor = consumeAllUntilCurrentName(); + if (!cursor) + return finished(); + + next(*cursor); + return *this; + } + + iterator operator++(int) noexcept + { + iterator tmp = *this; + ++*this; + return tmp; + } + + bool operator==(const iterator & rhs) const noexcept + { + return current == rhs.current; + } + }; + + using const_iterator = iterator; void push_back(const Attr & attr) { - assert(size_ < capacity_); - attrs[size_++] = attr; + attrs[numAttrs++] = attr; + numAttrsInChain = numAttrs; } - const_iterator find(Symbol name) const + /** + * Get attribute by name or nullptr if no such attribute exists. + */ + const Attr * get(Symbol name) const noexcept { - Attr key(name, 0); - const_iterator i = std::lower_bound(begin(), end(), key); - if (i != end() && i->name == name) - return i; - return end(); - } + auto getInChunk = [key = Attr{name, nullptr}](const Bindings & chunk) -> const Attr * { + auto first = chunk.attrs; + auto last = first + chunk.numAttrs; + const Attr * i = std::lower_bound(first, last, key); + if (i != last && i->name == key.name) + return i; + return nullptr; + }; + + const Bindings * currentChunk = this; + while (currentChunk) { + const Attr * maybeAttr = getInChunk(*currentChunk); + if (maybeAttr) + return maybeAttr; + currentChunk = currentChunk->baseLayer; + } - const Attr * get(Symbol name) const - { - Attr key(name, 0); - const_iterator i = std::lower_bound(begin(), end(), key); - if (i != end() && i->name == name) - return &*i; return nullptr; } - iterator begin() + /** + * Check if the layer chain is full. + */ + bool isLayerListFull() const noexcept { - return &attrs[0]; + return numLayers == Bindings::maxLayers; } - iterator end() + /** + * Test if the length of the linked list of layers is greater than 1. + */ + bool isLayered() const noexcept { - return &attrs[size_]; + return numLayers > 1; } const_iterator begin() const { - return &attrs[0]; + return const_iterator(*this); } const_iterator end() const { - return &attrs[size_]; + return const_iterator(); } - Attr & operator[](size_t pos) + Attr & operator[](size_type pos) { + if (isLayered()) [[unlikely]] + unreachable(); return attrs[pos]; } - const Attr & operator[](size_t pos) const + const Attr & operator[](size_type pos) const { + if (isLayered()) [[unlikely]] + unreachable(); return attrs[pos]; } void sort(); - size_t capacity() const - { - return capacity_; - } - /** * Returns the attributes in lexicographically sorted order. */ std::vector lexicographicOrder(const SymbolTable & symbols) const { std::vector res; - res.reserve(size_); - for (size_t n = 0; n < size_; n++) - res.emplace_back(&attrs[n]); - std::sort(res.begin(), res.end(), [&](const Attr * a, const Attr * b) { + res.reserve(size()); + std::ranges::transform(*this, std::back_inserter(res), [](const Attr & a) { return &a; }); + std::ranges::sort(res, [&](const Attr * a, const Attr * b) { std::string_view sa = symbols[a->name], sb = symbols[b->name]; return sa < sb; }); return res; } - friend class EvalState; + friend class EvalMemory; }; +static_assert(std::forward_iterator); +static_assert(std::ranges::forward_range); + /** * A wrapper around Bindings that ensures that its always in sorted * order at the end. The only way to consume a BindingsBuilder is to * call finish(), which sorts the bindings. */ -class BindingsBuilder +class BindingsBuilder final { - Bindings * bindings; - public: // needed by std::back_inserter using value_type = Attr; + using size_type = Bindings::size_type; + +private: + Bindings * bindings; + Bindings::size_type capacity_; - EvalState & state; + friend class EvalMemory; - BindingsBuilder(EvalState & state, Bindings * bindings) + BindingsBuilder(EvalMemory & mem, SymbolTable & symbols, Bindings * bindings, size_type capacity) : bindings(bindings) - , state(state) + , capacity_(capacity) + , mem(mem) + , symbols(symbols) + { + } + + bool hasBaseLayer() const noexcept { + return bindings->baseLayer; } + void finishSizeIfNecessary() + { + if (hasBaseLayer()) + /* NOTE: Do not use std::ranges::distance, since Bindings is a sized + range, but we are calculating this size here. */ + bindings->numAttrsInChain = std::distance(bindings->begin(), bindings->end()); + } + +public: + std::reference_wrapper mem; + std::reference_wrapper symbols; + void insert(Symbol name, Value * value, PosIdx pos = noPos) { insert(Attr(name, value, pos)); @@ -193,9 +487,26 @@ public: void push_back(const Attr & attr) { + assert(bindings->numAttrs < capacity_); bindings->push_back(attr); } + /** + * "Layer" the newly constructured Bindings on top of another attribute set. + * + * This effectively performs an attribute set merge, while giving preference + * to attributes from the newly constructed Bindings in case of duplicate attribute + * names. + * + * This operation amortizes the need to copy over all attributes and allows + * for efficient implementation of attribute set merges (ExprOpUpdate::eval). + */ + void layerOnTopOf(const Bindings & base) noexcept + { + bindings->baseLayer = &base; + bindings->numLayers = base.numLayers + 1; + } + Value & alloc(Symbol name, PosIdx pos = noPos); Value & alloc(std::string_view name, PosIdx pos = noPos); @@ -203,24 +514,26 @@ public: Bindings * finish() { bindings->sort(); + finishSizeIfNecessary(); return bindings; } Bindings * alreadySorted() { + finishSizeIfNecessary(); return bindings; } - size_t capacity() + size_t capacity() const noexcept { - return bindings->capacity(); + return capacity_; } - void grow(Bindings * newBindings) + void grow(BindingsBuilder newBindings) { for (auto & i : *bindings) - newBindings->push_back(i); - bindings = newBindings; + newBindings.push_back(i); + std::swap(*this, newBindings); } friend struct ExprAttrs; diff --git a/src/libexpr/include/nix/expr/counter.hh b/src/libexpr/include/nix/expr/counter.hh new file mode 100644 index 00000000000..efbf23de349 --- /dev/null +++ b/src/libexpr/include/nix/expr/counter.hh @@ -0,0 +1,70 @@ +#pragma once + +#include +#include + +namespace nix { + +/** + * An atomic counter aligned on a cache line to prevent false sharing. + * The counter is only enabled when the `NIX_SHOW_STATS` environment + * variable is set. This is to prevent contention on these counters + * when multi-threaded evaluation is enabled. + */ +struct alignas(64) Counter +{ + using value_type = uint64_t; + + std::atomic inner{0}; + + static bool enabled; + + Counter() {} + + operator value_type() const noexcept + { + return inner; + } + + void operator=(value_type n) noexcept + { + inner = n; + } + + value_type load() const noexcept + { + return inner; + } + + value_type operator++() noexcept + { + return enabled ? ++inner : 0; + } + + value_type operator++(int) noexcept + { + return enabled ? inner++ : 0; + } + + value_type operator--() noexcept + { + return enabled ? --inner : 0; + } + + value_type operator--(int) noexcept + { + return enabled ? inner-- : 0; + } + + value_type operator+=(value_type n) noexcept + { + return enabled ? inner += n : 0; + } + + value_type operator-=(value_type n) noexcept + { + return enabled ? inner -= n : 0; + } +}; + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/eval-inline.hh b/src/libexpr/include/nix/expr/eval-inline.hh index 749e51537c4..c6c81db8fdc 100644 --- a/src/libexpr/include/nix/expr/eval-inline.hh +++ b/src/libexpr/include/nix/expr/eval-inline.hh @@ -26,13 +26,16 @@ inline void * allocBytes(size_t n) } [[gnu::always_inline]] -Value * EvalState::allocValue() +Value * EvalMemory::allocValue() { #if NIX_USE_BOEHMGC /* We use the boehm batch allocator to speed up allocations of Values (of which there are many). GC_malloc_many returns a linked list of objects of the given size, where the first word of each object is also the pointer to the next object in the list. This also means that we have to explicitly clear the first word of every object we take. */ + thread_local static std::shared_ptr valueAllocCache{ + std::allocate_shared(traceable_allocator(), nullptr)}; + if (!*valueAllocCache) { *valueAllocCache = GC_malloc_many(sizeof(Value)); if (!*valueAllocCache) @@ -48,21 +51,24 @@ Value * EvalState::allocValue() void * p = allocBytes(sizeof(Value)); #endif - nrValues++; + stats.nrValues++; return (Value *) p; } [[gnu::always_inline]] -Env & EvalState::allocEnv(size_t size) +Env & EvalMemory::allocEnv(size_t size) { - nrEnvs++; - nrValuesInEnvs += size; + stats.nrEnvs++; + stats.nrValuesInEnvs += size; Env * env; #if NIX_USE_BOEHMGC if (size == 1) { /* see allocValue for explanations. */ + thread_local static std::shared_ptr env1AllocCache{ + std::allocate_shared(traceable_allocator(), nullptr)}; + if (!*env1AllocCache) { *env1AllocCache = GC_malloc_many(sizeof(Env) + sizeof(Value *)); if (!*env1AllocCache) @@ -82,27 +88,68 @@ Env & EvalState::allocEnv(size_t size) return *env; } -[[gnu::always_inline]] -void EvalState::forceValue(Value & v, const PosIdx pos) +/** + * An identifier of the current thread for deadlock detection, stored + * in p0 of pending/awaited thunks. We're not using std::thread::id + * because it's not guaranteed to fit. + */ +extern thread_local uint32_t myEvalThreadId; + +template +void ValueStorage>>::force( + EvalState & state, PosIdx pos) { - if (v.isThunk()) { - Env * env = v.thunk().env; - assert(env || v.isBlackhole()); - Expr * expr = v.thunk().expr; + auto p0_ = p0.load(std::memory_order_acquire); + + auto pd = static_cast(p0_ & discriminatorMask); + + if (pd == pdThunk) { try { - v.mkBlackhole(); - // checkInterrupt(); - if (env) [[likely]] - expr->eval(*this, *env, v); - else - ExprBlackHole::throwInfiniteRecursionError(*this, v); + // The value we get here is only valid if we can set the + // thunk to pending. + auto p1_ = p1; + + // Atomically set the thunk to "pending". + if (!p0.compare_exchange_strong( + p0_, + pdPending | (myEvalThreadId << discriminatorBits), + std::memory_order_acquire, + std::memory_order_acquire)) { + pd = static_cast(p0_ & discriminatorMask); + if (pd == pdPending || pd == pdAwaited) { + // The thunk is already "pending" or "awaited", so + // we need to wait for it. + p0_ = waitOnThunk(state, p0_); + goto done; + } + assert(pd != pdThunk); + // Another thread finished this thunk, no need to wait. + goto done; + } + + bool isApp = p1_ & discriminatorMask; + if (isApp) { + auto left = untagPointer(p0_); + auto right = untagPointer(p1_); + state.callFunction(*left, *right, (Value &) *this, pos); + } else { + auto env = untagPointer(p0_); + auto expr = untagPointer(p1_); + expr->eval(state, *env, (Value &) *this); + } } catch (...) { - v.mkThunk(env, expr); - tryFixupBlackHolePos(v, pos); + state.tryFixupBlackHolePos((Value &) *this, pos); + setStorage(new Value::Failed{.ex = std::current_exception()}); throw; } - } else if (v.isApp()) - callFunction(*v.app().left, *v.app().right, v, pos); + } + + else if (pd == pdPending || pd == pdAwaited) + p0_ = waitOnThunk(state, p0_); + +done: + if (InternalType(p0_ & 0xff) == tFailed) + std::rethrow_exception((std::bit_cast(p1))->ex); } [[gnu::always_inline]] diff --git a/src/libexpr/include/nix/expr/eval-settings.hh b/src/libexpr/include/nix/expr/eval-settings.hh index 4c9db0c736b..6383bb6cd41 100644 --- a/src/libexpr/include/nix/expr/eval-settings.hh +++ b/src/libexpr/include/nix/expr/eval-settings.hh @@ -91,7 +91,7 @@ struct EvalSettings : Config - `$HOME/.nix-defexpr/channels` - The [user channel link](@docroot@/command-ref/files/default-nix-expression.md#user-channel-link), pointing to the current state of [channels](@docroot@/command-ref/files/channels.md) for the current user. + The user channel link pointing to the current state of channels for the current user. - `nixpkgs=$NIX_STATE_DIR/profiles/per-user/root/channels/nixpkgs` @@ -101,7 +101,7 @@ struct EvalSettings : Config The current state of all channels for the `root` user. - These files are set up by the [Nix installer](@docroot@/installation/installing-binary.md). + These files are set up by the Nix installer. See [`NIX_STATE_DIR`](@docroot@/command-ref/env-common.md#env-NIX_STATE_DIR) for details on the environment variable. > **Note** @@ -142,7 +142,7 @@ struct EvalSettings : Config R"( If set to `true`, the Nix evaluator doesn't allow access to any files outside of - [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath), + [`builtins.nixPath`](@docroot@/language/builtins.md#builtins-nixPath) or to URIs outside of [`allowed-uris`](@docroot@/command-ref/conf-file.md#conf-allowed-uris). )"}; @@ -271,7 +271,7 @@ struct EvalSettings : Config "ignore-try", R"( If set to true, ignore exceptions inside 'tryEval' calls when evaluating Nix expressions in - debug mode (using the --debugger flag). By default the debugger pauses on all exceptions. + debug mode (using the --debugger flag). By default, the debugger pauses on all exceptions. )"}; Setting traceVerbose{ @@ -289,7 +289,7 @@ struct EvalSettings : Config "debugger-on-trace", R"( If set to true and the `--debugger` flag is given, the following functions - enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break): + enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break). * [`builtins.trace`](@docroot@/language/builtins.md#builtins-trace) * [`builtins.traceVerbose`](@docroot@/language/builtins.md#builtins-traceVerbose) @@ -305,7 +305,7 @@ struct EvalSettings : Config "debugger-on-warn", R"( If set to true and the `--debugger` flag is given, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) - will enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break). + enter the debugger like [`builtins.break`](@docroot@/language/builtins.md#builtins-break). This is useful for debugging warnings in third-party Nix code. @@ -319,7 +319,7 @@ struct EvalSettings : Config R"( If set to true, [`builtins.warn`](@docroot@/language/builtins.md#builtins-warn) throws an error when logging a warning. - This will give you a stack trace that leads to the location of the warning. + This gives you a stack trace that leads to the location of the warning. This is useful for finding information about warnings in third-party Nix code when you can not start the interactive debugger, such as when Nix is called from a non-interactive script. See [`debugger-on-warn`](#conf-debugger-on-warn). @@ -342,6 +342,63 @@ struct EvalSettings : Config This is useful for improving code readability and making path literals more explicit. )"}; + + Setting bindingsUpdateLayerRhsSizeThreshold{ + this, + sizeof(void *) == 4 ? 8192 : 16, + "eval-attrset-update-layer-rhs-threshold", + R"( + Tunes the maximum size of an attribute set that, when used + as a right operand in an [attribute set update expression](@docroot@/language/operators.md#update), + uses a more space-efficient linked-list representation of attribute sets. + + Setting this to larger values generally leads to less memory allocations, + but may lead to worse evaluation performance. + + A value of `0` disables this optimization completely. + + This is an advanced performance tuning option and typically should not be changed. + The default value is chosen to balance performance and memory usage. On 32 bit systems + where memory is scarce, the default is a large value to reduce the amount of allocations. + )"}; + + Setting lazyTrees{ + this, + false, + "lazy-trees", + R"( + If set to true, flakes and trees fetched by [`builtins.fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) are only copied to the Nix store when they're used as a dependency of a derivation. This avoids copying (potentially large) source trees unnecessarily. + )"}; + + // FIXME: this setting should really be in libflake, but it's + // currently needed in mountInput(). + Setting lazyLocks{ + this, + false, + "lazy-locks", + R"( + If enabled, Nix only includes NAR hashes in lock file entries if they're necessary to lock the input (i.e. when there is no other attribute that allows the content to be verified, like a Git revision). + This is not backward compatible with older versions of Nix. + If disabled, lock file entries always contain a NAR hash. + )"}; + + Setting evalCores{ + this, + 1, + "eval-cores", + R"( + The number of threads used to evaluate Nix expressions. This currently affects the following commands: + + * `nix search` + * `nix flake check` + * `nix flake show` + * `nix eval --json` + * Any evaluation that uses `builtins.parallel` + + The value `0` causes Nix to use all available CPU cores in the system. + + Note that enabling the debugger (`--debugger`) disables multi-threaded evaluation. + )"}; }; /** @@ -349,4 +406,9 @@ struct EvalSettings : Config */ Path getNixDefExpr(); +/** + * Stack size for evaluator threads. + */ +constexpr size_t evalStackSize = 64 * 1024 * 1024; + } // namespace nix diff --git a/src/libexpr/include/nix/expr/eval.hh b/src/libexpr/include/nix/expr/eval.hh index d52ccb5457e..c9db6f48e74 100644 --- a/src/libexpr/include/nix/expr/eval.hh +++ b/src/libexpr/include/nix/expr/eval.hh @@ -16,10 +16,14 @@ #include "nix/expr/search-path.hh" #include "nix/expr/repl-exit-status.hh" #include "nix/util/ref.hh" +#include "nix/expr/counter.hh" // For `NIX_USE_BOEHMGC`, and if that's set, `GC_THREADS` #include "nix/expr/config.hh" +#include +#include + #include #include #include @@ -38,6 +42,7 @@ class Store; namespace fetchers { struct Settings; struct InputCache; +struct Input; } // namespace fetchers struct EvalSettings; class EvalState; @@ -45,10 +50,13 @@ class StorePath; struct SingleDerivedPath; enum RepairFlag : bool; struct MemorySourceAccessor; +struct MountedSourceAccessor; +struct AsyncPathWriter; namespace eval_cache { class EvalCache; } +struct Executor; /** * Increments a count on construction and decrements on destruction. @@ -162,7 +170,7 @@ typedef std:: map, traceable_allocator>> ValMap; -typedef std::unordered_map DocCommentMap; +typedef boost::unordered_flat_map> DocCommentMap; struct Env { @@ -185,7 +193,7 @@ std::ostream & operator<<(std::ostream & os, const ValueType t); struct RegexCache; -std::shared_ptr makeRegexCache(); +ref makeRegexCache(); struct DebugTrace { @@ -213,70 +221,175 @@ struct DebugTrace } }; -class EvalState : public std::enable_shared_from_this +struct StaticEvalSymbols { -public: - const fetchers::Settings & fetchSettings; - const EvalSettings & settings; - SymbolTable symbols; - PosTable positions; + Symbol with, outPath, drvPath, type, meta, name, value, system, overrides, outputs, outputName, ignoreNulls, file, + line, column, functor, toString, right, wrong, structuredAttrs, json, allowedReferences, allowedRequisites, + disallowedReferences, disallowedRequisites, maxSize, maxClosureSize, builder, args, contentAddressed, impure, + outputHash, outputHashAlgo, outputHashMode, recurseForDerivations, description, self, epsilon, startSet, + operator_, key, path, prefix, outputSpecified; - const Symbol sWith, sOutPath, sDrvPath, sType, sMeta, sName, sValue, sSystem, sOverrides, sOutputs, sOutputName, - sIgnoreNulls, sFile, sLine, sColumn, sFunctor, sToString, sRight, sWrong, sStructuredAttrs, sJson, - sAllowedReferences, sAllowedRequisites, sDisallowedReferences, sDisallowedRequisites, sMaxSize, sMaxClosureSize, - sBuilder, sArgs, sContentAddressed, sImpure, sOutputHash, sOutputHashAlgo, sOutputHashMode, - sRecurseForDerivations, sDescription, sSelf, sEpsilon, sStartSet, sOperator, sKey, sPath, sPrefix, - sOutputSpecified; + Expr::AstSymbols exprSymbols; - const Expr::AstSymbols exprSymbols; + static constexpr auto preallocate() + { + StaticSymbolTable alloc; + + StaticEvalSymbols staticSymbols = { + .with = alloc.create(""), + .outPath = alloc.create("outPath"), + .drvPath = alloc.create("drvPath"), + .type = alloc.create("type"), + .meta = alloc.create("meta"), + .name = alloc.create("name"), + .value = alloc.create("value"), + .system = alloc.create("system"), + .overrides = alloc.create("__overrides"), + .outputs = alloc.create("outputs"), + .outputName = alloc.create("outputName"), + .ignoreNulls = alloc.create("__ignoreNulls"), + .file = alloc.create("file"), + .line = alloc.create("line"), + .column = alloc.create("column"), + .functor = alloc.create("__functor"), + .toString = alloc.create("__toString"), + .right = alloc.create("right"), + .wrong = alloc.create("wrong"), + .structuredAttrs = alloc.create("__structuredAttrs"), + .json = alloc.create("__json"), + .allowedReferences = alloc.create("allowedReferences"), + .allowedRequisites = alloc.create("allowedRequisites"), + .disallowedReferences = alloc.create("disallowedReferences"), + .disallowedRequisites = alloc.create("disallowedRequisites"), + .maxSize = alloc.create("maxSize"), + .maxClosureSize = alloc.create("maxClosureSize"), + .builder = alloc.create("builder"), + .args = alloc.create("args"), + .contentAddressed = alloc.create("__contentAddressed"), + .impure = alloc.create("__impure"), + .outputHash = alloc.create("outputHash"), + .outputHashAlgo = alloc.create("outputHashAlgo"), + .outputHashMode = alloc.create("outputHashMode"), + .recurseForDerivations = alloc.create("recurseForDerivations"), + .description = alloc.create("description"), + .self = alloc.create("self"), + .epsilon = alloc.create(""), + .startSet = alloc.create("startSet"), + .operator_ = alloc.create("operator"), + .key = alloc.create("key"), + .path = alloc.create("path"), + .prefix = alloc.create("prefix"), + .outputSpecified = alloc.create("outputSpecified"), + .exprSymbols = { + .sub = alloc.create("__sub"), + .lessThan = alloc.create("__lessThan"), + .mul = alloc.create("__mul"), + .div = alloc.create("__div"), + .or_ = alloc.create("or"), + .findFile = alloc.create("__findFile"), + .nixPath = alloc.create("__nixPath"), + .body = alloc.create("body"), + }}; + + return std::pair{staticSymbols, alloc}; + } - /** - * If set, force copying files to the Nix store even if they - * already exist there. - */ - RepairFlag repair; + static consteval StaticEvalSymbols create() + { + return preallocate().first; + } - Bindings emptyBindings; + static constexpr StaticSymbolTable staticSymbolTable() + { + return preallocate().second; + } +}; +class EvalMemory +{ +#if NIX_USE_BOEHMGC /** - * Empty list constant. + * Allocation cache for GC'd Value objects. */ - Value vEmptyList; + std::shared_ptr valueAllocCache; /** - * `null` constant. - * - * This is _not_ a singleton. Pointer equality is _not_ sufficient. + * Allocation cache for size-1 Env objects. */ - Value vNull; + std::shared_ptr env1AllocCache; +#endif + +public: + struct Statistics + { + Counter nrEnvs; + Counter nrValuesInEnvs; + Counter nrValues; + Counter nrAttrsets; + Counter nrAttrsInAttrsets; + Counter nrListElems; + }; + + EvalMemory(); + + EvalMemory(const EvalMemory &) = delete; + EvalMemory(EvalMemory &&) = delete; + EvalMemory & operator=(const EvalMemory &) = delete; + EvalMemory & operator=(EvalMemory &&) = delete; + + inline Value * allocValue(); + inline Env & allocEnv(size_t size); + + Bindings * allocBindings(size_t capacity); + + BindingsBuilder buildBindings(SymbolTable & symbols, size_t capacity) + { + return BindingsBuilder(*this, symbols, allocBindings(capacity), capacity); + } + + ListBuilder buildList(size_t size) + { + stats.nrListElems += size; + return ListBuilder(size); + } + + const Statistics & getStats() const & + { + return stats; + } /** - * `true` constant. - * - * This is _not_ a singleton. Pointer equality is _not_ sufficient. + * Storage for the AST nodes */ - Value vTrue; + Exprs exprs; + +private: + Statistics stats; +}; + +class EvalState : public std::enable_shared_from_this +{ +public: + static constexpr StaticEvalSymbols s = StaticEvalSymbols::create(); + + const fetchers::Settings & fetchSettings; + const EvalSettings & settings; + + SymbolTable symbols; + PosTable positions; + + EvalMemory mem; /** - * `true` constant. - * - * This is _not_ a singleton. Pointer equality is _not_ sufficient. + * If set, force copying files to the Nix store even if they + * already exist there. */ - Value vFalse; - - /** `"regular"` */ - Value vStringRegular; - /** `"directory"` */ - Value vStringDirectory; - /** `"symlink"` */ - Value vStringSymlink; - /** `"unknown"` */ - Value vStringUnknown; + RepairFlag repair; /** * The accessor corresponding to `store`. */ - const ref storeFS; + const ref storeFS; /** * The accessor for the root filesystem. @@ -308,7 +421,7 @@ public: RootValue vImportedDrvToDerivation = nullptr; - ref inputCache; + const ref inputCache; /** * Debugger @@ -318,7 +431,9 @@ public: bool inDebugger = false; int trylevel; std::list debugTraces; - std::map> exprEnvs; + boost::unordered_flat_map> exprEnvs; + + ref asyncPathWriter; const std::shared_ptr getStaticEnv(const Expr & expr) const { @@ -361,58 +476,41 @@ private: /* Cache for calls to addToStore(); maps source paths to the store paths. */ - Sync> srcToStore; + ref> srcToStore; /** - * A cache from path names to parse trees. + * A cache that maps paths to "resolved" paths for importing Nix + * expressions, i.e. `/foo` to `/foo/default.nix`. */ - typedef std::unordered_map< - SourcePath, - Expr *, - std::hash, - std::equal_to, - traceable_allocator>> - FileParseCache; - FileParseCache fileParseCache; + ref> importResolutionCache; /** - * A cache from path names to values. + * A cache from resolved paths to values. */ - typedef std::unordered_map< + ref, std::equal_to, - traceable_allocator>> - FileEvalCache; - FileEvalCache fileEvalCache; + traceable_allocator>>> + fileEvalCache; /** * Associate source positions of certain AST nodes with their preceding doc comment, if they have one. * Grouped by file. */ - std::unordered_map positionToDocComment; + SharedSync>> positionToDocComment; LookupPath lookupPath; - std::map> lookupPathResolved; + // FIXME: make thread-safe. + boost::unordered_flat_map, StringViewHash, std::equal_to<>> + lookupPathResolved; /** * Cache used by prim_match(). */ - std::shared_ptr regexCache; - -#if NIX_USE_BOEHMGC - /** - * Allocation cache for GC'd Value objects. - */ - std::shared_ptr valueAllocCache; - - /** - * Allocation cache for size-1 Env objects. - */ - std::shared_ptr env1AllocCache; -#endif + ref regexCache; public: @@ -424,6 +522,15 @@ public: std::shared_ptr buildStore = nullptr); ~EvalState(); + /** + * A wrapper around EvalMemory::allocValue() to avoid code churn when it + * was introduced. + */ + inline Value * allocValue() + { + return mem.allocValue(); + } + LookupPath getLookupPath() { return lookupPath; @@ -451,8 +558,11 @@ public: /** * Allow access to a path. + * + * Only for restrict eval: pure eval just whitelist store paths, + * never arbitrary paths. */ - void allowPath(const Path & path); + void allowPathLegacy(const Path & path); /** * Allow access to a store path. Note that this gets remapped to @@ -472,6 +582,16 @@ public: void checkURI(const std::string & uri); + /** + * Mount an input on the Nix store. + */ + StorePath mountInput( + fetchers::Input & input, + const fetchers::Input & originalInput, + ref accessor, + bool requireLockable, + bool forceNarHash = false); + /** * Parse a Nix expression from the specified file. */ @@ -531,7 +651,10 @@ public: * application, call the function and overwrite `v` with the * result. Otherwise, this is a no-op. */ - inline void forceValue(Value & v, const PosIdx pos); + inline void forceValue(Value & v, const PosIdx pos) + { + v.force(*this, pos); + } void tryFixupBlackHolePos(Value & v, PosIdx pos); @@ -570,7 +693,7 @@ public: /** * Get attribute from an attribute set and throw an error if it doesn't exist. */ - Bindings::const_iterator getAttr(Symbol attrSym, const Bindings * attrSet, std::string_view errorCtx); + const Attr * getAttr(Symbol attrSym, const Bindings * attrSet, std::string_view errorCtx); template [[gnu::noinline]] @@ -589,6 +712,12 @@ public: std::optional tryAttrsToString( const PosIdx pos, Value & v, NixStringContext & context, bool coerceMore = false, bool copyToStore = true); + StorePath devirtualize(const StorePath & path, StringMap * rewrites = nullptr); + + SingleDerivedPath devirtualize(const SingleDerivedPath & path, StringMap * rewrites = nullptr); + + std::string devirtualize(std::string_view s, const NixStringContext & context); + /** * String coercion. * @@ -606,7 +735,19 @@ public: bool copyToStore = true, bool canonicalizePath = true); - StorePath copyPathToStore(NixStringContext & context, const SourcePath & path); + StorePath copyPathToStore(NixStringContext & context, const SourcePath & path, PosIdx pos); + + /** + * Compute the base name for a `SourcePath`. For non-store paths, + * this is just `SourcePath::baseName()`. But for store paths, for + * backwards compatibility, it needs to be `-source`, + * i.e. as if the path were copied to the Nix store. This results + * in a "double-copied" store path like + * `/nix/store/--source`. We don't need to + * materialize /nix/store/-source though. Still, this + * requires reading/hashing the path twice. + */ + std::string computeBaseName(const SourcePath & path, PosIdx pos); /** * Path coercion. @@ -669,11 +810,11 @@ public: /** * Internal primops not exposed to the user. */ - std::unordered_map< + boost::unordered_flat_map< std::string, Value *, - std::hash, - std::equal_to, + StringViewHash, + std::equal_to<>, traceable_allocator>> internalPrimOps; @@ -749,10 +890,11 @@ private: std::shared_ptr & staticEnv); /** - * Current Nix call stack depth, used with `max-call-depth` setting to throw stack overflow hopefully before we run - * out of system stack. + * Current Nix call stack depth, used with `max-call-depth` + * setting to throw stack overflow hopefully before we run out of + * system stack. */ - size_t callDepth = 0; + thread_local static size_t callDepth; public: @@ -792,22 +934,14 @@ public: */ void autoCallFunction(const Bindings & args, Value & fun, Value & res); - /** - * Allocation primitives. - */ - inline Value * allocValue(); - inline Env & allocEnv(size_t size); - - Bindings * allocBindings(size_t capacity); - BindingsBuilder buildBindings(size_t capacity) { - return BindingsBuilder(*this, allocBindings(capacity)); + return mem.buildBindings(symbols, capacity); } ListBuilder buildList(size_t size) { - return ListBuilder(*this, size); + return mem.buildList(size); } /** @@ -907,6 +1041,10 @@ public: DocComment getDocCommentForPos(PosIdx pos); + void waitForPath(const StorePath & path); + void waitForPath(const SingleDerivedPath & path); + void waitForAllPaths(); + private: /** @@ -924,26 +1062,30 @@ private: */ std::string mkSingleDerivedPathStringRaw(const SingleDerivedPath & p); - unsigned long nrEnvs = 0; - unsigned long nrValuesInEnvs = 0; - unsigned long nrValues = 0; - unsigned long nrListElems = 0; - unsigned long nrLookups = 0; - unsigned long nrAttrsets = 0; - unsigned long nrAttrsInAttrsets = 0; - unsigned long nrAvoided = 0; - unsigned long nrOpUpdates = 0; - unsigned long nrOpUpdateValuesCopied = 0; - unsigned long nrListConcats = 0; - unsigned long nrPrimOpCalls = 0; - unsigned long nrFunctionCalls = 0; + Counter nrLookups; + Counter nrAvoided; + Counter nrOpUpdates; + Counter nrOpUpdateValuesCopied; + Counter nrListConcats; + Counter nrPrimOpCalls; + Counter nrFunctionCalls; + +public: + Counter nrThunksAwaited; + Counter nrThunksAwaitedSlow; + Counter microsecondsWaiting; + Counter currentlyWaiting; + Counter maxWaiting; + Counter nrSpuriousWakeups; +private: bool countCalls; - typedef std::map PrimOpCalls; + // FIXME: make thread-safe. + typedef boost::unordered_flat_map> PrimOpCalls; PrimOpCalls primOpCalls; - typedef std::map FunctionCalls; + typedef boost::unordered_flat_map FunctionCalls; FunctionCalls functionCalls; /** Evaluation/call profiler. */ @@ -951,7 +1093,8 @@ private: void incrFunctionCall(ExprLambda * fun); - typedef std::map AttrSelects; + // FIXME: make thread-safe. + typedef boost::unordered_flat_map> AttrSelects; AttrSelects attrSelects; friend struct ExprOpUpdate; @@ -968,6 +1111,17 @@ private: friend struct Value; friend class ListBuilder; + +public: + /** + * Worker threads manager. + * + * Note: keep this last to ensure that it's destroyed first, so we + * don't have any background work items (e.g. from + * `builtins.parallel`) referring to a partially destroyed + * `EvalState`. + */ + ref executor; }; struct DebugTraceStacker diff --git a/src/libexpr/include/nix/expr/gc-small-vector.hh b/src/libexpr/include/nix/expr/gc-small-vector.hh index fdd80b2c784..95c028e5a37 100644 --- a/src/libexpr/include/nix/expr/gc-small-vector.hh +++ b/src/libexpr/include/nix/expr/gc-small-vector.hh @@ -26,4 +26,20 @@ using SmallValueVector = SmallVector; template using SmallTemporaryValueVector = SmallVector; +/** + * For functions where we do not expect deep recursion, we can use a sizable + * part of the stack a free allocation space. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t nonRecursiveStackReservation = 128; + +/** + * Functions that maybe applied to self-similar inputs, such as concatMap on a + * tree, should reserve a smaller part of the stack for allocation. + * + * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. + */ +constexpr size_t conservativeStackReservation = 16; + } // namespace nix diff --git a/src/libexpr/include/nix/expr/meson.build b/src/libexpr/include/nix/expr/meson.build index 04f8eaf71ea..58881506c91 100644 --- a/src/libexpr/include/nix/expr/meson.build +++ b/src/libexpr/include/nix/expr/meson.build @@ -10,6 +10,7 @@ config_pub_h = configure_file( headers = [ config_pub_h ] + files( 'attr-path.hh', 'attr-set.hh', + 'counter.hh', 'eval-cache.hh', 'eval-error.hh', 'eval-gc.hh', @@ -23,6 +24,7 @@ headers = [ config_pub_h ] + files( 'get-drvs.hh', 'json-to-value.hh', 'nixexpr.hh', + 'parallel-eval.hh', 'parser-state.hh', 'primops.hh', 'print-ambiguous.hh', diff --git a/src/libexpr/include/nix/expr/nixexpr.hh b/src/libexpr/include/nix/expr/nixexpr.hh index 49bd7a3b659..031ef508503 100644 --- a/src/libexpr/include/nix/expr/nixexpr.hh +++ b/src/libexpr/include/nix/expr/nixexpr.hh @@ -2,12 +2,16 @@ ///@file #include +#include #include +#include +#include #include "nix/expr/value.hh" #include "nix/expr/symbol-table.hh" #include "nix/expr/eval-error.hh" #include "nix/util/pos-idx.hh" +#include "nix/expr/counter.hh" namespace nix { @@ -76,9 +80,20 @@ struct AttrName : expr(e) {}; }; +static_assert(std::is_trivially_copy_constructible_v); + typedef std::vector AttrPath; -std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath); +std::string showAttrPath(const SymbolTable & symbols, std::span attrPath); + +class Exprs +{ + // FIXME: use std::pmr::monotonic_buffer_resource when parallel + // eval is disabled? + std::pmr::synchronized_pool_resource pool; +public: + std::pmr::polymorphic_allocator alloc{&pool}; +}; /* Abstract syntax of Nix expressions. */ @@ -89,7 +104,7 @@ struct Expr Symbol sub, lessThan, mul, div, or_, findFile, nixPath, body; }; - static unsigned long nrExprs; + static Counter nrExprs; Expr() { @@ -99,7 +114,16 @@ struct Expr virtual ~Expr() {}; virtual void show(const SymbolTable & symbols, std::ostream & str) const; virtual void bindVars(EvalState & es, const std::shared_ptr & env); + + /** Normal evaluation, implemented directly by all subclasses. */ virtual void eval(EvalState & state, Env & env, Value & v); + + /** + * Create a thunk for the delayed computation of the given expression + * in the given environment. But if the expression is a variable, + * then look it up right away. This significantly reduces the number + * of thunks allocated. + */ virtual Value * maybeThunk(EvalState & state, Env & env); virtual void setName(Symbol name); virtual void setDocComment(DocComment docComment) {}; @@ -152,13 +176,28 @@ struct ExprFloat : Expr struct ExprString : Expr { - std::string s; Value v; - ExprString(std::string && s) - : s(std::move(s)) + /** + * This is only for strings already allocated in our polymorphic allocator, + * or that live at least that long (e.g. c++ string literals) + */ + ExprString(const char * s) + { + v.mkStringNoCopy(s); + }; + + ExprString(std::pmr::polymorphic_allocator & alloc, std::string_view sv) { - v.mkString(this->s.data()); + auto len = sv.length(); + if (len == 0) { + v.mkStringNoCopy(""); + return; + } + char * s = alloc.allocate(len + 1); + sv.copy(s, len); + s[len] = '\0'; + v.mkStringNoCopy(s); }; Value * maybeThunk(EvalState & state, Env & env) override; @@ -168,14 +207,16 @@ struct ExprString : Expr struct ExprPath : Expr { ref accessor; - std::string s; Value v; - ExprPath(ref accessor, std::string s) + ExprPath(std::pmr::polymorphic_allocator & alloc, ref accessor, std::string_view sv) : accessor(accessor) - , s(std::move(s)) { - v.mkPath(&*accessor, this->s.c_str()); + auto len = sv.length(); + char * s = alloc.allocate(len + 1); + sv.copy(s, len); + s[len] = '\0'; + v.mkPath(&*accessor, s); } Value * maybeThunk(EvalState & state, Env & env) override; @@ -242,20 +283,33 @@ struct ExprInheritFrom : ExprVar struct ExprSelect : Expr { PosIdx pos; + uint32_t nAttrPath; Expr *e, *def; - AttrPath attrPath; - ExprSelect(const PosIdx & pos, Expr * e, AttrPath attrPath, Expr * def) + AttrName * attrPathStart; + + ExprSelect( + std::pmr::polymorphic_allocator & alloc, + const PosIdx & pos, + Expr * e, + std::span attrPath, + Expr * def) : pos(pos) + , nAttrPath(attrPath.size()) , e(e) , def(def) - , attrPath(std::move(attrPath)) {}; + , attrPathStart(alloc.allocate_object(nAttrPath)) + { + std::ranges::copy(attrPath, attrPathStart); + }; - ExprSelect(const PosIdx & pos, Expr * e, Symbol name) + ExprSelect(std::pmr::polymorphic_allocator & alloc, const PosIdx & pos, Expr * e, Symbol name) : pos(pos) + , nAttrPath(1) , e(e) , def(0) + , attrPathStart((alloc.allocate_object())) { - attrPath.push_back(AttrName(name)); + *attrPathStart = AttrName(name); }; PosIdx getPos() const override @@ -263,6 +317,11 @@ struct ExprSelect : Expr return pos; } + std::span getAttrPath() const + { + return {attrPathStart, nAttrPath}; + } + /** * Evaluate the `a.b.c` part of `a.b.c.d`. This exists mostly for the purpose of :doc in the repl. * @@ -280,10 +339,14 @@ struct ExprSelect : Expr struct ExprOpHasAttr : Expr { Expr * e; - AttrPath attrPath; - ExprOpHasAttr(Expr * e, AttrPath attrPath) + std::span attrPath; + + ExprOpHasAttr(std::pmr::polymorphic_allocator & alloc, Expr * e, std::vector attrPath) : e(e) - , attrPath(std::move(attrPath)) {}; + , attrPath({alloc.allocate_object(attrPath.size()), attrPath.size()}) + { + std::ranges::copy(attrPath, this->attrPath.begin()); + }; PosIdx getPos() const override { @@ -565,42 +628,54 @@ struct ExprOpNot : Expr COMMON_METHODS }; -#define MakeBinOp(name, s) \ - struct name : Expr \ - { \ - PosIdx pos; \ - Expr *e1, *e2; \ - name(Expr * e1, Expr * e2) \ - : e1(e1) \ - , e2(e2) {}; \ - name(const PosIdx & pos, Expr * e1, Expr * e2) \ - : pos(pos) \ - , e1(e1) \ - , e2(e2) {}; \ - void show(const SymbolTable & symbols, std::ostream & str) const override \ - { \ - str << "("; \ - e1->show(symbols, str); \ - str << " " s " "; \ - e2->show(symbols, str); \ - str << ")"; \ - } \ - void bindVars(EvalState & es, const std::shared_ptr & env) override \ - { \ - e1->bindVars(es, env); \ - e2->bindVars(es, env); \ - } \ - void eval(EvalState & state, Env & env, Value & v) override; \ - PosIdx getPos() const override \ - { \ - return pos; \ - } \ +#define MakeBinOpMembers(name, s) \ + PosIdx pos; \ + Expr *e1, *e2; \ + name(Expr * e1, Expr * e2) \ + : e1(e1) \ + , e2(e2){}; \ + name(const PosIdx & pos, Expr * e1, Expr * e2) \ + : pos(pos) \ + , e1(e1) \ + , e2(e2){}; \ + void show(const SymbolTable & symbols, std::ostream & str) const override \ + { \ + str << "("; \ + e1->show(symbols, str); \ + str << " " s " "; \ + e2->show(symbols, str); \ + str << ")"; \ + } \ + void bindVars(EvalState & es, const std::shared_ptr & env) override \ + { \ + e1->bindVars(es, env); \ + e2->bindVars(es, env); \ + } \ + void eval(EvalState & state, Env & env, Value & v) override; \ + PosIdx getPos() const override \ + { \ + return pos; \ + } + +#define MakeBinOp(name, s) \ + struct name : Expr \ + { \ + MakeBinOpMembers(name, s) \ }; -MakeBinOp(ExprOpEq, "==") MakeBinOp(ExprOpNEq, "!=") MakeBinOp(ExprOpAnd, "&&") MakeBinOp(ExprOpOr, "||") - MakeBinOp(ExprOpImpl, "->") MakeBinOp(ExprOpUpdate, "//") MakeBinOp(ExprOpConcatLists, "++") +MakeBinOp(ExprOpEq, "=="); +MakeBinOp(ExprOpNEq, "!="); +MakeBinOp(ExprOpAnd, "&&"); +MakeBinOp(ExprOpOr, "||"); +MakeBinOp(ExprOpImpl, "->"); +MakeBinOp(ExprOpConcatLists, "++"); - struct ExprConcatStrings : Expr +struct ExprOpUpdate : Expr +{ + MakeBinOpMembers(ExprOpUpdate, "//") +}; + +struct ExprConcatStrings : Expr { PosIdx pos; bool forceString; @@ -632,20 +707,6 @@ struct ExprPos : Expr COMMON_METHODS }; -/* only used to mark thunks as black holes. */ -struct ExprBlackHole : Expr -{ - void show(const SymbolTable & symbols, std::ostream & str) const override {} - - void eval(EvalState & state, Env & env, Value & v) override; - - void bindVars(EvalState & es, const std::shared_ptr & env) override {} - - [[noreturn]] static void throwInfiniteRecursionError(EvalState & state, Value & v); -}; - -extern ExprBlackHole eBlackHole; - /* Static environments are used to map variable names onto (level, displacement) pairs used to obtain the value of the variable at runtime. */ diff --git a/src/libexpr/include/nix/expr/parallel-eval.hh b/src/libexpr/include/nix/expr/parallel-eval.hh new file mode 100644 index 00000000000..4ccb3cfb843 --- /dev/null +++ b/src/libexpr/include/nix/expr/parallel-eval.hh @@ -0,0 +1,90 @@ +#pragma once + +#include +#include +#include +#include + +#include + +#include "nix/util/sync.hh" +#include "nix/util/logging.hh" +#include "nix/util/environment-variables.hh" +#include "nix/util/util.hh" +#include "nix/util/signals.hh" + +#if NIX_USE_BOEHMGC +# include +#endif + +namespace nix { + +struct Executor +{ + using work_t = std::function; + + struct Item + { + std::promise promise; + work_t work; + }; + + struct State + { + std::multimap queue; + std::vector threads; + }; + + std::atomic_bool quit{false}; + + const unsigned int evalCores; + + const bool enabled; + + const std::unique_ptr interruptCallback; + + Sync state_; + + std::condition_variable wakeup; + + static unsigned int getEvalCores(const EvalSettings & evalSettings); + + Executor(const EvalSettings & evalSettings); + + ~Executor(); + + void createWorker(State & state); + + void worker(); + + std::vector> spawn(std::vector> && items); + + static thread_local bool amWorkerThread; +}; + +struct FutureVector +{ + Executor & executor; + + struct State + { + std::vector> futures; + }; + + Sync state_; + + ~FutureVector(); + + // FIXME: add a destructor that cancels/waits for all futures. + + void spawn(std::vector> && work); + + void spawn(uint8_t prioPrefix, Executor::work_t && work) + { + spawn({{std::move(work), prioPrefix}}); + } + + void finishAll(); +}; + +} // namespace nix diff --git a/src/libexpr/include/nix/expr/parser-state.hh b/src/libexpr/include/nix/expr/parser-state.hh index dd99192c075..55dce30470b 100644 --- a/src/libexpr/include/nix/expr/parser-state.hh +++ b/src/libexpr/include/nix/expr/parser-state.hh @@ -24,7 +24,6 @@ struct StringToken } }; -// This type must be trivially copyable; see YYLTYPE_IS_TRIVIAL in parser.y. struct ParserLocation { int beginOffset; @@ -44,9 +43,6 @@ struct ParserLocation beginOffset = stashedBeginOffset; endOffset = stashedEndOffset; } - - /** Latest doc comment position, or 0. */ - int doc_comment_first_column, doc_comment_last_column; }; struct LexerState @@ -71,7 +67,7 @@ struct LexerState /** * @brief Maps some positions to a DocComment, where the comment is relevant to the location. */ - std::unordered_map & positionToDocComment; + DocCommentMap & positionToDocComment; PosTable & positions; PosTable::Origin origin; @@ -82,13 +78,14 @@ struct LexerState struct ParserState { const LexerState & lexerState; + std::pmr::polymorphic_allocator & alloc; SymbolTable & symbols; PosTable & positions; Expr * result; SourcePath basePath; PosTable::Origin origin; const ref rootFS; - const Expr::AstSymbols & s; + static constexpr Expr::AstSymbols s = StaticEvalSymbols::create().exprSymbols; const EvalSettings & settings; void dupAttr(const AttrPath & attrPath, const PosIdx pos, const PosIdx prevPos); @@ -327,7 +324,7 @@ ParserState::stripIndentation(const PosIdx pos, std::vectoremplace_back(i->first, new ExprString(std::move(s2))); + es2->emplace_back(i->first, new ExprString(alloc, s2)); } }; for (; i != es.end(); ++i, --n) { diff --git a/src/libexpr/include/nix/expr/primops.hh b/src/libexpr/include/nix/expr/primops.hh index 885a53e9aa1..6407ba84e50 100644 --- a/src/libexpr/include/nix/expr/primops.hh +++ b/src/libexpr/include/nix/expr/primops.hh @@ -8,22 +8,6 @@ namespace nix { -/** - * For functions where we do not expect deep recursion, we can use a sizable - * part of the stack a free allocation space. - * - * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. - */ -constexpr size_t nonRecursiveStackReservation = 128; - -/** - * Functions that maybe applied to self-similar inputs, such as concatMap on a - * tree, should reserve a smaller part of the stack for allocation. - * - * Note: this is expected to be multiplied by sizeof(Value), or about 24 bytes. - */ -constexpr size_t conservativeStackReservation = 16; - struct RegisterPrimOp { typedef std::vector PrimOps; diff --git a/src/libexpr/include/nix/expr/print-ambiguous.hh b/src/libexpr/include/nix/expr/print-ambiguous.hh index c0d811d4b93..e64f7f9bf8d 100644 --- a/src/libexpr/include/nix/expr/print-ambiguous.hh +++ b/src/libexpr/include/nix/expr/print-ambiguous.hh @@ -15,7 +15,6 @@ namespace nix { * * See: https://github.com/NixOS/nix/issues/9730 */ -void printAmbiguous( - Value & v, const SymbolTable & symbols, std::ostream & str, std::set * seen, int depth); +void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, int depth); } // namespace nix diff --git a/src/libexpr/include/nix/expr/symbol-table.hh b/src/libexpr/include/nix/expr/symbol-table.hh index ec1456e2d45..3d68def9919 100644 --- a/src/libexpr/include/nix/expr/symbol-table.hh +++ b/src/libexpr/include/nix/expr/symbol-table.hh @@ -2,12 +2,13 @@ ///@file #include + #include "nix/expr/value.hh" -#include "nix/util/chunked-vector.hh" #include "nix/util/error.hh" +#include "nix/util/sync.hh" #include -#include +#include namespace nix { @@ -16,18 +17,30 @@ class SymbolValue : protected Value friend class SymbolStr; friend class SymbolTable; - uint32_t size_; - uint32_t idx; - - SymbolValue() = default; - -public: operator std::string_view() const noexcept { - return {c_str(), size_}; + // The actual string is stored directly after the value. + return reinterpret_cast(this + 1); } }; +struct ContiguousArena +{ + const char * data; + const size_t maxSize; + + // Put this in a separate cache line to ensure that a thread + // adding a symbol doesn't slow down threads dereferencing symbols + // by invalidating the read-only `data` field. + alignas(64) std::atomic size{0}; + + ContiguousArena(size_t maxSize); + + size_t allocate(size_t bytes); +}; + +class StaticSymbolTable; + /** * Symbols have the property that they can be compared efficiently * (using an equality test), because the symbol table stores only one @@ -37,38 +50,43 @@ class Symbol { friend class SymbolStr; friend class SymbolTable; + friend class StaticSymbolTable; private: + /// The offset of the symbol in `SymbolTable::arena`. uint32_t id; - explicit Symbol(uint32_t id) noexcept + explicit constexpr Symbol(uint32_t id) noexcept : id(id) { } public: - Symbol() noexcept + constexpr Symbol() noexcept : id(0) { } [[gnu::always_inline]] - explicit operator bool() const noexcept + constexpr explicit operator bool() const noexcept { return id > 0; } - auto operator<=>(const Symbol other) const noexcept + /** + * The ID is a private implementation detail that should generally not be observed. However, we expose here just for + * sake of `switch...case`, which needs to dispatch on numbers. */ + [[gnu::always_inline]] + constexpr uint32_t getId() const noexcept { - return id <=> other.id; + return id; } - bool operator==(const Symbol other) const noexcept - { - return id == other.id; - } + constexpr auto operator<=>(const Symbol & other) const noexcept = default; friend class std::hash; + + constexpr static size_t alignment = alignof(SymbolValue); }; /** @@ -80,25 +98,20 @@ class SymbolStr { friend class SymbolTable; - constexpr static size_t chunkSize{8192}; - using SymbolValueStore = ChunkedVector; - const SymbolValue * s; struct Key { using HashType = boost::hash; - SymbolValueStore & store; std::string_view s; std::size_t hash; - std::pmr::polymorphic_allocator & alloc; + ContiguousArena & arena; - Key(SymbolValueStore & store, std::string_view s, std::pmr::polymorphic_allocator & stringAlloc) - : store(store) - , s(s) + Key(std::string_view s, ContiguousArena & arena) + : s(s) , hash(HashType{}(s)) - , alloc(stringAlloc) + , arena(arena) { } }; @@ -109,26 +122,7 @@ public: { } - SymbolStr(const Key & key) - { - auto size = key.s.size(); - if (size >= std::numeric_limits::max()) { - throw Error("Size of symbol exceeds 4GiB and cannot be stored"); - } - // for multi-threaded implementations: lock store and allocator here - const auto & [v, idx] = key.store.add(SymbolValue{}); - if (size == 0) { - v.mkString("", nullptr); - } else { - auto s = key.alloc.allocate(size + 1); - memcpy(s, key.s.data(), size); - s[size] = '\0'; - v.mkString(s, nullptr); - } - v.size_ = size; - v.idx = idx; - this->s = &v; - } + SymbolStr(const Key & key); bool operator==(std::string_view s2) const noexcept { @@ -151,13 +145,13 @@ public: [[gnu::always_inline]] bool empty() const noexcept { - return s->size_ == 0; + return static_cast(*s).empty(); } [[gnu::always_inline]] size_t size() const noexcept { - return s->size_; + return static_cast(*s).size(); } [[gnu::always_inline]] @@ -166,11 +160,6 @@ public: return s; } - explicit operator Symbol() const noexcept - { - return Symbol{s->idx + 1}; - } - struct Hash { using is_transparent = void; @@ -208,6 +197,47 @@ public: return operator()(b, a); } }; + + constexpr static size_t computeSize(std::string_view s) + { + auto rawSize = sizeof(Value) + s.size() + 1; + return ((rawSize + Symbol::alignment - 1) / Symbol::alignment) * Symbol::alignment; + } +}; + +class SymbolTable; + +/** + * Convenience class to statically assign symbol identifiers at compile-time. + */ +class StaticSymbolTable +{ + static constexpr std::size_t maxSize = 1024; + + struct StaticSymbolInfo + { + std::string_view str; + Symbol sym; + }; + + std::array symbols; + std::size_t size = 0; + std::size_t nextId = alignof(SymbolValue); + +public: + constexpr StaticSymbolTable() = default; + + constexpr Symbol create(std::string_view str) + { + /* No need to check bounds because out of bounds access is + a compilation error. */ + auto sym = Symbol(nextId); + symbols[size++] = {str, sym}; + nextId += SymbolStr::computeSize(str); + return sym; + } + + void copyIntoSymbolTable(SymbolTable & symtab) const; }; /** @@ -221,61 +251,86 @@ private: * SymbolTable is an append only data structure. * During its lifetime the monotonic buffer holds all strings and nodes, if the symbol set is node based. */ - std::pmr::monotonic_buffer_resource buffer; - std::pmr::polymorphic_allocator stringAlloc{&buffer}; - SymbolStr::SymbolValueStore store{16}; + ContiguousArena arena; /** - * Transparent lookup of string view for a pointer to a ChunkedVector entry -> return offset into the store. - * ChunkedVector references are never invalidated. + * Transparent lookup of string view for a pointer to a + * SymbolValue in the arena. */ - boost::unordered_flat_set symbols{SymbolStr::chunkSize}; + boost::concurrent_flat_set symbols; public: + SymbolTable(const StaticSymbolTable & staticSymtab) + : arena(1 << 30) + { + // Reserve symbol ID 0 and ensure alignment of the first allocation. + arena.allocate(Symbol::alignment); + + staticSymtab.copyIntoSymbolTable(*this); + } /** * Converts a string into a symbol. */ - Symbol create(std::string_view s) - { - // Most symbols are looked up more than once, so we trade off insertion performance - // for lookup performance. - // FIXME: make this thread-safe. - return Symbol(*symbols.insert(SymbolStr::Key{store, s, stringAlloc}).first); - } + Symbol create(std::string_view s); std::vector resolve(const std::vector & symbols) const { std::vector result; result.reserve(symbols.size()); - for (auto sym : symbols) + for (auto & sym : symbols) result.push_back((*this)[sym]); return result; } SymbolStr operator[](Symbol s) const { - uint32_t idx = s.id - uint32_t(1); - if (idx >= store.size()) - unreachable(); - return store[idx]; + assert(s.id); + // Note: we don't check arena.size here to avoid a dependency + // on other threads creating new symbols. + return SymbolStr(*reinterpret_cast(arena.data + s.id)); } - [[gnu::always_inline]] size_t size() const noexcept { - return store.size(); + return symbols.size(); } - size_t totalSize() const; + size_t totalSize() const + { + return arena.size; + } template void dump(T callback) const { - store.forEach(callback); + std::string_view left{arena.data, arena.size}; + left = left.substr(Symbol::alignment); + while (true) { + if (left.empty()) + break; + left = left.substr(sizeof(Value)); + auto p = left.find('\0'); + assert(p != left.npos); + auto sym = left.substr(0, p); + callback(sym); + // skip alignment padding + auto n = sym.size() + 1; + left = left.substr(n + (n % Symbol::alignment ? Symbol::alignment - (n % Symbol::alignment) : 0)); + } } }; +inline void StaticSymbolTable::copyIntoSymbolTable(SymbolTable & symtab) const +{ + for (std::size_t i = 0; i < size; ++i) { + auto [str, staticSym] = symbols[i]; + auto sym = symtab.create(str); + if (sym != staticSym) [[unlikely]] + unreachable(); + } +} + } // namespace nix template<> diff --git a/src/libexpr/include/nix/expr/value.hh b/src/libexpr/include/nix/expr/value.hh index a2833679bef..fc6dc557456 100644 --- a/src/libexpr/include/nix/expr/value.hh +++ b/src/libexpr/include/nix/expr/value.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include #include #include #include @@ -12,6 +13,7 @@ #include "nix/expr/print-options.hh" #include "nix/util/checked-arithmetic.hh" +#include #include namespace nix { @@ -19,6 +21,19 @@ namespace nix { struct Value; class BindingsBuilder; +static constexpr int discriminatorBits = 3; + +enum PrimaryDiscriminator : int { + pdSingleDWord = 0, + pdThunk = 1, + pdPending = 2, + pdAwaited = 3, + pdPairOfPointers = 4, + pdListN = 5, // FIXME: get rid of this by putting the size in the first word + pdString = 6, + pdPath = 7, // FIXME: get rid of this by ditching the `accessor` field +}; + /** * Internal type discriminator, which is more detailed than `ValueType`, as * it specifies the exact representation used (for types that have multiple @@ -29,27 +44,50 @@ class BindingsBuilder; * This also restricts the number of internal types represented with distinct memory layouts. */ typedef enum { - tUninitialized = 0, - /* layout: Single/zero field payload */ - tInt = 1, - tBool, - tNull, - tFloat, - tExternal, - tPrimOp, - tAttrs, - /* layout: Pair of pointers payload */ - tListSmall, - tPrimOpApp, - tApp, - tThunk, - tLambda, - /* layout: Single untaggable field */ - tListN, - tString, - tPath, + /* Values that have more type bits in the first word, and the + payload (a single word) in the second word. */ + tUninitialized = PrimaryDiscriminator::pdSingleDWord | (0 << discriminatorBits), + tInt = PrimaryDiscriminator::pdSingleDWord | (1 << discriminatorBits), + tFloat = PrimaryDiscriminator::pdSingleDWord | (2 << discriminatorBits), + tBool = PrimaryDiscriminator::pdSingleDWord | (3 << discriminatorBits), + tNull = PrimaryDiscriminator::pdSingleDWord | (4 << discriminatorBits), + tAttrs = PrimaryDiscriminator::pdSingleDWord | (5 << discriminatorBits), + tPrimOp = PrimaryDiscriminator::pdSingleDWord | (6 << discriminatorBits), + tFailed = PrimaryDiscriminator::pdSingleDWord | (7 << discriminatorBits), + tExternal = PrimaryDiscriminator::pdSingleDWord | (8 << discriminatorBits), + + /* Thunks. */ + tThunk = PrimaryDiscriminator::pdThunk | (0 << discriminatorBits), + tApp = PrimaryDiscriminator::pdThunk | (1 << discriminatorBits), + + tPending = PrimaryDiscriminator::pdPending, + tAwaited = PrimaryDiscriminator::pdAwaited, + + /* Values that consist of two pointers. The second word contains + more type bits in its alignment niche. */ + tListSmall = PrimaryDiscriminator::pdPairOfPointers | (0 << discriminatorBits), + tPrimOpApp = PrimaryDiscriminator::pdPairOfPointers | (1 << discriminatorBits), + tLambda = PrimaryDiscriminator::pdPairOfPointers | (2 << discriminatorBits), + + /* Special values. */ + tListN = PrimaryDiscriminator::pdListN, + tString = PrimaryDiscriminator::pdString, + tPath = PrimaryDiscriminator::pdPath, } InternalType; +/** + * Return true if `type` denotes a "finished" value, i.e. a weak-head + * normal form. + * + * Note that tPrimOpApp is considered "finished" because it represents + * a primop call with an incomplete number of arguments, and therefore + * cannot be evaluated further. + */ +inline bool isFinished(InternalType t) +{ + return t != tUninitialized && t != tThunk && t != tApp && t != tPending && t != tAwaited; +} + /** * This type abstracts over all actual value types in the language, * grouping together implementation details like tList*, different function @@ -57,6 +95,7 @@ typedef enum { */ typedef enum { nThunk, + nFailed, nInt, nFloat, nBool, @@ -73,7 +112,6 @@ class Bindings; struct Env; struct Expr; struct ExprLambda; -struct ExprBlackHole; struct PrimOp; class Symbol; class SymbolStr; @@ -154,7 +192,7 @@ class ListBuilder Value * inlineElems[2] = {nullptr, nullptr}; public: Value ** elems; - ListBuilder(EvalState & state, size_t size); + ListBuilder(size_t size); // NOTE: Can be noexcept because we are just copying integral values and // raw pointers. @@ -189,7 +227,7 @@ namespace detail { /** * Implementation mixin class for defining the public types - * In can be inherited from by the actual ValueStorage implementations + * In can be inherited by the actual ValueStorage implementations * for free due to Empty Base Class Optimization (EBCO). */ struct ValueBase @@ -265,6 +303,11 @@ struct ValueBase size_t size; Value * const * elems; }; + + struct Failed : gc + { + std::exception_ptr ex; + }; }; template @@ -291,6 +334,7 @@ struct PayloadTypeToInternalType MACRO(PrimOp *, primOp, tPrimOp) \ MACRO(ValueBase::PrimOpApplicationThunk, primOpApp, tPrimOpApp) \ MACRO(ExternalValueBase *, external, tExternal) \ + MACRO(ValueBase::Failed *, failed, tFailed) \ MACRO(NixFloat, fpoint, tFloat) #define NIX_VALUE_PAYLOAD_TYPE(T, FIELD_NAME, DISCRIMINATOR) \ @@ -369,7 +413,7 @@ namespace detail { /* Whether to use a specialization of ValueStorage that does bitpacking into alignment niches. */ template -inline constexpr bool useBitPackedValueStorage = (ptrSize == 8) && (__STDCPP_DEFAULT_NEW_ALIGNMENT__ >= 8); +inline constexpr bool useBitPackedValueStorage = (ptrSize == 8) && (__STDCPP_DEFAULT_NEW_ALIGNMENT__ >= 16); } // namespace detail @@ -378,7 +422,8 @@ inline constexpr bool useBitPackedValueStorage = (ptrSize == 8) && (__STDCPP_DEF * Packs discriminator bits into the pointer alignment niches. */ template -class ValueStorage>> : public detail::ValueBase +class alignas(16) ValueStorage>> + : public detail::ValueBase { /* Needs a dependent type name in order for member functions (and * potentially ill-formed bit casts) to be SFINAE'd out. @@ -393,12 +438,44 @@ class ValueStorage::type; - using Payload = std::array; - Payload payload = {}; - static constexpr int discriminatorBits = 3; + /** + * For multithreaded evaluation, we have to make sure that thunks/apps + * (the only mutable types of values) are updated in a safe way. A + * value can have the following states (see `force()`): + * + * * "thunk"/"app". When forced, this value transitions to + * "pending". The current thread will evaluate the + * thunk/app. When done, it will override the value with the + * result. If the value is at that point in the "awaited" state, + * the thread will wake up any waiting threads. + * + * * "pending". This means it's currently being evaluated. If + * another thread forces this value, it transitions to "awaited" + * and the thread will wait for the value to be updated (see + * `waitOnThunk()`). + * + * * "awaited". Like pending, only it means that there already are + * one or more threads waiting for this thunk. + * + * To ensure race-free access, the non-atomic word `p1` must + * always be updated before `p0`. Writes to `p0` should use + * *release* semantics (so that `p1` and any referenced values become + * visible to threads that read `p0`), and reads from `p0` should + * use `*acquire* semantics. + * + * Note: at some point, we may want to switch to 128-bit atomics + * so that `p0` and `p1` can be updated together + * atomically. However, 128-bit atomics are a bit problematic at + * present on x86_64 (see + * e.g. https://ibraheem.ca/posts/128-bit-atomics/). + */ + std::atomic p0{0}; + PackedPointer p1{0}; + static constexpr PackedPointer discriminatorMask = (PackedPointer(1) << discriminatorBits) - 1; + // FIXME: move/update /** * The value is stored as a pair of 8-byte double words. All pointers are assumed * to be 8-byte aligned. This gives us at most 6 bits of discriminator bits @@ -428,15 +505,6 @@ class ValueStorage requires std::is_pointer_v @@ -447,7 +515,7 @@ class ValueStorage(payload[0] & discriminatorMask); + return static_cast(p0 & discriminatorMask); } static void assertAligned(PackedPointer val) noexcept @@ -455,13 +523,30 @@ class ValueStorage(p0_ & discriminatorMask); + if (pd == pdPending) + // Nothing to do; no thread is waiting on this thunk. + ; + else if (pd == pdAwaited) + // Slow path: wake up the threads that are waiting on this + // thunk. + notifyWaiters(); + else if (pd == pdThunk) + unreachable(); + } + template void setSingleDWordPayload(PackedPointer untaggedVal) noexcept { - /* There's plenty of free upper bits in the first dword, which is - used only for the discriminator. */ - payload[0] = static_cast(pdSingleDWord) | (static_cast(type) << discriminatorBits); - payload[1] = untaggedVal; + /* There's plenty of free upper bits in the first byte, which + is used only for the discriminator. */ + finish(static_cast(type), untaggedVal); } template @@ -470,32 +555,42 @@ class ValueStorage= pdListN && discriminator <= pdPath); auto firstFieldPayload = std::bit_cast(firstPtrField); assertAligned(firstFieldPayload); - payload[0] = static_cast(discriminator) | firstFieldPayload; - payload[1] = std::bit_cast(untaggableField); + finish(static_cast(discriminator) | firstFieldPayload, std::bit_cast(untaggableField)); } template void setPairOfPointersPayload(T * firstPtrField, U * secondPtrField) noexcept { static_assert(type >= tListSmall && type <= tLambda); - { - auto firstFieldPayload = std::bit_cast(firstPtrField); - assertAligned(firstFieldPayload); - payload[0] = static_cast(pdPairOfPointers) | firstFieldPayload; - } - { - auto secondFieldPayload = std::bit_cast(secondPtrField); - assertAligned(secondFieldPayload); - payload[1] = (type - tListSmall) | secondFieldPayload; - } + auto firstFieldPayload = std::bit_cast(firstPtrField); + assertAligned(firstFieldPayload); + auto secondFieldPayload = std::bit_cast(secondPtrField); + assertAligned(secondFieldPayload); + finish( + static_cast(pdPairOfPointers) | firstFieldPayload, + ((type - tListSmall) >> discriminatorBits) | secondFieldPayload); + } + + template + void setThunkPayload(T * firstPtrField, U * secondPtrField) noexcept + { + static_assert(type >= tThunk && type <= tApp); + auto secondFieldPayload = std::bit_cast(secondPtrField); + assertAligned(secondFieldPayload); + p1 = ((type - tThunk) >> discriminatorBits) | secondFieldPayload; + auto firstFieldPayload = std::bit_cast(firstPtrField); + assertAligned(firstFieldPayload); + // Note: awaited values can never become a thunk, so no need + // to check for waiters. + p0.store(static_cast(pdThunk) | firstFieldPayload, std::memory_order_release); } template requires std::is_pointer_v && std::is_pointer_v void getPairOfPointersPayload(T & firstPtrField, U & secondPtrField) const noexcept { - firstPtrField = untagPointer(payload[0]); - secondPtrField = untagPointer(payload[1]); + firstPtrField = untagPointer(p0); + secondPtrField = untagPointer(p1); } protected: @@ -503,42 +598,45 @@ protected: InternalType getInternalType() const noexcept { switch (auto pd = getPrimaryDiscriminator()) { - case pdUninitialized: - /* Discriminator value of zero is used to distinguish uninitialized values. */ - return tUninitialized; case pdSingleDWord: - /* Payloads that only use up a single double word store the InternalType - in the upper bits of the first double word. */ - return InternalType(payload[0] >> discriminatorBits); + /* Payloads that only use up a single double word store + the full InternalType in the first byte. */ + return InternalType(p0 & 0xff); + case pdThunk: + return static_cast(tThunk + ((p1 & discriminatorMask) << discriminatorBits)); + case pdPending: + return tPending; + case pdAwaited: + return tAwaited; + case pdPairOfPointers: + return static_cast(tListSmall + ((p1 & discriminatorMask) << discriminatorBits)); /* The order must match that of the enumerations defined in InternalType. */ case pdListN: case pdString: case pdPath: return static_cast(tListN + (pd - pdListN)); - case pdPairOfPointers: - return static_cast(tListSmall + (payload[1] & discriminatorMask)); [[unlikely]] default: unreachable(); } } -#define NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(TYPE, MEMBER_A, MEMBER_B) \ - \ - void getStorage(TYPE & val) const noexcept \ - { \ - getPairOfPointersPayload(val MEMBER_A, val MEMBER_B); \ - } \ - \ - void setStorage(TYPE val) noexcept \ - { \ - setPairOfPointersPayload>(val MEMBER_A, val MEMBER_B); \ +#define NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(TYPE, SET, MEMBER_A, MEMBER_B) \ + \ + void getStorage(TYPE & val) const noexcept \ + { \ + getPairOfPointersPayload(val MEMBER_A, val MEMBER_B); \ + } \ + \ + void setStorage(TYPE val) noexcept \ + { \ + SET>(val MEMBER_A, val MEMBER_B); \ } - NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(SmallList, [0], [1]) - NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(PrimOpApplicationThunk, .left, .right) - NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(FunctionApplicationThunk, .left, .right) - NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(ClosureThunk, .env, .expr) - NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(Lambda, .env, .fun) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(SmallList, setPairOfPointersPayload, [0], [1]) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(PrimOpApplicationThunk, setPairOfPointersPayload, .left, .right) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(Lambda, setPairOfPointersPayload, .env, .fun) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(FunctionApplicationThunk, setThunkPayload, .left, .right) + NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS(ClosureThunk, setThunkPayload, .env, .expr) #undef NIX_VALUE_STORAGE_DEF_PAIR_OF_PTRS @@ -546,52 +644,57 @@ protected: { /* PackedPointerType -> int64_t here is well-formed, since the standard requires this conversion to follow 2's complement rules. This is just a no-op. */ - integer = NixInt(payload[1]); + integer = NixInt(p1); } void getStorage(bool & boolean) const noexcept { - boolean = payload[1]; + boolean = p1; } void getStorage(Null & null) const noexcept {} void getStorage(NixFloat & fpoint) const noexcept { - fpoint = std::bit_cast(payload[1]); + fpoint = std::bit_cast(p1); } void getStorage(ExternalValueBase *& external) const noexcept { - external = std::bit_cast(payload[1]); + external = std::bit_cast(p1); } void getStorage(PrimOp *& primOp) const noexcept { - primOp = std::bit_cast(payload[1]); + primOp = std::bit_cast(p1); } void getStorage(Bindings *& attrs) const noexcept { - attrs = std::bit_cast(payload[1]); + attrs = std::bit_cast(p1); } void getStorage(List & list) const noexcept { - list.elems = untagPointer(payload[0]); - list.size = payload[1]; + list.elems = untagPointer(p0); + list.size = p1; } void getStorage(StringWithContext & string) const noexcept { - string.context = untagPointer(payload[0]); - string.c_str = std::bit_cast(payload[1]); + string.context = untagPointer(p0); + string.c_str = std::bit_cast(p1); } void getStorage(Path & path) const noexcept { - path.accessor = untagPointer(payload[0]); - path.path = std::bit_cast(payload[1]); + path.accessor = untagPointer(p0); + path.path = std::bit_cast(p1); + } + + void getStorage(Failed *& failed) const noexcept + { + failed = std::bit_cast(p1); } void setStorage(NixInt integer) noexcept @@ -643,8 +746,85 @@ protected: { setUntaggablePayload(path.accessor, path.path); } + + void setStorage(Failed * failed) noexcept + { + setSingleDWordPayload(std::bit_cast(failed)); + } + + ValueStorage() {} + + ValueStorage(const ValueStorage & v) + { + *this = v; + } + + /** + * Copy a value. This is not allowed to be a thunk to avoid + * accidental work duplication. + */ + ValueStorage & operator=(const ValueStorage & v) + { + auto p0_ = v.p0.load(std::memory_order_acquire); + auto p1_ = v.p1; // must be loaded after p0 + auto pd = static_cast(p0_ & discriminatorMask); + if (pd == pdThunk || pd == pdPending || pd == pdAwaited) + unreachable(); + finish(p0_, p1_); + return *this; + } + +public: + + /** + * Check whether forcing this value requires a trivial amount of + * computation. A value is trivial if it's finished or if it's a + * thunk whose expression is an attrset with no dynamic + * attributes, a lambda or a list. Note that it's up to the caller + * to check whether the members of those attrsets or lists must be + * trivial. + */ + bool isTrivial() const; + + inline void reset() + { + p1 = 0; + p0.store(0, std::memory_order_relaxed); + } + + /// Only used for testing. + inline void mkBlackhole() + { + p0.store(pdPending, std::memory_order_relaxed); + } + + void force(EvalState & state, PosIdx pos); + +private: + + /** + * Given a thunk that was observed to be in the pending or awaited + * state, wait for it to finish. Returns the first word of the + * value. + */ + PackedPointer waitOnThunk(EvalState & state, PackedPointer p0); + + /** + * Wake up any threads that are waiting on this value. + */ + void notifyWaiters(); }; +template<> +void ValueStorage::notifyWaiters(); + +template<> +ValueStorage::PackedPointer +ValueStorage::waitOnThunk(EvalState & state, PackedPointer p0); + +template<> +bool ValueStorage::isTrivial() const; + /** * View into a list of Value * that is itself immutable. * @@ -832,6 +1012,35 @@ struct Value : public ValueStorage { friend std::string showType(const Value & v); + /** + * Empty list constant. + * + * This is _not_ a singleton. Pointer equality is _not_ sufficient. + */ + static Value vEmptyList; + + /** + * `null` constant. + * + * This is _not_ a singleton. Pointer equality is _not_ sufficient. + */ + static Value vNull; + + /** + * `true` constant. + * + * This is _not_ a singleton. Pointer equality is _not_ sufficient. + */ + static Value vTrue; + + /** + * `true` constant. + * + * This is _not_ a singleton. Pointer equality is _not_ sufficient. + */ + static Value vFalse; + +private: template bool isa() const noexcept { @@ -857,47 +1066,58 @@ public: void print(EvalState & state, std::ostream & str, PrintOptions options = PrintOptions{}); + // FIXME: optimize, only look at first word + inline bool isFinished() const + { + return nix::isFinished(getInternalType()); + } + // Functions needed to distinguish the type // These should be removed eventually, by putting the functionality that's // needed by callers into methods of this type - // type() == nThunk inline bool isThunk() const { return isa(); - }; + } inline bool isApp() const { return isa(); - }; + } - inline bool isBlackhole() const; + inline bool isBlackhole() const + { + auto t = getInternalType(); + return t == tPending || t == tAwaited; + } // type() == nFunction inline bool isLambda() const { return isa(); - }; + } inline bool isPrimOp() const { return isa(); - }; + } inline bool isPrimOpApp() const { return isa(); - }; + } + + inline bool isFailed() const + { + return isa(); + } /** * Returns the normal type of a Value. This only returns nThunk if * the Value hasn't been forceValue'd - * - * @param invalidIsThunk Instead of aborting an an invalid (probably - * 0, so uninitialized) internal type, return `nThunk`. */ - inline ValueType type(bool invalidIsThunk = false) const + inline ValueType type() const { switch (getInternalType()) { case tUninitialized: @@ -925,14 +1145,15 @@ public: return nExternal; case tFloat: return nFloat; + case tFailed: + return nFailed; case tThunk: case tApp: + case tPending: + case tAwaited: return nThunk; } - if (invalidIsThunk) - return nThunk; - else - unreachable(); + unreachable(); } /** @@ -960,7 +1181,7 @@ public: setStorage(b); } - inline void mkString(const char * s, const char ** context = 0) noexcept + void mkStringNoCopy(const char * s, const char ** context = 0) noexcept { setStorage(StringWithContext{.c_str = s, .context = context}); } @@ -972,7 +1193,6 @@ public: void mkStringMove(const char * s, const NixStringContext & context); void mkPath(const SourcePath & path); - void mkPath(std::string_view path); inline void mkPath(SourceAccessor * accessor, const char * path) noexcept { @@ -993,12 +1213,20 @@ public: void mkList(const ListBuilder & builder) noexcept { - if (builder.size == 1) + switch (builder.size) { + case 0: + setStorage(List{.size = 0, .elems = nullptr}); + break; + case 1: setStorage(std::array{builder.inlineElems[0], nullptr}); - else if (builder.size == 2) + break; + case 2: setStorage(std::array{builder.inlineElems[0], builder.inlineElems[1]}); - else + break; + default: setStorage(List{.size = builder.size, .elems = builder.elems}); + break; + } } inline void mkThunk(Env * e, Expr * ex) noexcept @@ -1016,8 +1244,6 @@ public: setStorage(Lambda{.env = e, .fun = f}); } - inline void mkBlackhole(); - void mkPrimOp(PrimOp * p); inline void mkPrimOpApp(Value * l, Value * r) noexcept @@ -1040,6 +1266,11 @@ public: setStorage(n); } + inline void mkFailed() noexcept + { + setStorage(new Value::Failed{.ex = std::current_exception()}); + } + bool isList() const noexcept { return isa(); @@ -1057,13 +1288,6 @@ public: PosIdx determinePos(const PosIdx pos) const; - /** - * Check whether forcing this value requires a trivial amount of - * computation. In particular, function applications are - * non-trivial. - */ - bool isTrivial() const; - SourcePath path() const { return SourcePath(ref(pathAccessor()->shared_from_this()), CanonPath(CanonPath::unchecked_t(), pathStr())); @@ -1119,6 +1343,7 @@ public: return getStorage(); } + // FIXME: remove this since reading it is racy. ClosureThunk thunk() const noexcept { return getStorage(); @@ -1129,6 +1354,7 @@ public: return getStorage(); } + // FIXME: remove this since reading it is racy. FunctionApplicationThunk app() const noexcept { return getStorage(); @@ -1143,22 +1369,15 @@ public: { return getStorage().accessor; } -}; - -extern ExprBlackHole eBlackHole; -bool Value::isBlackhole() const -{ - return isThunk() && thunk().expr == (Expr *) &eBlackHole; -} - -void Value::mkBlackhole() -{ - mkThunk(nullptr, (Expr *) &eBlackHole); -} + Failed * failed() const noexcept + { + return getStorage(); + } +}; typedef std::vector> ValueVector; -typedef std::unordered_map< +typedef boost::unordered_flat_map< Symbol, Value *, std::hash, diff --git a/src/libexpr/include/nix/expr/value/context.hh b/src/libexpr/include/nix/expr/value/context.hh index dcfacbb214b..bb7e8e72790 100644 --- a/src/libexpr/include/nix/expr/value/context.hh +++ b/src/libexpr/include/nix/expr/value/context.hh @@ -56,7 +56,31 @@ struct NixStringContextElem */ using Built = SingleDerivedPath::Built; - using Raw = std::variant; + /** + * A store path that will not result in a store reference when + * used in a derivation or toFile. + * + * When you apply `builtins.toString` to a path value representing + * a path in the Nix store (as is the case with flake inputs), + * historically you got a string without context + * (e.g. `/nix/store/...-source`). This is broken, since it allows + * you to pass a store path to a derivation/toFile without a + * proper store reference. This is especially a problem with lazy + * trees, since the store path is a virtual path that doesn't + * exist. + * + * For backwards compatibility, and to warn users about this + * unsafe use of `toString`, we keep track of such strings as a + * special type of context. + */ + struct Path + { + StorePath storePath; + + GENERATE_CMP(Path, me->storePath); + }; + + using Raw = std::variant; Raw raw; @@ -79,4 +103,10 @@ struct NixStringContextElem typedef std::set NixStringContext; +/** + * Returns false if `context` has no elements other than + * `NixStringContextElem::Path`. + */ +bool hasContext(const NixStringContext & context); + } // namespace nix diff --git a/src/libexpr/lexer-helpers.cc b/src/libexpr/lexer-helpers.cc index 927e3cc7324..59f6f6f70df 100644 --- a/src/libexpr/lexer-helpers.cc +++ b/src/libexpr/lexer-helpers.cc @@ -1,11 +1,11 @@ #include "lexer-helpers.hh" -void nix::lexer::internal::initLoc(YYLTYPE * loc) +void nix::lexer::internal::initLoc(Parser::location_type * loc) { loc->beginOffset = loc->endOffset = 0; } -void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len) +void nix::lexer::internal::adjustLoc(yyscan_t yyscanner, Parser::location_type * loc, const char * s, size_t len) { loc->stash(); diff --git a/src/libexpr/lexer-helpers.hh b/src/libexpr/lexer-helpers.hh index 49865f79440..b60fb9e7d98 100644 --- a/src/libexpr/lexer-helpers.hh +++ b/src/libexpr/lexer-helpers.hh @@ -2,16 +2,12 @@ #include -// including the generated headers twice leads to errors -#ifndef BISON_HEADER -# include "lexer-tab.hh" -# include "parser-tab.hh" -#endif +#include "parser-scanner-decls.hh" namespace nix::lexer::internal { -void initLoc(YYLTYPE * loc); +void initLoc(Parser::location_type * loc); -void adjustLoc(yyscan_t yyscanner, YYLTYPE * loc, const char * s, size_t len); +void adjustLoc(yyscan_t yyscanner, Parser::location_type * loc, const char * s, size_t len); } // namespace nix::lexer::internal diff --git a/src/libexpr/lexer.l b/src/libexpr/lexer.l index 1005f9f7ea5..f420fc13f34 100644 --- a/src/libexpr/lexer.l +++ b/src/libexpr/lexer.l @@ -82,6 +82,10 @@ static void requireExperimentalFeature(const ExperimentalFeature & feature, cons } +using enum nix::Parser::token::token_kind_type; +using YYSTYPE = nix::Parser::value_type; +using YYLTYPE = nix::Parser::location_type; + // yacc generates code that uses unannotated fallthrough. #pragma GCC diagnostic ignored "-Wimplicit-fallthrough" diff --git a/src/libexpr/meson.build b/src/libexpr/meson.build index 0331d3c6116..fc5acd118dc 100644 --- a/src/libexpr/meson.build +++ b/src/libexpr/meson.build @@ -40,7 +40,11 @@ endforeach boost = dependency( 'boost', - modules : [ 'container', 'context' ], + modules : [ + 'container', + 'context', + 'thread', + ], include_type : 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we @@ -50,17 +54,25 @@ deps_other += boost nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json -bdw_gc = dependency('bdw-gc', required : get_option('gc')) +bdw_gc_required = get_option('gc').disable_if( + 'address' in get_option('b_sanitize'), + error_message : 'Building with Boehm GC and ASAN is not supported', +) + +bdw_gc = dependency('bdw-gc', required : bdw_gc_required) if bdw_gc.found() deps_public += bdw_gc foreach funcspec : [ - 'pthread_attr_get_np', 'pthread_getattr_np', ] define_name = 'HAVE_' + funcspec.underscorify().to_upper() define_value = cxx.has_function(funcspec).to_int() configdata_priv.set(define_name, define_value) endforeach + if host_machine.system() == 'cygwin' + # undefined reference to `__wrap__Znwm' + configdata_pub.set('GC_NO_INLINE_STD_NEW', 1) + endif endif # Used in public header. Affects ABI! configdata_pub.set('NIX_USE_BOEHMGC', bdw_gc.found().to_int()) @@ -85,6 +97,7 @@ config_priv_h = configure_file( ) subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') parser_tab = custom_target( input : 'parser.y', @@ -153,13 +166,16 @@ sources = files( 'json-to-value.cc', 'lexer-helpers.cc', 'nixexpr.cc', + 'parallel-eval.cc', 'paths.cc', 'primops.cc', 'print-ambiguous.cc', 'print.cc', 'search-path.cc', + 'symbol-table.cc', 'value-to-json.cc', 'value-to-xml.cc', + 'value.cc', 'value/context.cc', ) @@ -177,6 +193,7 @@ this_library = library( parser_tab, lexer_tab, generated_headers, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, diff --git a/src/libexpr/nixexpr.cc b/src/libexpr/nixexpr.cc index c0a25d1d4d6..93853fc86a7 100644 --- a/src/libexpr/nixexpr.cc +++ b/src/libexpr/nixexpr.cc @@ -11,9 +11,7 @@ namespace nix { -unsigned long Expr::nrExprs = 0; - -ExprBlackHole eBlackHole; +Counter Expr::nrExprs; // FIXME: remove, because *symbols* are abstract and do not have a single // textual representation; see printIdentifier() @@ -40,12 +38,12 @@ void ExprFloat::show(const SymbolTable & symbols, std::ostream & str) const void ExprString::show(const SymbolTable & symbols, std::ostream & str) const { - printLiteralString(str, s); + printLiteralString(str, v.string_view()); } void ExprPath::show(const SymbolTable & symbols, std::ostream & str) const { - str << s; + str << v.pathStr(); } void ExprVar::show(const SymbolTable & symbols, std::ostream & str) const @@ -57,7 +55,7 @@ void ExprSelect::show(const SymbolTable & symbols, std::ostream & str) const { str << "("; e->show(symbols, str); - str << ")." << showAttrPath(symbols, attrPath); + str << ")." << showAttrPath(symbols, getAttrPath()); if (def) { str << " or ("; def->show(symbols, str); @@ -261,7 +259,7 @@ void ExprPos::show(const SymbolTable & symbols, std::ostream & str) const str << "__curPos"; } -std::string showAttrPath(const SymbolTable & symbols, const AttrPath & attrPath) +std::string showAttrPath(const SymbolTable & symbols, std::span attrPath) { std::ostringstream out; bool first = true; @@ -362,7 +360,7 @@ void ExprSelect::bindVars(EvalState & es, const std::shared_ptr e->bindVars(es, env); if (def) def->bindVars(es, env); - for (auto & i : attrPath) + for (auto & i : getAttrPath()) if (!i.symbol) i.expr->bindVars(es, env); } @@ -605,15 +603,6 @@ void ExprLambda::setDocComment(DocComment docComment) // belongs in the same conditional. body->setDocComment(docComment); } -}; - -/* Symbol table. */ - -size_t SymbolTable::totalSize() const -{ - size_t n = 0; - dump([&](SymbolStr s) { n += s.size(); }); - return n; } std::string DocComment::getInnerText(const PosTable & positions) const diff --git a/src/libexpr/package.nix b/src/libexpr/package.nix index a67a8cc49ab..683e63d2cd7 100644 --- a/src/libexpr/package.nix +++ b/src/libexpr/package.nix @@ -36,7 +36,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-expr"; + pname = "determinate-nix-expr"; inherit version; workDir = ./.; diff --git a/src/libexpr/parallel-eval.cc b/src/libexpr/parallel-eval.cc new file mode 100644 index 00000000000..d63e931845e --- /dev/null +++ b/src/libexpr/parallel-eval.cc @@ -0,0 +1,299 @@ +#include "nix/expr/eval.hh" +#include "nix/expr/parallel-eval.hh" +#include "nix/store/globals.hh" +#include "nix/expr/primops.hh" + +namespace nix { + +// cache line alignment to prevent false sharing +struct alignas(64) WaiterDomain +{ + std::condition_variable cv; +}; + +static std::array, 128> waiterDomains; + +thread_local bool Executor::amWorkerThread{false}; + +unsigned int Executor::getEvalCores(const EvalSettings & evalSettings) +{ + return evalSettings.evalCores == 0UL ? Settings::getDefaultCores() : evalSettings.evalCores; +} + +Executor::Executor(const EvalSettings & evalSettings) + : evalCores(getEvalCores(evalSettings)) + , enabled(evalCores > 1) + , interruptCallback(createInterruptCallback([&]() { + for (auto & domain : waiterDomains) + domain.lock()->cv.notify_all(); + })) +{ + debug("executor using %d threads", evalCores); + auto state(state_.lock()); + for (size_t n = 0; n < evalCores; ++n) + createWorker(*state); +} + +Executor::~Executor() +{ + std::vector threads; + { + auto state(state_.lock()); + quit = true; + std::swap(threads, state->threads); + debug("executor shutting down with %d items left", state->queue.size()); + } + + wakeup.notify_all(); + + for (auto & thr : threads) + thr.join(); +} + +void Executor::createWorker(State & state) +{ + boost::thread::attributes attrs; + attrs.set_stack_size(evalStackSize); + state.threads.push_back(boost::thread(attrs, [&]() { +#if NIX_USE_BOEHMGC + GC_stack_base sb; + GC_get_stack_base(&sb); + GC_register_my_thread(&sb); +#endif + worker(); +#if NIX_USE_BOEHMGC + GC_unregister_my_thread(); +#endif + })); +} + +void Executor::worker() +{ + ReceiveInterrupts receiveInterrupts; + + unix::interruptCheck = [&]() { return (bool) quit; }; + + amWorkerThread = true; + + while (true) { + Item item; + + while (true) { + auto state(state_.lock()); + if (quit) { + // Set an `Interrupted` exception on all promises so + // we get a nicer error than "std::future_error: + // Broken promise". + auto ex = std::make_exception_ptr(Interrupted("interrupted by the user")); + for (auto & item : state->queue) + item.second.promise.set_exception(ex); + state->queue.clear(); + return; + } + if (!state->queue.empty()) { + item = std::move(state->queue.begin()->second); + state->queue.erase(state->queue.begin()); + break; + } + state.wait(wakeup); + } + + try { + item.work(); + item.promise.set_value(); + } catch (const Interrupted &) { + quit = true; + item.promise.set_exception(std::current_exception()); + } catch (...) { + item.promise.set_exception(std::current_exception()); + } + } +} + +std::vector> Executor::spawn(std::vector> && items) +{ + if (items.empty()) + return {}; + + std::vector> futures; + + { + auto state(state_.lock()); + for (auto & item : items) { + std::promise promise; + futures.push_back(promise.get_future()); + thread_local std::random_device rd; + thread_local std::uniform_int_distribution dist(0, 1ULL << 48); + auto key = (uint64_t(item.second) << 48) | dist(rd); + state->queue.emplace(key, Item{.promise = std::move(promise), .work = std::move(item.first)}); + } + } + + if (items.size() == 1) + wakeup.notify_one(); + else + wakeup.notify_all(); + + return futures; +} + +FutureVector::~FutureVector() +{ + try { + finishAll(); + } catch (...) { + ignoreExceptionInDestructor(); + } +} + +void FutureVector::spawn(std::vector> && work) +{ + auto futures = executor.spawn(std::move(work)); + auto state(state_.lock()); + for (auto & future : futures) + state->futures.push_back(std::move(future)); +} + +void FutureVector::finishAll() +{ + std::exception_ptr ex; + while (true) { + std::vector> futures; + { + auto state(state_.lock()); + std::swap(futures, state->futures); + } + debug("got %d futures", futures.size()); + if (futures.empty()) + break; + for (auto & future : futures) + try { + future.get(); + } catch (...) { + if (ex) { + if (!getInterrupted()) + ignoreExceptionExceptInterrupt(); + } else + ex = std::current_exception(); + } + } + if (ex) + std::rethrow_exception(ex); +} + +static Sync & getWaiterDomain(detail::ValueBase & v) +{ + auto domain = (((size_t) &v) >> 5) % waiterDomains.size(); + return waiterDomains[domain]; +} + +static std::atomic nextEvalThreadId{1}; +thread_local uint32_t myEvalThreadId(nextEvalThreadId++); + +template<> +ValueStorage::PackedPointer +ValueStorage::waitOnThunk(EvalState & state, PackedPointer expectedP0) +{ + state.nrThunksAwaited++; + + auto domain = getWaiterDomain(*this).lock(); + + auto threadId = expectedP0 >> discriminatorBits; + + if (static_cast(expectedP0 & discriminatorMask) == pdAwaited) { + /* Make sure that the value is still awaited, now that we're + holding the domain lock. */ + auto p0_ = p0.load(std::memory_order_acquire); + auto pd = static_cast(p0_ & discriminatorMask); + + /* If the value has been finalized in the meantime (i.e. is no + longer pending), we're done. */ + if (pd != pdAwaited) { + assert(pd != pdThunk && pd != pdPending); + return p0_; + } + } else { + /* Mark this value as being waited on. */ + PackedPointer p0_ = expectedP0; + if (!p0.compare_exchange_strong( + p0_, + pdAwaited | (threadId << discriminatorBits), + std::memory_order_acquire, + std::memory_order_acquire)) { + /* If the value has been finalized in the meantime (i.e. is + no longer pending), we're done. */ + auto pd = static_cast(p0_ & discriminatorMask); + if (pd != pdAwaited) { + assert(pd != pdThunk && pd != pdPending); + return p0_; + } + /* The value was already in the "waited on" state, so we're + not the only thread waiting on it. */ + } + } + + /* Wait for another thread to finish this value. */ + if (threadId == myEvalThreadId) + state.error("infinite recursion encountered") + .atPos(((Value &) *this).determinePos(noPos)) + .debugThrow(); + + state.nrThunksAwaitedSlow++; + state.currentlyWaiting++; + state.maxWaiting = std::max(state.maxWaiting, state.currentlyWaiting); + + auto now1 = std::chrono::steady_clock::now(); + + while (true) { + domain.wait(domain->cv); + auto p0_ = p0.load(std::memory_order_acquire); + auto pd = static_cast(p0_ & discriminatorMask); + if (pd != pdAwaited) { + assert(pd != pdThunk && pd != pdPending); + auto now2 = std::chrono::steady_clock::now(); + state.microsecondsWaiting += std::chrono::duration_cast(now2 - now1).count(); + state.currentlyWaiting--; + return p0_; + } + state.nrSpuriousWakeups++; + checkInterrupt(); + } +} + +template<> +void ValueStorage::notifyWaiters() +{ + auto domain = getWaiterDomain(*this).lock(); + + domain->cv.notify_all(); +} + +static void prim_parallel(EvalState & state, const PosIdx pos, Value ** args, Value & v) +{ + state.forceList(*args[0], pos, "while evaluating the first argument passed to builtins.parallel"); + + if (state.executor->evalCores > 1) { + std::vector> work; + for (auto value : args[0]->listView()) + if (!value->isFinished()) + work.emplace_back([value(allocRootValue(value)), &state, pos]() { state.forceValue(**value, pos); }, 0); + state.executor->spawn(std::move(work)); + } + + state.forceValue(*args[1], pos); + v = *args[1]; +} + +// FIXME: gate this behind an experimental feature. +static RegisterPrimOp r_parallel({ + .name = "__parallel", + .args = {"xs", "x"}, + .arity = 2, + .doc = R"( + Start evaluation of the values `xs` in the background and return `x`. + )", + .fun = prim_parallel, + .experimentalFeature = Xp::ParallelEval, +}); + +} // namespace nix diff --git a/src/libexpr/parser-scanner-decls.hh b/src/libexpr/parser-scanner-decls.hh new file mode 100644 index 00000000000..e4e06188334 --- /dev/null +++ b/src/libexpr/parser-scanner-decls.hh @@ -0,0 +1,17 @@ +#pragma once + +#ifndef BISON_HEADER +# include "parser-tab.hh" +using YYSTYPE = nix::parser::BisonParser::value_type; +using YYLTYPE = nix::parser::BisonParser::location_type; +# include "lexer-tab.hh" // IWYU pragma: export +#endif + +namespace nix { + +class Parser : public parser::BisonParser +{ + using BisonParser::BisonParser; +}; + +} // namespace nix diff --git a/src/libexpr/parser.y b/src/libexpr/parser.y index 2b2566208ff..9186fcf4b3f 100644 --- a/src/libexpr/parser.y +++ b/src/libexpr/parser.y @@ -1,5 +1,7 @@ +%skeleton "lalr1.cc" %define api.location.type { ::nix::ParserLocation } -%define api.pure +%define api.namespace { ::nix::parser } +%define api.parser.class { BisonParser } %locations %define parse.error verbose %defines @@ -26,19 +28,12 @@ #include "nix/expr/eval-settings.hh" #include "nix/expr/parser-state.hh" -// Bison seems to have difficulty growing the parser stack when using C++ with -// a custom location type. This undocumented macro tells Bison that our -// location type is "trivially copyable" in C++-ese, so it is safe to use the -// same memcpy macro it uses to grow the stack that it uses with its own -// default location type. Without this, we get "error: memory exhausted" when -// parsing some large Nix files. Our other options are to increase the initial -// stack size (200 by default) to be as large as we ever want to support (so -// that growing the stack is unnecessary), or redefine the stack-relocation -// macro ourselves (which is also undocumented). -#define YYLTYPE_IS_TRIVIAL 1 - -#define YY_DECL int yylex \ - (YYSTYPE * yylval_param, YYLTYPE * yylloc_param, yyscan_t yyscanner, nix::ParserState * state) +#define YY_DECL \ + int yylex( \ + nix::Parser::value_type * yylval_param, \ + nix::Parser::location_type * yylloc_param, \ + yyscan_t yyscanner, \ + nix::ParserState * state) // For efficiency, we only track offsets; not line,column coordinates # define YYLLOC_DEFAULT(Current, Rhs, N) \ @@ -57,19 +52,19 @@ namespace nix { -typedef std::unordered_map DocCommentMap; +typedef boost::unordered_flat_map> DocCommentMap; Expr * parseExprFromBuf( char * text, size_t length, Pos::Origin origin, const SourcePath & basePath, + std::pmr::polymorphic_allocator & alloc, SymbolTable & symbols, const EvalSettings & settings, PosTable & positions, DocCommentMap & docComments, - const ref rootFS, - const Expr::AstSymbols & astSymbols); + const ref rootFS); } @@ -79,24 +74,30 @@ Expr * parseExprFromBuf( %{ -#include "parser-tab.hh" -#include "lexer-tab.hh" +/* The parser is very performance sensitive and loses out on a lot + of performance even with basic stdlib assertions. Since those don't + affect ABI we can disable those just for this file. */ +#if defined(_GLIBCXX_ASSERTIONS) && !defined(_GLIBCXX_DEBUG) +#undef _GLIBCXX_ASSERTIONS +#endif + +#include "parser-scanner-decls.hh" YY_DECL; using namespace nix; -#define CUR_POS state->at(yyloc) +#define CUR_POS state->at(yylhs.location) - -void yyerror(YYLTYPE * loc, yyscan_t scanner, ParserState * state, const char * error) +void parser::BisonParser::error(const location_type &loc_, const std::string &error) { + auto loc = loc_; if (std::string_view(error).starts_with("syntax error, unexpected end of file")) { - loc->beginOffset = loc->endOffset; + loc.beginOffset = loc.endOffset; } throw ParseError({ .msg = HintFmt(error), - .pos = state->positions[state->at(*loc)] + .pos = state->positions[state->at(loc)] }); } @@ -135,6 +136,7 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) { std::vector * attrNames; std::vector> * inheritAttrs; std::vector> * string_parts; + std::variant * to_be_string; std::vector>> * ind_string_parts; } @@ -149,7 +151,8 @@ static Expr * makeCall(PosIdx pos, Expr * fn, Expr * arg) { %type attrs %type string_parts_interpolated %type ind_string_parts -%type path_start string_parts string_attr +%type path_start +%type string_parts string_attr %type attr %token ID %token STR IND_STR @@ -183,7 +186,7 @@ start: expr { state->result = $1; // This parser does not use yynerrs; suppress the warning. - (void) yynerrs; + (void) yynerrs_; }; expr: expr_function; @@ -258,7 +261,7 @@ expr_op | expr_op OR expr_op { $$ = new ExprOpOr(state->at(@2), $1, $3); } | expr_op IMPL expr_op { $$ = new ExprOpImpl(state->at(@2), $1, $3); } | expr_op UPDATE expr_op { $$ = new ExprOpUpdate(state->at(@2), $1, $3); } - | expr_op '?' attrpath { $$ = new ExprOpHasAttr($1, std::move(*$3)); delete $3; } + | expr_op '?' attrpath { $$ = new ExprOpHasAttr(state->alloc, $1, std::move(*$3)); delete $3; } | expr_op '+' expr_op { $$ = new ExprConcatStrings(state->at(@2), false, new std::vector >({{state->at(@1), $1}, {state->at(@3), $3}})); } | expr_op '-' expr_op { $$ = new ExprCall(state->at(@2), new ExprVar(state->s.sub), {$1, $3}); } @@ -279,9 +282,9 @@ expr_app expr_select : expr_simple '.' attrpath - { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), nullptr); delete $3; } + { $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move(*$3), nullptr); delete $3; } | expr_simple '.' attrpath OR_KW expr_select - { $$ = new ExprSelect(CUR_POS, $1, std::move(*$3), $5); delete $3; $5->warnIfCursedOr(state->symbols, state->positions); } + { $$ = new ExprSelect(state->alloc, CUR_POS, $1, std::move(*$3), $5); delete $3; $5->warnIfCursedOr(state->symbols, state->positions); } | /* Backwards compatibility: because Nixpkgs has a function named ‘or’, allow stuff like ‘map or [...]’. This production is problematic (see https://github.com/NixOS/nix/issues/11118) and will be refactored in the @@ -304,7 +307,13 @@ expr_simple } | INT_LIT { $$ = new ExprInt($1); } | FLOAT_LIT { $$ = new ExprFloat($1); } - | '"' string_parts '"' { $$ = $2; } + | '"' string_parts '"' { + std::visit(overloaded{ + [&](std::string_view str) { $$ = new ExprString(state->alloc, str); }, + [&](Expr * expr) { $$ = expr; }}, + *$2); + delete $2; + } | IND_STRING_OPEN ind_string_parts IND_STRING_CLOSE { $$ = state->stripIndentation(CUR_POS, std::move(*$2)); delete $2; @@ -315,11 +324,11 @@ expr_simple $$ = new ExprConcatStrings(CUR_POS, false, $2); } | SPATH { - std::string path($1.p + 1, $1.l - 2); + std::string_view path($1.p + 1, $1.l - 2); $$ = new ExprCall(CUR_POS, new ExprVar(state->s.findFile), {new ExprVar(state->s.nixPath), - new ExprString(std::move(path))}); + new ExprString(state->alloc, path)}); } | URI { static bool noURLLiterals = experimentalFeatureSettings.isEnabled(Xp::NoUrlLiterals); @@ -328,13 +337,13 @@ expr_simple .msg = HintFmt("URL literals are disabled"), .pos = state->positions[CUR_POS] }); - $$ = new ExprString(std::string($1)); + $$ = new ExprString(state->alloc, $1); } | '(' expr ')' { $$ = $2; } /* Let expressions `let {..., body = ...}' are just desugared into `(rec {..., body = ...}).body'. */ | LET '{' binds '}' - { $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(noPos, $3, state->s.body); } + { $3->recursive = true; $3->pos = CUR_POS; $$ = new ExprSelect(state->alloc, noPos, $3, state->s.body); } | REC '{' binds '}' { $3->recursive = true; $3->pos = CUR_POS; $$ = $3; } | '{' binds1 '}' @@ -345,19 +354,19 @@ expr_simple ; string_parts - : STR { $$ = new ExprString(std::string($1)); } - | string_parts_interpolated { $$ = new ExprConcatStrings(CUR_POS, true, $1); } - | { $$ = new ExprString(""); } + : STR { $$ = new std::variant($1); } + | string_parts_interpolated { $$ = new std::variant(new ExprConcatStrings(CUR_POS, true, $1)); } + | { $$ = new std::variant(std::string_view()); } ; string_parts_interpolated : string_parts_interpolated STR - { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(std::string($2))); } + { $$ = $1; $1->emplace_back(state->at(@2), new ExprString(state->alloc, $2)); } | string_parts_interpolated DOLLAR_CURLY expr '}' { $$ = $1; $1->emplace_back(state->at(@2), $3); } | DOLLAR_CURLY expr '}' { $$ = new std::vector>; $$->emplace_back(state->at(@1), $2); } | STR DOLLAR_CURLY expr '}' { $$ = new std::vector>; - $$->emplace_back(state->at(@1), new ExprString(std::string($1))); + $$->emplace_back(state->at(@1), new ExprString(state->alloc, $1)); $$->emplace_back(state->at(@2), $3); } ; @@ -383,8 +392,8 @@ path_start root filesystem accessor, rather than the accessor of the current Nix expression. */ literal.front() == '/' - ? new ExprPath(state->rootFS, std::move(path)) - : new ExprPath(state->basePath.accessor, std::move(path)); + ? new ExprPath(state->alloc, state->rootFS, path) + : new ExprPath(state->alloc, state->basePath.accessor, path); } | HPATH { if (state->settings.pureEval) { @@ -394,7 +403,7 @@ path_start ); } Path path(getHome() + std::string($1.p + 1, $1.l - 1)); - $$ = new ExprPath(ref(state->rootFS), std::move(path)); + $$ = new ExprPath(state->alloc, ref(state->rootFS), path); } ; @@ -438,7 +447,7 @@ binds1 $accum->attrs.emplace( i.symbol, ExprAttrs::AttrDef( - new ExprSelect(iPos, from, i.symbol), + new ExprSelect(state->alloc, iPos, from, i.symbol), iPos, ExprAttrs::AttrDef::Kind::InheritedFrom)); } @@ -455,15 +464,16 @@ attrs : attrs attr { $$ = $1; $1->emplace_back(AttrName(state->symbols.create($2)), state->at(@2)); } | attrs string_attr { $$ = $1; - ExprString * str = dynamic_cast($2); - if (str) { - $$->emplace_back(AttrName(state->symbols.create(str->s)), state->at(@2)); - delete str; - } else - throw ParseError({ - .msg = HintFmt("dynamic attributes not allowed in inherit"), - .pos = state->positions[state->at(@2)] - }); + std::visit(overloaded { + [&](std::string_view str) { $$->emplace_back(AttrName(state->symbols.create(str)), state->at(@2)); }, + [&](Expr * expr) { + throw ParseError({ + .msg = HintFmt("dynamic attributes not allowed in inherit"), + .pos = state->positions[state->at(@2)] + }); + } + }, *$2); + delete $2; } | { $$ = new std::vector>; } ; @@ -472,22 +482,20 @@ attrpath : attrpath '.' attr { $$ = $1; $1->push_back(AttrName(state->symbols.create($3))); } | attrpath '.' string_attr { $$ = $1; - ExprString * str = dynamic_cast($3); - if (str) { - $$->push_back(AttrName(state->symbols.create(str->s))); - delete str; - } else - $$->push_back(AttrName($3)); + std::visit(overloaded { + [&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); }, + [&](Expr * expr) { $$->push_back(AttrName(expr)); } + }, *$3); + delete $3; } | attr { $$ = new std::vector; $$->push_back(AttrName(state->symbols.create($1))); } | string_attr { $$ = new std::vector; - ExprString *str = dynamic_cast($1); - if (str) { - $$->push_back(AttrName(state->symbols.create(str->s))); - delete str; - } else - $$->push_back(AttrName($1)); + std::visit(overloaded { + [&](std::string_view str) { $$->push_back(AttrName(state->symbols.create(str))); }, + [&](Expr * expr) { $$->push_back(AttrName(expr)); } + }, *$1); + delete $1; } ; @@ -498,7 +506,7 @@ attr string_attr : '"' string_parts '"' { $$ = $2; } - | DOLLAR_CURLY expr '}' { $$ = $2; } + | DOLLAR_CURLY expr '}' { $$ = new std::variant($2); } ; expr_list @@ -538,12 +546,12 @@ Expr * parseExprFromBuf( size_t length, Pos::Origin origin, const SourcePath & basePath, + std::pmr::polymorphic_allocator & alloc, SymbolTable & symbols, const EvalSettings & settings, PosTable & positions, DocCommentMap & docComments, - const ref rootFS, - const Expr::AstSymbols & astSymbols) + const ref rootFS) { yyscan_t scanner; LexerState lexerState { @@ -553,12 +561,12 @@ Expr * parseExprFromBuf( }; ParserState state { .lexerState = lexerState, + .alloc = alloc, .symbols = symbols, .positions = positions, .basePath = basePath, .origin = lexerState.origin, .rootFS = rootFS, - .s = astSymbols, .settings = settings, }; @@ -566,7 +574,8 @@ Expr * parseExprFromBuf( Finally _destroy([&] { yylex_destroy(scanner); }); yy_scan_buffer(text, length, scanner); - yyparse(scanner, &state); + Parser parser(scanner, &state); + parser.parse(); return state.result; } diff --git a/src/libexpr/paths.cc b/src/libexpr/paths.cc index f90bc37df0a..d5dd9518ae2 100644 --- a/src/libexpr/paths.cc +++ b/src/libexpr/paths.cc @@ -1,5 +1,7 @@ #include "nix/store/store-api.hh" #include "nix/expr/eval.hh" +#include "nix/util/mounted-source-accessor.hh" +#include "nix/fetchers/fetch-to-store.hh" namespace nix { @@ -18,4 +20,99 @@ SourcePath EvalState::storePath(const StorePath & path) return {rootFS, CanonPath{store->printStorePath(path)}}; } +StorePath EvalState::devirtualize(const StorePath & path, StringMap * rewrites) +{ + if (auto mount = storeFS->getMount(CanonPath(store->printStorePath(path)))) { + auto storePath = fetchToStore( + fetchSettings, + *store, + SourcePath{ref(mount)}, + settings.readOnlyMode ? FetchMode::DryRun : FetchMode::Copy, + path.name()); + assert(storePath.name() == path.name()); + if (rewrites) + rewrites->emplace(path.hashPart(), storePath.hashPart()); + return storePath; + } else + return path; +} + +SingleDerivedPath EvalState::devirtualize(const SingleDerivedPath & path, StringMap * rewrites) +{ + if (auto o = std::get_if(&path.raw())) + return SingleDerivedPath::Opaque{devirtualize(o->path, rewrites)}; + else + return path; +} + +std::string EvalState::devirtualize(std::string_view s, const NixStringContext & context) +{ + StringMap rewrites; + + for (auto & c : context) + if (auto o = std::get_if(&c.raw)) + devirtualize(o->path, &rewrites); + + return rewriteStrings(std::string(s), rewrites); +} + +std::string EvalState::computeBaseName(const SourcePath & path, PosIdx pos) +{ + if (path.accessor == rootFS) { + if (auto storePath = store->maybeParseStorePath(path.path.abs())) { + debug( + "Copying '%s' to the store again.\n" + "You can make Nix evaluate faster and copy fewer files by replacing `./.` with the `self` flake input, " + "or `builtins.path { path = ./.; name = \"source\"; }`.\n", + path); + return std::string( + fetchToStore(fetchSettings, *store, path, FetchMode::DryRun, storePath->name()).to_string()); + } + } + return std::string(path.baseName()); +} + +StorePath EvalState::mountInput( + fetchers::Input & input, + const fetchers::Input & originalInput, + ref accessor, + bool requireLockable, + bool forceNarHash) +{ + auto storePath = settings.lazyTrees + ? StorePath::random(input.getName()) + : fetchToStore(fetchSettings, *store, accessor, FetchMode::Copy, input.getName()); + + allowPath(storePath); // FIXME: should just whitelist the entire virtual store + + std::optional _narHash; + + auto getNarHash = [&]() { + if (!_narHash) { + if (store->isValidPath(storePath)) + _narHash = store->queryPathInfo(storePath)->narHash; + else + _narHash = fetchToStore2(fetchSettings, *store, accessor, FetchMode::DryRun, input.getName()).second; + } + return _narHash; + }; + + storeFS->mount(CanonPath(store->printStorePath(storePath)), accessor); + + if (forceNarHash + || (requireLockable && (!settings.lazyTrees || !settings.lazyLocks || !input.isLocked()) + && !input.getNarHash())) + input.attrs.insert_or_assign("narHash", getNarHash()->to_string(HashFormat::SRI, true)); + + if (originalInput.getNarHash() && *getNarHash() != *originalInput.getNarHash()) + throw Error( + (unsigned int) 102, + "NAR hash mismatch in input '%s', expected '%s' but got '%s'", + originalInput.to_string(), + getNarHash()->to_string(HashFormat::SRI, true), + originalInput.getNarHash()->to_string(HashFormat::SRI, true)); + + return storePath; +} + } // namespace nix diff --git a/src/libexpr/primops.cc b/src/libexpr/primops.cc index ca84f303833..98cb7f60480 100644 --- a/src/libexpr/primops.cc +++ b/src/libexpr/primops.cc @@ -16,8 +16,11 @@ #include "nix/expr/primops.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/sort.hh" +#include "nix/util/mounted-source-accessor.hh" #include +#include +#include #include #include @@ -64,6 +67,7 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS for (auto & c : context) { auto ensureValid = [&](const StorePath & p) { + waitForPath(p); if (!store->isValidPath(p)) error(store->printStorePath(p)).debugThrow(); }; @@ -78,7 +82,10 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS ensureValid(b.drvPath->getBaseStorePath()); }, [&](const NixStringContextElem::Opaque & o) { - ensureValid(o.path); + // We consider virtual store paths valid here. They'll + // be devirtualized if needed elsewhere. + if (!storeFS->getMount(CanonPath(store->printStorePath(o.path)))) + ensureValid(o.path); if (maybePathsOut) maybePathsOut->emplace(o.path); }, @@ -88,6 +95,9 @@ StringMap EvalState::realiseContext(const NixStringContext & context, StorePathS if (maybePathsOut) maybePathsOut->emplace(d.drvPath); }, + [&](const NixStringContextElem::Path & p) { + // FIXME: do something? + }, }, c.raw); } @@ -214,20 +224,20 @@ void derivationToValue( auto path2 = path.path.abs(); Derivation drv = state.store->readDerivation(storePath); auto attrs = state.buildBindings(3 + drv.outputs.size()); - attrs.alloc(state.sDrvPath) + attrs.alloc(state.s.drvPath) .mkString( path2, { NixStringContextElem::DrvDeep{.drvPath = storePath}, }); - attrs.alloc(state.sName).mkString(drv.env["name"]); + attrs.alloc(state.s.name).mkString(drv.env["name"]); auto list = state.buildList(drv.outputs.size()); for (const auto & [i, o] : enumerate(drv.outputs)) { mkOutputString(state, attrs, storePath, o); (list[i] = state.allocValue())->mkString(o.first); } - attrs.alloc(state.sOutputs).mkList(list); + attrs.alloc(state.s.outputs).mkList(list); auto w = state.allocValue(); w->mkAttrs(attrs); @@ -260,7 +270,7 @@ static void scopedImport(EvalState & state, const PosIdx pos, SourcePath & path, { state.forceAttrs(*vScope, pos, "while evaluating the first argument passed to builtins.scopedImport"); - Env * env = &state.allocEnv(vScope->attrs()->size()); + Env * env = &state.mem.allocEnv(vScope->attrs()->size()); env->up = &state.baseEnv; auto staticEnv = std::make_shared(nullptr, state.staticBaseEnv, vScope->attrs()->size()); @@ -292,6 +302,7 @@ static void import(EvalState & state, const PosIdx pos, Value & vPath, Value * v if (!state.store->isStorePath(path2)) return std::nullopt; auto storePath = state.store->parseStorePath(path2); + state.waitForPath(storePath); if (!(state.store->isValidPath(storePath) && isDerivation(path2))) return std::nullopt; return storePath; @@ -483,42 +494,41 @@ void prim_exec(EvalState & state, const PosIdx pos, Value ** args, Value & v) static void prim_typeOf(EvalState & state, const PosIdx pos, Value ** args, Value & v) { state.forceValue(*args[0], pos); - std::string t; switch (args[0]->type()) { case nInt: - t = "int"; + v.mkStringNoCopy("int"); break; case nBool: - t = "bool"; + v.mkStringNoCopy("bool"); break; case nString: - t = "string"; + v.mkStringNoCopy("string"); break; case nPath: - t = "path"; + v.mkStringNoCopy("path"); break; case nNull: - t = "null"; + v.mkStringNoCopy("null"); break; case nAttrs: - t = "set"; + v.mkStringNoCopy("set"); break; case nList: - t = "list"; + v.mkStringNoCopy("list"); break; case nFunction: - t = "lambda"; + v.mkStringNoCopy("lambda"); break; case nExternal: - t = args[0]->external()->typeOf(); + v.mkString(args[0]->external()->typeOf()); break; case nFloat: - t = "float"; + v.mkStringNoCopy("float"); break; case nThunk: + case nFailed: unreachable(); } - v.mkString(t); } static RegisterPrimOp primop_typeOf({ @@ -731,7 +741,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value ** ar /* Get the start set. */ auto startSet = state.getAttr( - state.sStartSet, args[0]->attrs(), "in the attrset passed as argument to builtins.genericClosure"); + state.s.startSet, args[0]->attrs(), "in the attrset passed as argument to builtins.genericClosure"); state.forceList( *startSet->value, @@ -749,7 +759,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value ** ar /* Get the operator. */ auto op = state.getAttr( - state.sOperator, args[0]->attrs(), "in the attrset passed as argument to builtins.genericClosure"); + state.s.operator_, args[0]->attrs(), "in the attrset passed as argument to builtins.genericClosure"); state.forceFunction( *op->value, noPos, "while evaluating the 'operator' attribute passed as argument to builtins.genericClosure"); @@ -771,7 +781,7 @@ static void prim_genericClosure(EvalState & state, const PosIdx pos, Value ** ar "while evaluating one of the elements generated by (or initially passed to) builtins.genericClosure"); auto key = state.getAttr( - state.sKey, + state.s.key, e->attrs(), "in one of the attrsets generated by (or initially passed to) builtins.genericClosure"); state.forceValue(*key->value, noPos); @@ -1053,7 +1063,7 @@ static RegisterPrimOp primop_floor({ a NixInt and if `*number* < -9007199254740992` or `*number* > 9007199254740992`. If the datatype of *number* is neither a NixInt (signed 64-bit integer) nor a NixFloat - (IEEE-754 double-precision floating-point number), an evaluation error will be thrown. + (IEEE-754 double-precision floating-point number), an evaluation error is thrown. )", .fun = prim_floor, }); @@ -1076,12 +1086,12 @@ static void prim_tryEval(EvalState & state, const PosIdx pos, Value ** args, Val try { state.forceValue(*args[0], pos); - attrs.insert(state.sValue, args[0]); - attrs.insert(state.symbols.create("success"), &state.vTrue); + attrs.insert(state.s.value, args[0]); + attrs.insert(state.symbols.create("success"), &Value::vTrue); } catch (AssertionError & e) { // `value = false;` is unfortunate but removing it is a breaking change. - attrs.insert(state.sValue, &state.vFalse); - attrs.insert(state.symbols.create("success"), &state.vFalse); + attrs.insert(state.s.value, &Value::vFalse); + attrs.insert(state.symbols.create("success"), &Value::vFalse); } // restore the debugRepl pointer if we saved it earlier. @@ -1100,7 +1110,7 @@ static RegisterPrimOp primop_tryEval({ `false` if an error was thrown) and `value`, equalling *e* if successful and `false` otherwise. `tryEval` only prevents errors created by `throw` or `assert` from being thrown. - Errors `tryEval` doesn't catch are, for example, those created + Errors that `tryEval` doesn't catch are, for example, those created by `abort` and type errors generated by builtins. Also note that this doesn't evaluate *e* deeply, so `let e = { x = throw ""; }; in (builtins.tryEval e).success` is `true`. Using @@ -1252,7 +1262,7 @@ static RegisterPrimOp primop_warn({ [`debugger-on-trace`](@docroot@/command-ref/conf-file.md#conf-debugger-on-trace) or [`debugger-on-warn`](@docroot@/command-ref/conf-file.md#conf-debugger-on-warn) option is set to `true` and the `--debugger` flag is given, the - interactive debugger will be started when `warn` is called (like + interactive debugger is started when `warn` is called (like [`break`](@docroot@/language/builtins.md#builtins-break)). If the @@ -1292,7 +1302,8 @@ static void prim_derivationStrict(EvalState & state, const PosIdx pos, Value ** auto attrs = args[0]->attrs(); /* Figure out the name first (for stack backtraces). */ - auto nameAttr = state.getAttr(state.sName, attrs, "in the attrset passed as argument to builtins.derivationStrict"); + auto nameAttr = + state.getAttr(state.s.name, attrs, "in the attrset passed as argument to builtins.derivationStrict"); std::string_view drvName; try { @@ -1366,8 +1377,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName using nlohmann::json; std::optional jsonObject; auto pos = v.determinePos(noPos); - auto attr = attrs->find(state.sStructuredAttrs); - if (attr != attrs->end() + auto attr = attrs->get(state.s.structuredAttrs); + if (attr && state.forceBool( *attr->value, pos, @@ -1377,8 +1388,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName /* Check whether null attributes should be ignored. */ bool ignoreNulls = false; - attr = attrs->find(state.sIgnoreNulls); - if (attr != attrs->end()) + attr = attrs->get(state.s.ignoreNulls); + if (attr) ignoreNulls = state.forceBool( *attr->value, pos, @@ -1401,7 +1412,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName outputs.insert("out"); for (auto & i : attrs->lexicographicOrder(state.symbols)) { - if (i->name == state.sIgnoreNulls) + if (i->name == state.s.ignoreNulls) continue; auto key = state.symbols[i->name]; vomit("processing attribute '%1%'", key); @@ -1453,19 +1464,22 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName continue; } - if (i->name == state.sContentAddressed && state.forceBool(*i->value, pos, context_below)) { - contentAddressed = true; - experimentalFeatureSettings.require(Xp::CaDerivations); - } - - else if (i->name == state.sImpure && state.forceBool(*i->value, pos, context_below)) { - isImpure = true; - experimentalFeatureSettings.require(Xp::ImpureDerivations); - } - + switch (i->name.getId()) { + case EvalState::s.contentAddressed.getId(): + if (state.forceBool(*i->value, pos, context_below)) { + contentAddressed = true; + experimentalFeatureSettings.require(Xp::CaDerivations); + } + break; + case EvalState::s.impure.getId(): + if (state.forceBool(*i->value, pos, context_below)) { + isImpure = true; + experimentalFeatureSettings.require(Xp::ImpureDerivations); + } + break; /* The `args' attribute is special: it supplies the command-line arguments to the builder. */ - else if (i->name == state.sArgs) { + case EvalState::s.args.getId(): state.forceList(*i->value, pos, context_below); for (auto elem : i->value->listView()) { auto s = state @@ -1474,86 +1488,116 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName .toOwned(); drv.args.push_back(s); } - } - + break; /* All other attributes are passed to the builder through the environment. */ - else { + default: if (jsonObject) { - if (i->name == state.sStructuredAttrs) + if (i->name == state.s.structuredAttrs) continue; jsonObject->structuredAttrs.emplace(key, printValueAsJSON(state, true, *i->value, pos, context)); - if (i->name == state.sBuilder) + switch (i->name.getId()) { + case EvalState::s.builder.getId(): drv.builder = state.forceString(*i->value, context, pos, context_below); - else if (i->name == state.sSystem) + break; + case EvalState::s.system.getId(): drv.platform = state.forceStringNoCtx(*i->value, pos, context_below); - else if (i->name == state.sOutputHash) + break; + case EvalState::s.outputHash.getId(): outputHash = state.forceStringNoCtx(*i->value, pos, context_below); - else if (i->name == state.sOutputHashAlgo) + break; + case EvalState::s.outputHashAlgo.getId(): outputHashAlgo = parseHashAlgoOpt(state.forceStringNoCtx(*i->value, pos, context_below)); - else if (i->name == state.sOutputHashMode) + break; + case EvalState::s.outputHashMode.getId(): handleHashMode(state.forceStringNoCtx(*i->value, pos, context_below)); - else if (i->name == state.sOutputs) { - /* Require ‘outputs’ to be a list of strings. */ + break; + case EvalState::s.outputs.getId(): { + /* Require 'outputs' to be a list of strings. */ state.forceList(*i->value, pos, context_below); Strings ss; for (auto elem : i->value->listView()) ss.emplace_back(state.forceStringNoCtx(*elem, pos, context_below)); handleOutputs(ss); + break; + } + default: + break; } - if (i->name == state.sAllowedReferences) + switch (i->name.getId()) { + case EvalState::s.allowedReferences.getId(): warn( "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedReferences'; use 'outputChecks..allowedReferences' instead", drvName); - if (i->name == state.sAllowedRequisites) + break; + case EvalState::s.allowedRequisites.getId(): warn( "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'allowedRequisites'; use 'outputChecks..allowedRequisites' instead", drvName); - if (i->name == state.sDisallowedReferences) + break; + case EvalState::s.disallowedReferences.getId(): warn( "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedReferences'; use 'outputChecks..disallowedReferences' instead", drvName); - if (i->name == state.sDisallowedRequisites) + break; + case EvalState::s.disallowedRequisites.getId(): warn( "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'disallowedRequisites'; use 'outputChecks..disallowedRequisites' instead", drvName); - if (i->name == state.sMaxSize) + break; + case EvalState::s.maxSize.getId(): warn( "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxSize'; use 'outputChecks..maxSize' instead", drvName); - if (i->name == state.sMaxClosureSize) + break; + case EvalState::s.maxClosureSize.getId(): warn( "In a derivation named '%s', 'structuredAttrs' disables the effect of the derivation attribute 'maxClosureSize'; use 'outputChecks..maxClosureSize' instead", drvName); + break; + default: + break; + } } else { auto s = state.coerceToString(pos, *i->value, context, context_below, true).toOwned(); - if (i->name == state.sJson) { + if (i->name == state.s.json) { warn( "In derivation '%s': setting structured attributes via '__json' is deprecated, and may be disallowed in future versions of Nix. Set '__structuredAttrs = true' instead.", drvName); drv.structuredAttrs = StructuredAttrs::parse(s); } else { drv.env.emplace(key, s); - if (i->name == state.sBuilder) + switch (i->name.getId()) { + case EvalState::s.builder.getId(): drv.builder = std::move(s); - else if (i->name == state.sSystem) + break; + case EvalState::s.system.getId(): drv.platform = std::move(s); - else if (i->name == state.sOutputHash) + break; + case EvalState::s.outputHash.getId(): outputHash = std::move(s); - else if (i->name == state.sOutputHashAlgo) + break; + case EvalState::s.outputHashAlgo.getId(): outputHashAlgo = parseHashAlgoOpt(s); - else if (i->name == state.sOutputHashMode) + break; + case EvalState::s.outputHashMode.getId(): handleHashMode(s); - else if (i->name == state.sOutputs) + break; + case EvalState::s.outputs.getId(): handleOutputs(tokenizeString(s)); + break; + default: + break; + } } } + break; } } catch (Error & e) { @@ -1573,6 +1617,10 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName /* Everything in the context of the strings in the derivation attributes should be added as dependencies of the resulting derivation. */ + StringMap rewrites; + + std::optional drvS; + for (auto & c : context) { std::visit( overloaded{ @@ -1584,6 +1632,8 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName [&](const NixStringContextElem::DrvDeep & d) { /* !!! This doesn't work if readOnlyMode is set. */ StorePathSet refs; + // FIXME: don't need to wait, we only need the references. + state.waitForPath(d.drvPath); state.store->computeFSClosure(d.drvPath, refs); for (auto & j : refs) { drv.inputSrcs.insert(j); @@ -1595,11 +1645,27 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName [&](const NixStringContextElem::Built & b) { drv.inputDrvs.ensureSlot(*b.drvPath).value.insert(b.output); }, - [&](const NixStringContextElem::Opaque & o) { drv.inputSrcs.insert(o.path); }, + [&](const NixStringContextElem::Opaque & o) { + drv.inputSrcs.insert(state.devirtualize(o.path, &rewrites)); + }, + [&](const NixStringContextElem::Path & p) { + if (!drvS) + drvS = drv.unparse(*state.store, true); + if (drvS->find(p.storePath.to_string()) != drvS->npos) { + auto devirtualized = state.devirtualize(p.storePath, &rewrites); + warn( + "Using 'builtins.derivation' to create a derivation named '%s' that references the store path '%s' without a proper context. " + "The resulting derivation will not have a correct store reference, so this is unreliable and may stop working in the future.", + drvName, + state.store->printStorePath(devirtualized)); + } + }, }, c.raw); } + drv.applyRewrites(rewrites); + /* Do we have all required attributes? */ if (drv.builder == "") state.error("required attribute 'builder' missing").atPos(v).debugThrow(); @@ -1708,7 +1774,7 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName } /* Write the resulting term into the Nix store directory. */ - auto drvPath = writeDerivation(*state.store, drv, state.repair); + auto drvPath = writeDerivation(*state.store, *state.asyncPathWriter, drv, state.repair); auto drvPathS = state.store->printStorePath(drvPath); printMsg(lvlChatty, "instantiated '%1%' -> '%2%'", drvName, drvPathS); @@ -1718,11 +1784,11 @@ static void derivationStrictInternal(EvalState & state, std::string_view drvName read them later. */ { auto h = hashDerivationModulo(*state.store, drv, false); - drvHashes.lock()->insert_or_assign(drvPath, h); + drvHashes.insert_or_assign(drvPath, std::move(h)); } auto result = state.buildBindings(1 + drv.outputs.size()); - result.alloc(state.sDrvPath) + result.alloc(state.s.drvPath) .mkString( drvPathS, { @@ -1966,14 +2032,17 @@ static void prim_readFile(EvalState & state, const PosIdx pos, Value ** args, Va .debugThrow(); StorePathSet refs; if (state.store->isInStore(path.path.abs())) { - try { - refs = state.store->queryPathInfo(state.store->toStorePath(path.path.abs()).first)->references; - } catch (Error &) { // FIXME: should be InvalidPathError + auto storePath = state.store->toStorePath(path.path.abs()).first; + // Skip virtual paths since they don't have references and + // don't exist anyway. + if (!state.storeFS->getMount(CanonPath(state.store->printStorePath(storePath)))) { + if (auto info = state.store->maybeQueryPathInfo(state.store->toStorePath(path.path.abs()).first)) { + // Re-scan references to filter down to just the ones that actually occur in the file. + auto refsSink = PathRefScanSink::fromPaths(info->references); + refsSink << s; + refs = refsSink.getResultPaths(); + } } - // Re-scan references to filter down to just the ones that actually occur in the file. - auto refsSink = PathRefScanSink::fromPaths(refs); - refsSink << s; - refs = refsSink.getResultPaths(); } NixStringContext context; for (auto && p : std::move(refs)) { @@ -2006,14 +2075,14 @@ static void prim_findFile(EvalState & state, const PosIdx pos, Value ** args, Va state.forceAttrs(*v2, pos, "while evaluating an element of the list passed to builtins.findFile"); std::string prefix; - auto i = v2->attrs()->find(state.sPrefix); - if (i != v2->attrs()->end()) + auto i = v2->attrs()->get(state.s.prefix); + if (i) prefix = state.forceStringNoCtx( *i->value, pos, "while evaluating the `prefix` attribute of an element of the list passed to builtins.findFile"); - i = state.getAttr(state.sPath, v2->attrs(), "in an element of the __nixPath"); + i = state.getAttr(state.s.path, v2->attrs(), "in an element of the __nixPath"); NixStringContext context; auto path = @@ -2209,19 +2278,45 @@ static RegisterPrimOp primop_hashFile({ .fun = prim_hashFile, }); -static Value * fileTypeToString(EvalState & state, SourceAccessor::Type type) +static const Value & fileTypeToString(EvalState & state, SourceAccessor::Type type) { - return type == SourceAccessor::Type::tRegular ? &state.vStringRegular - : type == SourceAccessor::Type::tDirectory ? &state.vStringDirectory - : type == SourceAccessor::Type::tSymlink ? &state.vStringSymlink - : &state.vStringUnknown; + struct Constants + { + Value regular; + Value directory; + Value symlink; + Value unknown; + }; + + static const Constants stringValues = []() { + Constants res; + res.regular.mkStringNoCopy("regular"); + res.directory.mkStringNoCopy("directory"); + res.symlink.mkStringNoCopy("symlink"); + res.unknown.mkStringNoCopy("unknown"); + return res; + }(); + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wswitch-enum" + using enum SourceAccessor::Type; + switch (type) { + case tRegular: + return stringValues.regular; + case tDirectory: + return stringValues.directory; + case tSymlink: + return stringValues.symlink; + default: + return stringValues.unknown; + } } static void prim_readFileType(EvalState & state, const PosIdx pos, Value ** args, Value & v) { auto path = realisePath(state, pos, *args[0], std::nullopt); /* Retrieve the directory entry type and stringize it. */ - v = *fileTypeToString(state, path.lstat().type); + v = fileTypeToString(state, path.lstat().type); } static RegisterPrimOp primop_readFileType({ @@ -2265,7 +2360,9 @@ static void prim_readDir(EvalState & state, const PosIdx pos, Value ** args, Val } else { // This branch of the conditional is much more likely. // Here we just stringize the directory entry type. - attrs.insert(state.symbols.create(name), fileTypeToString(state, *type)); + // N.B. const_cast here is ok, because these values will never be modified, since + // only thunks are mutable - other types do not change once constructed. + attrs.insert(state.symbols.create(name), const_cast(&fileTypeToString(state, *type))); } } @@ -2350,7 +2447,7 @@ static void prim_toXML(EvalState & state, const PosIdx pos, Value ** args, Value std::ostringstream out; NixStringContext context; printValueAsXML(state, true, false, *args[0], out, context, pos); - v.mkString(toView(out), context); + v.mkString(out.view(), context); } static RegisterPrimOp primop_toXML({ @@ -2458,7 +2555,7 @@ static void prim_toJSON(EvalState & state, const PosIdx pos, Value ** args, Valu std::ostringstream out; NixStringContext context; printValueAsJSON(state, true, *args[0], pos, out, context); - v.mkString(toView(out), context); + v.mkString(out.view(), context); } static RegisterPrimOp primop_toJSON({ @@ -2508,15 +2605,25 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value ** args, Valu { NixStringContext context; auto name = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.toFile"); - auto contents = - state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile"); + std::string contents( + state.forceString(*args[1], context, pos, "while evaluating the second argument passed to builtins.toFile")); StorePathSet refs; + StringMap rewrites; for (auto c : context) { if (auto p = std::get_if(&c.raw)) refs.insert(p->path); - else + else if (auto p = std::get_if(&c.raw)) { + if (contents.find(p->storePath.to_string()) != contents.npos) { + auto devirtualized = state.devirtualize(p->storePath, &rewrites); + warn( + "Using 'builtins.toFile' to create a file named '%s' that references the store path '%s' without a proper context. " + "The resulting file will not have a correct store reference, so this is unreliable and may stop working in the future.", + name, + state.store->printStorePath(devirtualized)); + } + } else state .error( "files created by %1% may not reference derivations, but %2% references %3%", @@ -2527,6 +2634,8 @@ static void prim_toFile(EvalState & state, const PosIdx pos, Value ** args, Valu .debugThrow(); } + contents = rewriteStrings(contents, rewrites); + auto storePath = settings.readOnlyMode ? state.store->makeFixedOutputPathFromCA( name, TextInfo{ @@ -2640,7 +2749,7 @@ bool EvalState::callPathFilter(Value * filterFun, const SourcePath & path, PosId arg1.mkString(path.path.abs()); // assert that type is not "unknown" - Value * args[]{&arg1, fileTypeToString(*this, st.type)}; + Value * args[]{&arg1, const_cast(&fileTypeToString(*this, st.type))}; Value res; callFunction(*filterFun, args, res, pos); @@ -2679,6 +2788,7 @@ static void addPath( name, ContentAddressWithReferences::fromParts(method, *expectedHash, {})); if (!expectedHash || !state.store->isValidPath(*expectedStorePath)) { + // FIXME: make this lazy? auto dstPath = fetchToStore( state.fetchSettings, *state.store, @@ -2712,7 +2822,15 @@ static void prim_filterSource(EvalState & state, const PosIdx pos, Value ** args state.forceFunction(*args[0], pos, "while evaluating the first argument passed to builtins.filterSource"); addPath( - state, pos, path.baseName(), path, args[0], ContentAddressMethod::Raw::NixArchive, std::nullopt, v, context); + state, + pos, + state.computeBaseName(path, pos), + path, + args[0], + ContentAddressMethod::Raw::NixArchive, + std::nullopt, + v, + context); } static RegisterPrimOp primop_filterSource({ @@ -2786,7 +2904,7 @@ static void prim_path(EvalState & state, const PosIdx pos, Value ** args, Value if (n == "path") path.emplace(state.coerceToPath( attr.pos, *attr.value, context, "while evaluating the 'path' attribute passed to 'builtins.path'")); - else if (attr.name == state.sName) + else if (attr.name == state.s.name) name = state.forceStringNoCtx( *attr.value, attr.pos, "while evaluating the `name` attribute passed to builtins.path"); else if (n == "filter") @@ -2946,8 +3064,8 @@ static void prim_unsafeGetAttrPos(EvalState & state, const PosIdx pos, Value ** auto attr = state.forceStringNoCtx( *args[0], pos, "while evaluating the first argument passed to builtins.unsafeGetAttrPos"); state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.unsafeGetAttrPos"); - auto i = args[1]->attrs()->find(state.symbols.create(attr)); - if (i == args[1]->attrs()->end()) + auto i = args[1]->attrs()->get(state.symbols.create(attr)); + if (!i) v.mkNull(); else state.mkPos(v, i->pos); @@ -3014,7 +3132,7 @@ static void prim_hasAttr(EvalState & state, const PosIdx pos, Value ** args, Val { auto attr = state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.hasAttr"); state.forceAttrs(*args[1], pos, "while evaluating the second argument passed to builtins.hasAttr"); - v.mkBool(args[1]->attrs()->find(state.symbols.create(attr)) != args[1]->attrs()->end()); + v.mkBool(args[1]->attrs()->get(state.symbols.create(attr))); } static RegisterPrimOp primop_hasAttr({ @@ -3099,13 +3217,13 @@ static void prim_listToAttrs(EvalState & state, const PosIdx pos, Value ** args, // Step 1. Sort the name-value attrsets in place using the memory we allocate for the result auto listView = args[0]->listView(); size_t listSize = listView.size(); - auto & bindings = *state.allocBindings(listSize); + auto & bindings = *state.mem.allocBindings(listSize); using ElemPtr = decltype(&bindings[0].value); for (const auto & [n, v2] : enumerate(listView)) { state.forceAttrs(*v2, pos, "while evaluating an element of the list passed to builtins.listToAttrs"); - auto j = state.getAttr(state.sName, v2->attrs(), "in a {name=...; value=...;} pair"); + auto j = state.getAttr(state.s.name, v2->attrs(), "in a {name=...; value=...;} pair"); auto name = state.forceStringNoCtx( *j->value, @@ -3132,7 +3250,7 @@ static void prim_listToAttrs(EvalState & state, const PosIdx pos, Value ** args, // Note that .value is actually a Value * *; see earlier comments Value * v2 = *std::bit_cast(attr.value); - auto j = state.getAttr(state.sValue, v2->attrs(), "in a {name=...; value=...;} pair"); + auto j = state.getAttr(state.s.value, v2->attrs(), "in a {name=...; value=...;} pair"); prev = attr.name; bindings.push_back({prev, j->value, j->pos}); } @@ -3224,14 +3342,14 @@ static void prim_intersectAttrs(EvalState & state, const PosIdx pos, Value ** ar if (left.size() < right.size()) { for (auto & l : left) { - auto r = right.find(l.name); - if (r != right.end()) + auto r = right.get(l.name); + if (r) attrs.insert(*r); } } else { for (auto & r : right) { - auto l = left.find(r.name); - if (l != left.end()) + auto l = left.get(r.name); + if (l) attrs.insert(r); } } @@ -3294,14 +3412,14 @@ static void prim_functionArgs(EvalState & state, const PosIdx pos, Value ** args { state.forceValue(*args[0], pos); if (args[0]->isPrimOpApp() || args[0]->isPrimOp()) { - v.mkAttrs(&state.emptyBindings); + v.mkAttrs(&Bindings::emptyBindings); return; } if (!args[0]->isLambda()) state.error("'functionArgs' requires a function").atPos(pos).debugThrow(); if (!args[0]->lambda().fun->hasFormals()) { - v.mkAttrs(&state.emptyBindings); + v.mkAttrs(&Bindings::emptyBindings); return; } @@ -3746,8 +3864,8 @@ static void anyOrAll(bool any, EvalState & state, const PosIdx pos, Value ** arg std::string_view errorCtx = any ? "while evaluating the return value of the function passed to builtins.any" : "while evaluating the return value of the function passed to builtins.all"; - Value vTmp; for (auto elem : args[1]->listView()) { + Value vTmp; state.callFunction(*args[0], *elem, vTmp, pos); bool res = state.forceBool(vTmp, pos, errorCtx); if (res == any) { @@ -3948,13 +4066,13 @@ static void prim_partition(EvalState & state, const PosIdx pos, Value ** args, V auto rlist = state.buildList(rsize); if (rsize) memcpy(rlist.elems, right.data(), sizeof(Value *) * rsize); - attrs.alloc(state.sRight).mkList(rlist); + attrs.alloc(state.s.right).mkList(rlist); auto wsize = wrong.size(); auto wlist = state.buildList(wsize); if (wsize) memcpy(wlist.elems, wrong.data(), sizeof(Value *) * wsize); - attrs.alloc(state.sWrong).mkList(wlist); + attrs.alloc(state.s.wrong).mkList(wlist); v.mkAttrs(attrs); } @@ -3995,7 +4113,7 @@ static void prim_groupBy(EvalState & state, const PosIdx pos, Value ** args, Val auto name = state.forceStringNoCtx( res, pos, "while evaluating the return value of the grouping function passed to builtins.groupBy"); auto sym = state.symbols.create(name); - auto vector = attrs.try_emplace(sym, ValueVector()).first; + auto vector = attrs.try_emplace(sym, {}).first; vector->second.push_back(vElem); } @@ -4348,7 +4466,7 @@ static void prim_substring(EvalState & state, const PosIdx pos, Value ** args, V if (len == 0) { state.forceValue(*args[2], pos); if (args[2]->type() == nString) { - v.mkString("", args[2]->context()); + v.mkStringNoCopy("", args[2]->context()); return; } } @@ -4530,33 +4648,27 @@ static RegisterPrimOp primop_convertHash({ struct RegexCache { - struct State - { - std::unordered_map> cache; - }; - - Sync state_; + boost::concurrent_flat_map> cache; std::regex get(std::string_view re) { - auto state(state_.lock()); - auto it = state->cache.find(re); - if (it != state->cache.end()) - return it->second; + std::regex regex; /* No std::regex constructor overload from std::string_view, but can be constructed from a pointer + size or an iterator range. */ - return state->cache - .emplace( - std::piecewise_construct, - std::forward_as_tuple(re), - std::forward_as_tuple(/*s=*/re.data(), /*count=*/re.size(), std::regex::extended)) - .first->second; + cache.try_emplace_and_cvisit( + re, + /*s=*/re.data(), + /*count=*/re.size(), + std::regex::extended, + [®ex](const auto & kv) { regex = kv.second; }, + [®ex](const auto & kv) { regex = kv.second; }); + return regex; } }; -std::shared_ptr makeRegexCache() +ref makeRegexCache() { - return std::make_shared(); + return make_ref(); } void prim_match(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -4581,7 +4693,7 @@ void prim_match(EvalState & state, const PosIdx pos, Value ** args, Value & v) auto list = state.buildList(match.size() - 1); for (const auto & [i, v2] : enumerate(list)) if (!match[i + 1].matched) - v2 = &state.vNull; + v2 = &Value::vNull; else v2 = mkString(state, match[i + 1]); v.mkList(list); @@ -4673,7 +4785,7 @@ void prim_split(EvalState & state, const PosIdx pos, Value ** args, Value & v) auto list2 = state.buildList(slen); for (const auto & [si, v2] : enumerate(list2)) { if (!match[si + 1].matched) - v2 = &state.vNull; + v2 = &Value::vNull; else v2 = mkString(state, match[si + 1]); } @@ -4794,7 +4906,7 @@ static void prim_replaceStrings(EvalState & state, const PosIdx pos, Value ** ar from.emplace_back(state.forceString( *elem, pos, "while evaluating one of the strings to replace passed to builtins.replaceStrings")); - std::unordered_map cache; + boost::unordered_flat_map cache; auto to = args[1]->listView(); NixStringContext context; @@ -4873,7 +4985,7 @@ static void prim_parseDrvName(EvalState & state, const PosIdx pos, Value ** args state.forceStringNoCtx(*args[0], pos, "while evaluating the first argument passed to builtins.parseDrvName"); DrvName parsed(name); auto attrs = state.buildBindings(2); - attrs.alloc(state.sName).mkString(parsed.name); + attrs.alloc(state.s.name).mkString(parsed.name); attrs.alloc("version").mkString(parsed.version); v.mkAttrs(attrs); } @@ -5027,7 +5139,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) addConstant( "null", - &vNull, + &Value::vNull, { .type = nNull, .doc = R"( @@ -5042,9 +5154,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) )", }); - if (!settings.pureEval) { - v.mkInt(time(0)); - } + v.mkInt(time(0)); addConstant( "__currentTime", v, @@ -5072,8 +5182,7 @@ void EvalState::createBaseEnv(const EvalSettings & evalSettings) .impureOnly = true, }); - if (!settings.pureEval) - v.mkString(settings.getCurrentSystem()); + v.mkString(settings.getCurrentSystem()); addConstant( "__currentSystem", v, diff --git a/src/libexpr/primops/context.cc b/src/libexpr/primops/context.cc index f037fdb8045..6990bb87cbd 100644 --- a/src/libexpr/primops/context.cc +++ b/src/libexpr/primops/context.cc @@ -8,10 +8,16 @@ namespace nix { static void prim_unsafeDiscardStringContext(EvalState & state, const PosIdx pos, Value ** args, Value & v) { - NixStringContext context; + NixStringContext context, filtered; + auto s = state.coerceToString( pos, *args[0], context, "while evaluating the argument passed to builtins.unsafeDiscardStringContext"); - v.mkString(*s); + + for (auto & c : context) + if (auto * p = std::get_if(&c.raw)) + filtered.insert(*p); + + v.mkString(*s, filtered); } static RegisterPrimOp primop_unsafeDiscardStringContext({ @@ -23,11 +29,19 @@ static RegisterPrimOp primop_unsafeDiscardStringContext({ .fun = prim_unsafeDiscardStringContext, }); +bool hasContext(const NixStringContext & context) +{ + for (auto & c : context) + if (!std::get_if(&c.raw)) + return true; + return false; +} + static void prim_hasContext(EvalState & state, const PosIdx pos, Value ** args, Value & v) { NixStringContext context; state.forceString(*args[0], context, pos, "while evaluating the argument passed to builtins.hasContext"); - v.mkBool(!context.empty()); + v.mkBool(hasContext(context)); } static RegisterPrimOp primop_hasContext( @@ -62,6 +76,7 @@ static void prim_unsafeDiscardOutputDependency(EvalState & state, const PosIdx p NixStringContext context2; for (auto && c : context) { if (auto * ptr = std::get_if(&c.raw)) { + state.waitForPath(ptr->drvPath); // FIXME: why? context2.emplace(NixStringContextElem::Opaque{.path = ptr->drvPath}); } else { /* Can reuse original item */ @@ -133,6 +148,11 @@ static void prim_addDrvOutputDependencies(EvalState & state, const PosIdx pos, V above does not make much sense. */ return std::move(c); }, + [&](const NixStringContextElem::Path & p) -> NixStringContextElem::DrvDeep { + state.error("`addDrvOutputDependencies` does not work on a string without context") + .atPos(pos) + .debugThrow(); + }, }, context.begin()->raw)}), }; @@ -201,6 +221,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value ** args, contextInfos[std::move(drvPath)].outputs.emplace_back(std::move(b.output)); }, [&](NixStringContextElem::Opaque && o) { contextInfos[std::move(o.path)].path = true; }, + [&](NixStringContextElem::Path && p) {}, }, ((NixStringContextElem &&) i).raw); } @@ -219,7 +240,7 @@ static void prim_getContext(EvalState & state, const PosIdx pos, Value ** args, auto list = state.buildList(info.second.outputs.size()); for (const auto & [i, output] : enumerate(info.second.outputs)) (list[i] = state.allocValue())->mkString(output); - infoAttrs.alloc(state.sOutputs).mkList(list); + infoAttrs.alloc(state.s.outputs).mkList(list); } attrs.alloc(state.store->printStorePath(info.first)).mkAttrs(infoAttrs); } @@ -300,7 +321,7 @@ static void prim_appendContext(EvalState & state, const PosIdx pos, Value ** arg } } - if (auto attr = i.value->attrs()->get(state.sOutputs)) { + if (auto attr = i.value->attrs()->get(state.s.outputs)) { state.forceList(*attr->value, attr->pos, "while evaluating the `outputs` attribute of a string context"); if (attr->value->listSize() && !isDerivation(name)) { state diff --git a/src/libexpr/primops/fetchClosure.cc b/src/libexpr/primops/fetchClosure.cc index 63da53aa941..c74d02b0069 100644 --- a/src/libexpr/primops/fetchClosure.cc +++ b/src/libexpr/primops/fetchClosure.cc @@ -64,6 +64,8 @@ static void runFetchClosureWithRewrite( .pos = state.positions[pos]}); } + state.allowClosure(toPath); + state.mkStorePathString(toPath, v); } @@ -91,6 +93,8 @@ static void runFetchClosureWithContentAddressedPath( .pos = state.positions[pos]}); } + state.allowClosure(fromPath); + state.mkStorePathString(fromPath, v); } @@ -115,6 +119,8 @@ static void runFetchClosureWithInputAddressedPath( .pos = state.positions[pos]}); } + state.allowClosure(fromPath); + state.mkStorePathString(fromPath, v); } @@ -130,7 +136,7 @@ static void prim_fetchClosure(EvalState & state, const PosIdx pos, Value ** args std::optional inputAddressedMaybe; for (auto & attr : *args[0]->attrs()) { - const auto & attrName = state.symbols[attr.name]; + std::string_view attrName = state.symbols[attr.name]; auto attrHint = [&]() -> std::string { return fmt("while evaluating the attribute '%s' passed to builtins.fetchClosure", attrName); }; diff --git a/src/libexpr/primops/fetchMercurial.cc b/src/libexpr/primops/fetchMercurial.cc index 9fc8e6c8341..c700c181c21 100644 --- a/src/libexpr/primops/fetchMercurial.cc +++ b/src/libexpr/primops/fetchMercurial.cc @@ -81,10 +81,10 @@ static void prim_fetchMercurial(EvalState & state, const PosIdx pos, Value ** ar attrs.insert_or_assign("rev", rev->gitRev()); auto input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); - auto [storePath, input2] = input.fetchToStore(state.store); + auto [storePath, accessor, input2] = input.fetchToStore(state.store); auto attrs2 = state.buildBindings(8); - state.mkStorePathString(storePath, attrs2.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs2.alloc(state.s.outPath)); if (input2.getRef()) attrs2.alloc("branch").mkString(*input2.getRef()); // Backward compatibility: set 'rev' to diff --git a/src/libexpr/primops/fetchTree.cc b/src/libexpr/primops/fetchTree.cc index 274f758a78a..5d2d7319293 100644 --- a/src/libexpr/primops/fetchTree.cc +++ b/src/libexpr/primops/fetchTree.cc @@ -10,6 +10,7 @@ #include "nix/util/url.hh" #include "nix/expr/value-to-json.hh" #include "nix/fetchers/fetch-to-store.hh" +#include "nix/fetchers/input-cache.hh" #include @@ -29,7 +30,7 @@ void emitTreeAttrs( { auto attrs = state.buildBindings(100); - state.mkStorePathString(storePath, attrs.alloc(state.sOutPath)); + state.mkStorePathString(storePath, attrs.alloc(state.s.outPath)); // FIXME: support arbitrary input attributes. @@ -95,7 +96,7 @@ static void fetchTree( fetchers::Attrs attrs; - if (auto aType = args[0]->attrs()->get(state.sType)) { + if (auto aType = args[0]->attrs()->get(state.s.type)) { if (type) state.error("unexpected argument 'type'").atPos(pos).debugThrow(); type = state.forceStringNoCtx( @@ -106,14 +107,14 @@ static void fetchTree( attrs.emplace("type", type.value()); for (auto & attr : *args[0]->attrs()) { - if (attr.name == state.sType) + if (attr.name == state.s.type) continue; state.forceValue(*attr.value, attr.pos); if (attr.value->type() == nPath || attr.value->type() == nString) { auto s = state.coerceToString(attr.pos, *attr.value, context, "", false, false).toOwned(); attrs.emplace( state.symbols[attr.name], - params.isFetchGit && state.symbols[attr.name] == "url" ? fixGitURL(s) : s); + params.isFetchGit && state.symbols[attr.name] == "url" ? fixGitURL(s).to_string() : s); } else if (attr.value->type() == nBool) attrs.emplace(state.symbols[attr.name], Explicit{attr.value->boolean()}); else if (attr.value->type() == nInt) { @@ -175,24 +176,18 @@ static void fetchTree( if (params.isFetchGit) { fetchers::Attrs attrs; attrs.emplace("type", "git"); - attrs.emplace("url", fixGitURL(url)); + attrs.emplace("url", fixGitURL(url).to_string()); if (!attrs.contains("exportIgnore") && (!attrs.contains("submodules") || !*fetchers::maybeGetBoolAttr(attrs, "submodules"))) { attrs.emplace("exportIgnore", Explicit{true}); } input = fetchers::Input::fromAttrs(state.fetchSettings, std::move(attrs)); } else { - if (!experimentalFeatureSettings.isEnabled(Xp::Flakes)) - state - .error( - "passing a string argument to '%s' requires the 'flakes' experimental feature", fetcher) - .atPos(pos) - .debugThrow(); input = fetchers::Input::fromURL(state.fetchSettings, url); } } - if (!state.settings.pureEval && !input.isDirect() && experimentalFeatureSettings.isEnabled(Xp::Flakes)) + if (!state.settings.pureEval && !input.isDirect()) input = lookupInRegistries(state.store, input, fetchers::UseRegistries::Limited).first; if (state.settings.pureEval && !input.isLocked()) { @@ -218,11 +213,11 @@ static void fetchTree( throw Error("input '%s' is not allowed to use the '__final' attribute", input.to_string()); } - auto [storePath, input2] = input.fetchToStore(state.store); + auto cachedInput = state.inputCache->getAccessor(state.store, input, fetchers::UseRegistries::No); - state.allowPath(storePath); + auto storePath = state.mountInput(cachedInput.lockedInput, input, cachedInput.accessor, true); - emitTreeAttrs(state, storePath, input2, v, params.emptyRevFallback, false); + emitTreeAttrs(state, storePath, cachedInput.lockedInput, v, params.emptyRevFallback, false); } static void prim_fetchTree(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -420,7 +415,6 @@ static RegisterPrimOp primop_fetchTree({ - `"mercurial"` *input* can also be a [URL-like reference](@docroot@/command-ref/new-cli/nix3-flake.md#flake-references). - The additional input types and the URL-like syntax requires the [`flakes` experimental feature](@docroot@/development/experimental-features.md#xp-feature-flakes) to be enabled. > **Example** > @@ -457,7 +451,6 @@ static RegisterPrimOp primop_fetchTree({ > ``` )", .fun = prim_fetchTree, - .experimentalFeature = Xp::FetchTree, }); void prim_fetchFinalTree(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -561,14 +554,22 @@ static void fetch( .hash = *expectedHash, .references = {}}); - if (state.store->isValidPath(expectedPath)) { + // Try to get the path from the local store or substituters + try { + state.store->ensurePath(expectedPath); + debug("using substituted/cached path '%s' for '%s'", state.store->printStorePath(expectedPath), *url); state.allowAndSetStorePathString(expectedPath, v); return; + } catch (Error & e) { + debug( + "substitution of '%s' failed, will try to download: %s", + state.store->printStorePath(expectedPath), + e.what()); + // Fall through to download } } - // TODO: fetching may fail, yet the path may be substitutable. - // https://github.com/NixOS/nix/issues/4313 + // Download the file/tarball if substitution failed or no hash was provided auto storePath = unpack ? fetchToStore( state.fetchSettings, *state.store, @@ -806,7 +807,7 @@ static RegisterPrimOp primop_fetchGit({ name in the `ref` attribute. However, if the revision you're looking for is in a future - branch for the non-default branch you will need to specify the + branch for the non-default branch you need to specify the the `ref` attribute as well. ```nix diff --git a/src/libexpr/primops/fromTOML.cc b/src/libexpr/primops/fromTOML.cc index 7d98a5de985..d2f91a75b63 100644 --- a/src/libexpr/primops/fromTOML.cc +++ b/src/libexpr/primops/fromTOML.cc @@ -136,10 +136,10 @@ static void prim_fromTOML(EvalState & state, const PosIdx pos, Value ** args, Va normalizeDatetimeFormat(t); #endif auto attrs = state.buildBindings(2); - attrs.alloc("_type").mkString("timestamp"); + attrs.alloc("_type").mkStringNoCopy("timestamp"); std::ostringstream s; s << t; - auto str = toView(s); + auto str = s.view(); forceNoNullByte(str); attrs.alloc("value").mkString(str); v.mkAttrs(attrs); diff --git a/src/libexpr/print-ambiguous.cc b/src/libexpr/print-ambiguous.cc index 8b80e2a6634..f80ef2b044b 100644 --- a/src/libexpr/print-ambiguous.cc +++ b/src/libexpr/print-ambiguous.cc @@ -6,8 +6,7 @@ namespace nix { // See: https://github.com/NixOS/nix/issues/9730 -void printAmbiguous( - Value & v, const SymbolTable & symbols, std::ostream & str, std::set * seen, int depth) +void printAmbiguous(EvalState & state, Value & v, std::ostream & str, std::set * seen, int depth) { checkInterrupt(); @@ -22,9 +21,13 @@ void printAmbiguous( case nBool: printLiteralBool(str, v.boolean()); break; - case nString: - printLiteralString(str, v.string_view()); + case nString: { + NixStringContext context; + copyContext(v, context); + // FIXME: make devirtualization configurable? + printLiteralString(str, state.devirtualize(v.string_view(), context)); break; + } case nPath: str << v.path().to_string(); // !!! escaping? break; @@ -36,9 +39,9 @@ void printAmbiguous( str << "«repeated»"; else { str << "{ "; - for (auto & i : v.attrs()->lexicographicOrder(symbols)) { - str << symbols[i->name] << " = "; - printAmbiguous(*i->value, symbols, str, seen, depth - 1); + for (auto & i : v.attrs()->lexicographicOrder(state.symbols)) { + str << state.symbols[i->name] << " = "; + printAmbiguous(state, *i->value, str, seen, depth - 1); str << "; "; } str << "}"; @@ -54,7 +57,7 @@ void printAmbiguous( str << "[ "; for (auto v2 : v.listView()) { if (v2) - printAmbiguous(*v2, symbols, str, seen, depth - 1); + printAmbiguous(state, *v2, str, seen, depth - 1); else str << "(nullptr)"; str << " "; @@ -75,6 +78,9 @@ void printAmbiguous( str << "«potential infinite recursion»"; } break; + case nFailed: + str << "«failed»"; + break; case nFunction: if (v.isLambda()) { str << ""; diff --git a/src/libexpr/print.cc b/src/libexpr/print.cc index 502f32ea186..bf856db45d5 100644 --- a/src/libexpr/print.cc +++ b/src/libexpr/print.cc @@ -1,5 +1,4 @@ #include -#include #include #include "nix/expr/print.hh" @@ -10,6 +9,8 @@ #include "nix/util/english.hh" #include "nix/expr/eval.hh" +#include + namespace nix { void printElided( @@ -81,7 +82,7 @@ std::ostream & printLiteralBool(std::ostream & str, bool boolean) // For example `or' doesn't need to be quoted. bool isReservedKeyword(const std::string_view str) { - static const std::unordered_set reservedKeywords = { + static const boost::unordered_flat_set reservedKeywords = { "if", "then", "else", "assert", "with", "let", "in", "rec", "inherit"}; return reservedKeywords.contains(str); } @@ -248,7 +249,11 @@ class Printer void printString(Value & v) { - printLiteralString(output, v.string_view(), options.maxStringLength, options.ansiColors); + NixStringContext context; + copyContext(v, context); + std::ostringstream s; + printLiteralString(s, v.string_view(), options.maxStringLength, options.ansiColors); + output << state.devirtualize(s.str(), context); } void printPath(Value & v) @@ -272,7 +277,7 @@ class Printer void printDerivation(Value & v) { std::optional storePath; - if (auto i = v.attrs()->get(state.sDrvPath)) { + if (auto i = v.attrs()->get(state.s.drvPath)) { NixStringContext context; storePath = state.coerceToStorePath(i->pos, *i->value, context, "while evaluating the drvPath of a derivation"); @@ -460,7 +465,7 @@ class Printer std::ostringstream s; s << state.positions[v.lambda().fun->pos]; - output << " @ " << filterANSIEscapes(toView(s)); + output << " @ " << filterANSIEscapes(s.view()); } } else if (v.isPrimOp()) { if (v.primOp()) @@ -497,7 +502,7 @@ class Printer output << "«potential infinite recursion»"; if (options.ansiColors) output << ANSI_NORMAL; - } else if (v.isThunk() || v.isApp()) { + } else if (!v.isFinished()) { if (options.ansiColors) output << ANSI_MAGENTA; output << "«thunk»"; @@ -508,6 +513,11 @@ class Printer } } + void printFailed(Value & v) + { + output << "«failed»"; + } + void printExternal(Value & v) { v.external()->print(output); @@ -583,6 +593,10 @@ class Printer printThunk(v); break; + case nFailed: + printFailed(v); + break; + case nExternal: printExternal(v); break; diff --git a/src/libexpr/symbol-table.cc b/src/libexpr/symbol-table.cc new file mode 100644 index 00000000000..1e0d31997f2 --- /dev/null +++ b/src/libexpr/symbol-table.cc @@ -0,0 +1,62 @@ +#include "nix/expr/symbol-table.hh" +#include "nix/util/logging.hh" + +#include + +namespace nix { + +#ifndef MAP_NORESERVE +# define MAP_NORESERVE 0 +#endif + +static void * allocateLazyMemory(size_t maxSize) +{ + auto p = mmap(nullptr, maxSize, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE, -1, 0); + if (p == MAP_FAILED) + throw SysError("allocating arena using mmap"); + return p; +} + +ContiguousArena::ContiguousArena(size_t maxSize) + : data((char *) allocateLazyMemory(maxSize)) + , maxSize(maxSize) +{ +} + +size_t ContiguousArena::allocate(size_t bytes) +{ + auto offset = size.fetch_add(bytes); + if (offset + bytes > maxSize) + throw Error("arena ran out of space"); + return offset; +} + +Symbol SymbolTable::create(std::string_view s) +{ + uint32_t idx; + + auto visit = [&](const SymbolStr & sym) { idx = ((const char *) sym.s) - arena.data; }; + + symbols.insert_and_visit(SymbolStr::Key{s, arena}, visit, visit); + + return Symbol(idx); +} + +SymbolStr::SymbolStr(const SymbolStr::Key & key) +{ + auto size = SymbolStr::computeSize(key.s); + + auto id = key.arena.allocate(size); + + auto v = (SymbolValue *) (const_cast(key.arena.data) + id); + auto s = (char *) (v + 1); + + memcpy(s, key.s.data(), key.s.size()); + s[key.s.size()] = 0; + + v->mkStringNoCopy(s, nullptr); + + this->s = v; +} + +} // namespace nix diff --git a/src/libexpr/value-to-json.cc b/src/libexpr/value-to-json.cc index 2578620f339..d1cdcccf12c 100644 --- a/src/libexpr/value-to-json.cc +++ b/src/libexpr/value-to-json.cc @@ -2,104 +2,147 @@ #include "nix/expr/eval-inline.hh" #include "nix/store/store-api.hh" #include "nix/util/signals.hh" +#include "nix/expr/parallel-eval.hh" #include #include #include namespace nix { + using json = nlohmann::json; +#pragma GCC diagnostic ignored "-Wswitch-enum" + +static void parallelForceDeep(EvalState & state, Value & v, PosIdx pos) +{ + state.forceValue(v, pos); + + std::vector> work; + + switch (v.type()) { + + case nAttrs: { + NixStringContext context; + if (state.tryAttrsToString(pos, v, context, false, false)) + return; + if (v.attrs()->get(state.s.outPath)) + return; + for (auto & a : *v.attrs()) + work.emplace_back( + [value(allocRootValue(a.value)), pos(a.pos), &state]() { parallelForceDeep(state, **value, pos); }, 0); + break; + } + + default: + break; + } + + state.executor->spawn(std::move(work)); +} + // TODO: rename. It doesn't print. json printValueAsJSON( EvalState & state, bool strict, Value & v, const PosIdx pos, NixStringContext & context, bool copyToStore) { - checkInterrupt(); + if (strict && state.executor->enabled && !Executor::amWorkerThread) + parallelForceDeep(state, v, pos); - if (strict) - state.forceValue(v, pos); + std::function recurse; - json out; + recurse = [&](json & res, Value & v, PosIdx pos) { + checkInterrupt(); - switch (v.type()) { + if (strict) + state.forceValue(v, pos); - case nInt: - out = v.integer().value; - break; + switch (v.type()) { - case nBool: - out = v.boolean(); - break; + case nInt: + res = v.integer().value; + break; - case nString: - copyContext(v, context); - out = v.c_str(); - break; + case nBool: + res = v.boolean(); + break; - case nPath: - if (copyToStore) - out = state.store->printStorePath(state.copyPathToStore(context, v.path())); - else - out = v.path().path.abs(); - break; + case nString: { + copyContext(v, context); + res = v.c_str(); + break; + } - case nNull: - // already initialized as null - break; + case nPath: + if (copyToStore) + res = state.store->printStorePath(state.copyPathToStore(context, v.path(), v.determinePos(pos))); + else + res = v.path().path.abs(); + break; - case nAttrs: { - auto maybeString = state.tryAttrsToString(pos, v, context, false, false); - if (maybeString) { - out = *maybeString; + case nNull: + // already initialized as null + break; + + case nAttrs: { + auto maybeString = state.tryAttrsToString(pos, v, context, false, false); + if (maybeString) { + res = *maybeString; + break; + } + if (auto i = v.attrs()->get(state.s.outPath)) + return recurse(res, *i->value, i->pos); + else { + res = json::object(); + for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) { + json & j = res.emplace(state.symbols[a->name], json()).first.value(); + try { + recurse(j, *a->value, a->pos); + } catch (Error & e) { + e.addTrace( + state.positions[a->pos], + HintFmt("while evaluating attribute '%1%'", state.symbols[a->name])); + throw; + } + } + } break; } - if (auto i = v.attrs()->get(state.sOutPath)) - return printValueAsJSON(state, strict, *i->value, i->pos, context, copyToStore); - else { - out = json::object(); - for (auto & a : v.attrs()->lexicographicOrder(state.symbols)) { + + case nList: { + res = json::array(); + for (const auto & [i, elem] : enumerate(v.listView())) { try { - out.emplace( - state.symbols[a->name], - printValueAsJSON(state, strict, *a->value, a->pos, context, copyToStore)); + res.push_back(json()); + recurse(res.back(), *elem, pos); } catch (Error & e) { - e.addTrace( - state.positions[a->pos], HintFmt("while evaluating attribute '%1%'", state.symbols[a->name])); + e.addTrace(state.positions[pos], HintFmt("while evaluating list element at index %1%", i)); throw; } } + break; } - break; - } - case nList: { - out = json::array(); - int i = 0; - for (auto elem : v.listView()) { - try { - out.push_back(printValueAsJSON(state, strict, *elem, pos, context, copyToStore)); - } catch (Error & e) { - e.addTrace(state.positions[pos], HintFmt("while evaluating list element at index %1%", i)); - throw; - } - i++; + case nExternal: { + res = v.external()->printValueAsJSON(state, strict, context, copyToStore); + break; } - break; - } - case nExternal: - return v.external()->printValueAsJSON(state, strict, context, copyToStore); - break; + case nFloat: + res = v.fpoint(); + break; - case nFloat: - out = v.fpoint(); - break; + case nThunk: + case nFailed: + case nFunction: + state.error("cannot convert %1% to JSON", showType(v)).atPos(v.determinePos(pos)).debugThrow(); + } + }; - case nThunk: - case nFunction: - state.error("cannot convert %1% to JSON", showType(v)).atPos(v.determinePos(pos)).debugThrow(); - } - return out; + json res; + + recurse(res, v, pos); + + return res; } void printValueAsJSON( diff --git a/src/libexpr/value-to-xml.cc b/src/libexpr/value-to-xml.cc index b3b986dae78..74529d5c0c6 100644 --- a/src/libexpr/value-to-xml.cc +++ b/src/libexpr/value-to-xml.cc @@ -98,14 +98,14 @@ static void printValueAsXML( XMLAttrs xmlAttrs; Path drvPath; - if (auto a = v.attrs()->get(state.sDrvPath)) { + if (auto a = v.attrs()->get(state.s.drvPath)) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) xmlAttrs["drvPath"] = drvPath = a->value->c_str(); } - if (auto a = v.attrs()->get(state.sOutPath)) { + if (auto a = v.attrs()->get(state.s.outPath)) { if (strict) state.forceValue(*a->value, a->pos); if (a->value->type() == nString) @@ -170,6 +170,11 @@ static void printValueAsXML( case nThunk: doc.writeEmptyElement("unevaluated"); + break; + + case nFailed: + doc.writeEmptyElement("failed"); + break; } } diff --git a/src/libexpr/value.cc b/src/libexpr/value.cc new file mode 100644 index 00000000000..07d036b0dd4 --- /dev/null +++ b/src/libexpr/value.cc @@ -0,0 +1,29 @@ +#include "nix/expr/value.hh" + +namespace nix { + +Value Value::vEmptyList = []() { + Value res; + res.setStorage(List{.size = 0, .elems = nullptr}); + return res; +}(); + +Value Value::vNull = []() { + Value res; + res.mkNull(); + return res; +}(); + +Value Value::vTrue = []() { + Value res; + res.mkBool(true); + return res; +}(); + +Value Value::vFalse = []() { + Value res; + res.mkBool(false); + return res; +}(); + +} // namespace nix diff --git a/src/libexpr/value/context.cc b/src/libexpr/value/context.cc index 6eb3132110d..d0c140ef795 100644 --- a/src/libexpr/value/context.cc +++ b/src/libexpr/value/context.cc @@ -51,6 +51,11 @@ NixStringContextElem NixStringContextElem::parse(std::string_view s0, const Expe .drvPath = StorePath{s.substr(1)}, }; } + case '@': { + return NixStringContextElem::Path{ + .storePath = StorePath{s.substr(1)}, + }; + } default: { // Ensure no '!' if (s.find("!") != std::string_view::npos) { @@ -91,6 +96,10 @@ std::string NixStringContextElem::to_string() const res += '='; res += d.drvPath.to_string(); }, + [&](const NixStringContextElem::Path & p) { + res += '@'; + res += p.storePath.to_string(); + }, }, raw); diff --git a/src/libfetchers-c/meson.build b/src/libfetchers-c/meson.build index 8542744b4da..3761b0df23b 100644 --- a/src/libfetchers-c/meson.build +++ b/src/libfetchers-c/meson.build @@ -32,6 +32,7 @@ add_project_arguments( ) subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'nix_api_fetchers.cc', @@ -53,6 +54,7 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nixfetchersc', sources, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, diff --git a/src/libfetchers-c/nix_api_fetchers.cc b/src/libfetchers-c/nix_api_fetchers.cc index 4e8037a5e5d..7fefedb0c70 100644 --- a/src/libfetchers-c/nix_api_fetchers.cc +++ b/src/libfetchers-c/nix_api_fetchers.cc @@ -2,6 +2,8 @@ #include "nix_api_fetchers_internal.hh" #include "nix_api_util_internal.h" +extern "C" { + nix_fetchers_settings * nix_fetchers_settings_new(nix_c_context * context) { try { @@ -17,3 +19,5 @@ void nix_fetchers_settings_free(nix_fetchers_settings * settings) { delete settings; } + +} // extern "C" diff --git a/src/libfetchers-tests/access-tokens.cc b/src/libfetchers-tests/access-tokens.cc index 7127434db9d..26cdcfb83fc 100644 --- a/src/libfetchers-tests/access-tokens.cc +++ b/src/libfetchers-tests/access-tokens.cc @@ -15,10 +15,7 @@ class AccessKeysTest : public ::testing::Test protected: public: - void SetUp() override - { - experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); - } + void SetUp() override {} void TearDown() override {} }; diff --git a/src/libfetchers-tests/git-utils.cc b/src/libfetchers-tests/git-utils.cc index f9fae23da56..a7eb55fc206 100644 --- a/src/libfetchers-tests/git-utils.cc +++ b/src/libfetchers-tests/git-utils.cc @@ -115,9 +115,10 @@ TEST_F(GitUtilsTest, sink_hardlink) try { sink->createHardlink(CanonPath("foo-1.1/link"), CanonPath("hello")); + sink->flush(); FAIL() << "Expected an exception"; } catch (const nix::Error & e) { - ASSERT_THAT(e.msg(), testing::HasSubstr("cannot find hard link target")); + ASSERT_THAT(e.msg(), testing::HasSubstr("does not exist")); ASSERT_THAT(e.msg(), testing::HasSubstr("/hello")); ASSERT_THAT(e.msg(), testing::HasSubstr("foo-1.1/link")); } diff --git a/src/libfetchers-tests/git.cc b/src/libfetchers-tests/git.cc index af987e26002..4f0e0d97479 100644 --- a/src/libfetchers-tests/git.cc +++ b/src/libfetchers-tests/git.cc @@ -1,5 +1,6 @@ #include "nix/store/store-open.hh" #include "nix/store/globals.hh" +#include "nix/store/dummy-store.hh" #include "nix/fetchers/fetch-settings.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/git-utils.hh" @@ -179,10 +180,11 @@ TEST_F(GitTest, submodulePeriodSupport) // 6) Commit the addition in super commitAll(super.get(), "Add submodule with branch='.'"); - // TODO: Use dummy:// store with MemorySourceAccessor. - Path storeTmpDir = createTempDir(); - auto storeTmpDirAutoDelete = AutoDelete(storeTmpDir, true); - ref store = openStore(storeTmpDir); + auto store = [] { + auto cfg = make_ref(StoreReference::Params{}); + cfg->readOnly = false; + return cfg->openStore(); + }(); auto settings = fetchers::Settings{}; auto input = fetchers::Input::fromAttrs( diff --git a/src/libfetchers-tests/meson.build b/src/libfetchers-tests/meson.build index a18f64d7981..858d7f3af9a 100644 --- a/src/libfetchers-tests/meson.build +++ b/src/libfetchers-tests/meson.build @@ -37,6 +37,7 @@ libgit2 = dependency('libgit2') deps_private += libgit2 subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'access-tokens.cc', @@ -63,7 +64,7 @@ this_exe = executable( test( meson.project_name(), this_exe, - env : { + env : asan_test_options_env + { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', }, protocol : 'gtest', diff --git a/src/libfetchers-tests/package.nix b/src/libfetchers-tests/package.nix index 78061872582..8e82430d7d9 100644 --- a/src/libfetchers-tests/package.nix +++ b/src/libfetchers-tests/package.nix @@ -61,6 +61,7 @@ mkMesonExecutable (finalAttrs: { buildInputs = [ writableTmpDirAsHomeHook ]; } '' + export ASAN_OPTIONS=abort_on_error=1:print_summary=1:detect_leaks=0 export _NIX_TEST_UNIT_DATA=${resolvePath ./data} ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out diff --git a/src/libfetchers-tests/public-key.cc b/src/libfetchers-tests/public-key.cc index 97a23244793..2991223f6b3 100644 --- a/src/libfetchers-tests/public-key.cc +++ b/src/libfetchers-tests/public-key.cc @@ -1,14 +1,14 @@ #include #include "nix/fetchers/fetchers.hh" #include "nix/util/json-utils.hh" -#include -#include "nix/util/tests/characterization.hh" +#include "nix/util/tests/json-characterization.hh" namespace nix { using nlohmann::json; -class PublicKeyTest : public CharacterizationTest +class PublicKeyTest : public JsonCharacterizationTest, + public ::testing::WithParamInterface> { std::filesystem::path unitTestData = getUnitTestData() / "public-key"; @@ -19,30 +19,35 @@ class PublicKeyTest : public CharacterizationTest } }; -#define TEST_JSON(FIXTURE, NAME, VAL) \ - TEST_F(FIXTURE, PublicKey_##NAME##_from_json) \ - { \ - readTest(#NAME ".json", [&](const auto & encoded_) { \ - fetchers::PublicKey expected{VAL}; \ - fetchers::PublicKey got = nlohmann::json::parse(encoded_); \ - ASSERT_EQ(got, expected); \ - }); \ - } \ - \ - TEST_F(FIXTURE, PublicKey_##NAME##_to_json) \ - { \ - writeTest( \ - #NAME ".json", \ - [&]() -> json { return nlohmann::json(fetchers::PublicKey{VAL}); }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ - } - -TEST_JSON(PublicKeyTest, simple, (fetchers::PublicKey{.type = "ssh-rsa", .key = "ABCDE"})) +TEST_P(PublicKeyTest, from_json) +{ + const auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} -TEST_JSON(PublicKeyTest, defaultType, fetchers::PublicKey{.key = "ABCDE"}) +TEST_P(PublicKeyTest, to_json) +{ + const auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} -#undef TEST_JSON +INSTANTIATE_TEST_SUITE_P( + PublicKeyJSON, + PublicKeyTest, + ::testing::Values( + std::pair{ + "simple", + fetchers::PublicKey{ + .type = "ssh-rsa", + .key = "ABCDE", + }, + }, + std::pair{ + "defaultType", + fetchers::PublicKey{ + .key = "ABCDE", + }, + })); TEST_F(PublicKeyTest, PublicKey_noRoundTrip_from_json) { diff --git a/src/libfetchers/builtin.cc b/src/libfetchers/builtin.cc new file mode 100644 index 00000000000..c1a912c25c6 --- /dev/null +++ b/src/libfetchers/builtin.cc @@ -0,0 +1,60 @@ +#include "nix/store/builtins.hh" +#include "nix/store/parsed-derivations.hh" +#include "nix/fetchers/fetchers.hh" +#include "nix/fetchers/fetch-settings.hh" +#include "nix/util/archive.hh" +#include "nix/store/filetransfer.hh" +#include "nix/store/store-open.hh" + +#include + +namespace nix { + +static void builtinFetchTree(const BuiltinBuilderContext & ctx) +{ + experimentalFeatureSettings.require(Xp::BuildTimeFetchTree); + + auto out = get(ctx.drv.outputs, "out"); + if (!out) + throw Error("'builtin:fetch-tree' requires an 'out' output"); + + if (!(ctx.drv.type().isFixed() || ctx.drv.type().isImpure())) + throw Error("'builtin:fetch-tree' must be a fixed-output or impure derivation"); + + if (!ctx.drv.structuredAttrs) + throw Error("'builtin:fetch-tree' must have '__structuredAttrs = true'"); + + setenv("NIX_CACHE_HOME", ctx.tmpDirInSandbox.c_str(), 1); + + using namespace fetchers; + + fetchers::Settings myFetchSettings; + myFetchSettings.accessTokens = fetchSettings.accessTokens.get(); + + // Make sure we don't use the FileTransfer object of the parent + // since it's in a broken state after the fork. We also must not + // delete it, so hang on to the shared_ptr. + // FIXME: move FileTransfer into fetchers::Settings. + static auto prevFileTransfer = resetFileTransfer(); + + // FIXME: disable use of the git/tarball cache + + auto input = Input::fromAttrs(myFetchSettings, jsonToAttrs(ctx.drv.structuredAttrs->structuredAttrs["input"])); + + std::cerr << fmt("fetching '%s'...\n", input.to_string()); + + /* Functions like downloadFile() expect a store. We can't use the + real one since we're in a forked process. FIXME: use recursive + Nix's daemon so we can use the real store? */ + auto tmpStore = openStore(ctx.tmpDirInSandbox + "/nix"); + + auto [accessor, lockedInput] = input.getAccessor(tmpStore); + + auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); }); + + restorePath(ctx.outputs.at("out"), *source); +} + +static RegisterBuiltinBuilder registerUnpackChannel("fetch-tree", builtinFetchTree); + +} // namespace nix diff --git a/src/libfetchers/cache.cc b/src/libfetchers/cache.cc index 67361c7657d..87fe3391c2c 100644 --- a/src/libfetchers/cache.cc +++ b/src/libfetchers/cache.cc @@ -109,7 +109,7 @@ struct CacheImpl : Cache upsert(key, value); } - std::optional lookupStorePath(Key key, Store & store) override + std::optional lookupStorePath(Key key, Store & store, bool allowInvalid) override { key.second.insert_or_assign("store", store.storeDir); @@ -123,7 +123,7 @@ struct CacheImpl : Cache ResultWithStorePath res2(*res, StorePath(storePathS)); store.addTempRoot(res2.storePath); - if (!store.isValidPath(res2.storePath)) { + if (!allowInvalid && !store.isValidPath(res2.storePath)) { // FIXME: we could try to substitute 'storePath'. debug( "ignoring disappeared cache entry '%s:%s' -> '%s'", @@ -145,7 +145,7 @@ struct CacheImpl : Cache std::optional lookupStorePathWithTTL(Key key, Store & store) override { - auto res = lookupStorePath(std::move(key), store); + auto res = lookupStorePath(std::move(key), store, false); return res && !res->expired ? res : std::nullopt; } }; diff --git a/src/libfetchers/fetch-settings.cc b/src/libfetchers/fetch-settings.cc index f92b94a0b3b..f50177f094e 100644 --- a/src/libfetchers/fetch-settings.cc +++ b/src/libfetchers/fetch-settings.cc @@ -1,7 +1,16 @@ #include "nix/fetchers/fetch-settings.hh" +#include "nix/util/config-global.hh" namespace nix::fetchers { Settings::Settings() {} } // namespace nix::fetchers + +namespace nix { + +fetchers::Settings fetchSettings; + +static GlobalConfig::Register rFetchSettings(&fetchSettings); + +} // namespace nix diff --git a/src/libfetchers/fetch-to-store.cc b/src/libfetchers/fetch-to-store.cc index 6ce78e115be..4b58ea16d05 100644 --- a/src/libfetchers/fetch-to-store.cc +++ b/src/libfetchers/fetch-to-store.cc @@ -1,15 +1,15 @@ #include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/fetchers.hh" #include "nix/fetchers/fetch-settings.hh" +#include "nix/util/environment-variables.hh" namespace nix { -fetchers::Cache::Key makeFetchToStoreCacheKey( - const std::string & name, const std::string & fingerprint, ContentAddressMethod method, const std::string & path) +fetchers::Cache::Key +makeSourcePathToHashCacheKey(const std::string & fingerprint, ContentAddressMethod method, const std::string & path) { return fetchers::Cache::Key{ - "fetchToStore", - {{"name", name}, {"fingerprint", fingerprint}, {"method", std::string{method.render()}}, {"path", path}}}; + "sourcePathToHash", {{"fingerprint", fingerprint}, {"method", std::string{method.render()}}, {"path", path}}}; } StorePath fetchToStore( @@ -22,19 +22,47 @@ StorePath fetchToStore( PathFilter * filter, RepairFlag repair) { - // FIXME: add an optimisation for the case where the accessor is - // a `PosixSourceAccessor` pointing to a store path. + return fetchToStore2(settings, store, path, mode, name, method, filter, repair).first; +} +std::pair fetchToStore2( + const fetchers::Settings & settings, + Store & store, + const SourcePath & path, + FetchMode mode, + std::string_view name, + ContentAddressMethod method, + PathFilter * filter, + RepairFlag repair) +{ std::optional cacheKey; - if (!filter && path.accessor->fingerprint) { - cacheKey = makeFetchToStoreCacheKey(std::string{name}, *path.accessor->fingerprint, method, path.path.abs()); - if (auto res = settings.getCache()->lookupStorePath(*cacheKey, store)) { - debug("store path cache hit for '%s'", path); - return res->storePath; + auto [subpath, fingerprint] = filter ? std::pair>{path.path, std::nullopt} + : path.accessor->getFingerprint(path.path); + + if (fingerprint) { + cacheKey = makeSourcePathToHashCacheKey(*fingerprint, method, subpath.abs()); + if (auto res = settings.getCache()->lookup(*cacheKey)) { + auto hash = Hash::parseSRI(fetchers::getStrAttr(*res, "hash")); + auto storePath = + store.makeFixedOutputPathFromCA(name, ContentAddressWithReferences::fromParts(method, hash, {})); + if (mode == FetchMode::DryRun || store.maybeQueryPathInfo(storePath)) { + debug( + "source path '%s' cache hit in '%s' (hash '%s')", + path, + store.printStorePath(storePath), + hash.to_string(HashFormat::SRI, true)); + return {storePath, hash}; + } + debug("source path '%s' not in store", path); } - } else + } else { + static auto barf = getEnv("_NIX_TEST_BARF_ON_UNCACHEABLE").value_or("") == "1"; + if (barf && !filter) + throw Error("source path '%s' is uncacheable (filter=%d)", path, (bool) filter); + // FIXME: could still provide in-memory caching keyed on `SourcePath`. debug("source path '%s' is uncacheable", path); + } Activity act( *logger, @@ -44,16 +72,41 @@ StorePath fetchToStore( auto filter2 = filter ? *filter : defaultPathFilter; - auto storePath = mode == FetchMode::DryRun - ? store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2).first - : store.addToStore(name, path, method, HashAlgorithm::SHA256, {}, filter2, repair); - - debug(mode == FetchMode::DryRun ? "hashed '%s'" : "copied '%s' to '%s'", path, store.printStorePath(storePath)); + auto [storePath, hash] = + mode == FetchMode::DryRun + ? ({ + auto [storePath, hash] = + store.computeStorePath(name, path, method, HashAlgorithm::SHA256, {}, filter2); + debug( + "hashed '%s' to '%s' (hash '%s')", + path, + store.printStorePath(storePath), + hash.to_string(HashFormat::SRI, true)); + std::make_pair(storePath, hash); + }) + : ({ + // FIXME: ideally addToStore() would return the hash + // right away (like computeStorePath()). + auto storePath = store.addToStore(name, path, method, HashAlgorithm::SHA256, {}, filter2, repair); + auto info = store.queryPathInfo(storePath); + assert(info->references.empty()); + auto hash = method == ContentAddressMethod::Raw::NixArchive ? info->narHash : ({ + if (!info->ca || info->ca->method != method) + throw Error("path '%s' lacks a CA field", store.printStorePath(storePath)); + info->ca->hash; + }); + debug( + "copied '%s' to '%s' (hash '%s')", + path, + store.printStorePath(storePath), + hash.to_string(HashFormat::SRI, true)); + std::make_pair(storePath, hash); + }); - if (cacheKey && mode == FetchMode::Copy) - settings.getCache()->upsert(*cacheKey, store, {}, storePath); + if (cacheKey) + settings.getCache()->upsert(*cacheKey, {{"hash", hash.to_string(HashFormat::SRI, true)}}); - return storePath; + return {storePath, hash}; } } // namespace nix diff --git a/src/libfetchers/fetchers.cc b/src/libfetchers/fetchers.cc index 54013bf556e..b5bfb4a085c 100644 --- a/src/libfetchers/fetchers.cc +++ b/src/libfetchers/fetchers.cc @@ -3,8 +3,10 @@ #include "nix/util/source-path.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/json-utils.hh" -#include "nix/fetchers/store-path-accessor.hh" #include "nix/fetchers/fetch-settings.hh" +#include "nix/fetchers/fetch-to-store.hh" +#include "nix/util/forwarding-source-accessor.hh" +#include "nix/util/archive.hh" #include @@ -191,35 +193,30 @@ bool Input::contains(const Input & other) const } // FIXME: remove -std::pair Input::fetchToStore(ref store) const +std::tuple, Input> Input::fetchToStore(ref store) const { if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); - auto [storePath, input] = [&]() -> std::pair { - try { - auto [accessor, result] = getAccessorUnchecked(store); - - auto storePath = - nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName()); + try { + auto [accessor, result] = getAccessorUnchecked(store); - auto narHash = store->queryPathInfo(storePath)->narHash; - result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); + auto storePath = nix::fetchToStore(*settings, *store, SourcePath(accessor), FetchMode::Copy, result.getName()); - result.attrs.insert_or_assign("__final", Explicit(true)); + auto narHash = store->queryPathInfo(storePath)->narHash; + result.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - assert(result.isFinal()); + result.attrs.insert_or_assign("__final", Explicit(true)); - checkLocks(*this, result); + assert(result.isFinal()); - return {storePath, result}; - } catch (Error & e) { - e.addTrace({}, "while fetching the input '%s'", to_string()); - throw; - } - }(); + checkLocks(*this, result); - return {std::move(storePath), input}; + return {std::move(storePath), accessor, result}; + } catch (Error & e) { + e.addTrace({}, "while fetching the input '%s'", to_string()); + throw; + } } void Input::checkLocks(Input specified, Input & result) @@ -237,6 +234,9 @@ void Input::checkLocks(Input specified, Input & result) if (auto prevNarHash = specified.getNarHash()) specified.attrs.insert_or_assign("narHash", prevNarHash->to_string(HashFormat::SRI, true)); + if (auto narHash = result.getNarHash()) + result.attrs.insert_or_assign("narHash", narHash->to_string(HashFormat::SRI, true)); + for (auto & field : specified.attrs) { auto field2 = result.attrs.find(field.first); if (field2 != result.attrs.end() && field.second != field2->second) @@ -306,6 +306,21 @@ std::pair, Input> Input::getAccessor(ref store) const } } +/** + * Helper class that ensures that paths in substituted source trees + * are rendered as `«input»/path` rather than + * `«input»/nix/store/-source/path`. + */ +struct SubstitutedSourceAccessor : ForwardingSourceAccessor +{ + using ForwardingSourceAccessor::ForwardingSourceAccessor; + + std::string showPath(const CanonPath & path) override + { + return displayPrefix + path.abs() + displaySuffix; + } +}; + std::pair, Input> Input::getAccessorUnchecked(ref store) const { // FIXME: cache the accessor @@ -313,43 +328,68 @@ std::pair, Input> Input::getAccessorUnchecked(ref sto if (!scheme) throw Error("cannot fetch unsupported input '%s'", attrsToJSON(toAttrs())); - /* The tree may already be in the Nix store, or it could be - substituted (which is often faster than fetching from the - original source). So check that. We only do this for final - inputs, otherwise there is a risk that we don't return the - same attributes (like `lastModified`) that the "real" fetcher - would return. - - FIXME: add a setting to disable this. - FIXME: substituting may be slower than fetching normally, - e.g. for fetchers like Git that are incremental! - */ - if (isFinal() && getNarHash()) { - try { - auto storePath = computeStorePath(*store); + std::optional storePath; + if (isFinal() && getNarHash()) + storePath = computeStorePath(*store); - store->ensurePath(storePath); + auto makeStoreAccessor = [&]() -> std::pair, Input> { + auto accessor = make_ref(ref{store->getFSAccessor(*storePath)}); - debug("using substituted/cached input '%s' in '%s'", to_string(), store->printStorePath(storePath)); + accessor->fingerprint = getFingerprint(store); - auto accessor = makeStorePathAccessor(store, storePath); + // Store a cache entry for the substituted tree so later fetches + // can reuse the existing nar instead of copying the unpacked + // input back into the store on every evaluation. + if (accessor->fingerprint) { + settings->getCache()->upsert( + makeSourcePathToHashCacheKey(*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"), + {{"hash", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true)}}); + } - accessor->fingerprint = getFingerprint(store); + // FIXME: ideally we would use the `showPath()` of the + // "real" accessor for this fetcher type. + accessor->setPathDisplay("«" + to_string() + "»"); - accessor->setPathDisplay("«" + to_string() + "»"); + return {accessor, *this}; + }; - return {accessor, *this}; - } catch (Error & e) { - debug("substitution of input '%s' failed: %s", to_string(), e.what()); - } + /* If a tree with the expected hash is already in the Nix store, + reuse it. We only do this for final inputs, since otherwise + there is a risk that we don't return the same attributes (like + `lastModified`) that the "real" fetcher would return. */ + if (storePath && store->isValidPath(*storePath)) { + debug("using input '%s' in '%s'", to_string(), store->printStorePath(*storePath)); + return makeStoreAccessor(); } - auto [accessor, result] = scheme->getAccessor(store, *this); + try { + auto [accessor, result] = scheme->getAccessor(store, *this); - assert(!accessor->fingerprint); - accessor->fingerprint = result.getFingerprint(store); + if (!accessor->fingerprint) + accessor->fingerprint = result.getFingerprint(store); + else + result.cachedFingerprint = accessor->fingerprint; - return {accessor, std::move(result)}; + return {accessor, std::move(result)}; + } catch (Error & e) { + if (storePath) { + // Fall back to substitution. + try { + store->ensurePath(*storePath); + warn( + "Successfully substituted input '%s' after failing to fetch it from its original location: %s", + to_string(), + e.info().msg); + return makeStoreAccessor(); + } + // Ignore any substitution error, rethrow the original error. + catch (Error & e2) { + debug("substitution of input '%s' failed: %s", to_string(), e2.info().msg); + } catch (...) { + } + } + throw; + } } Input Input::applyOverrides(std::optional ref, std::optional rev) const @@ -359,10 +399,10 @@ Input Input::applyOverrides(std::optional ref, std::optional return scheme->applyOverrides(*this, ref, rev); } -void Input::clone(const Path & destDir) const +void Input::clone(ref store, const std::filesystem::path & destDir) const { assert(scheme); - scheme->clone(*this, destDir); + scheme->clone(store, *this, destDir); } std::optional Input::getSourcePath() const @@ -475,9 +515,18 @@ void InputScheme::putFile( throw Error("input '%s' does not support modifying file '%s'", input.to_string(), path); } -void InputScheme::clone(const Input & input, const Path & destDir) const +void InputScheme::clone(ref store, const Input & input, const std::filesystem::path & destDir) const { - throw Error("do not know how to clone input '%s'", input.to_string()); + if (std::filesystem::exists(destDir)) + throw Error("cannot clone into existing path %s", destDir); + + auto [accessor, input2] = getAccessor(store, input); + + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying '%s' to %s...", input2.to_string(), destDir)); + + auto source = sinkToSource([&](Sink & sink) { accessor->dumpPath(CanonPath::root, sink); }); + + restorePath(destDir, *source); } std::optional InputScheme::experimentalFeature() const @@ -509,7 +558,7 @@ fetchers::PublicKey adl_serializer::from_json(const json & return res; } -void adl_serializer::to_json(json & json, fetchers::PublicKey p) +void adl_serializer::to_json(json & json, const fetchers::PublicKey & p) { json["type"] = p.type; json["key"] = p.key; diff --git a/src/libfetchers/filtering-source-accessor.cc b/src/libfetchers/filtering-source-accessor.cc index 17f224ad299..f883c092190 100644 --- a/src/libfetchers/filtering-source-accessor.cc +++ b/src/libfetchers/filtering-source-accessor.cc @@ -1,4 +1,7 @@ #include "nix/fetchers/filtering-source-accessor.hh" +#include "nix/util/sync.hh" + +#include namespace nix { @@ -14,15 +17,26 @@ std::string FilteringSourceAccessor::readFile(const CanonPath & path) return next->readFile(prefix / path); } +void FilteringSourceAccessor::readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) +{ + checkAccess(path); + return next->readFile(prefix / path, sink, sizeCallback); +} + bool FilteringSourceAccessor::pathExists(const CanonPath & path) { return isAllowed(path) && next->pathExists(prefix / path); } std::optional FilteringSourceAccessor::maybeLstat(const CanonPath & path) +{ + return isAllowed(path) ? next->maybeLstat(prefix / path) : std::nullopt; +} + +SourceAccessor::Stat FilteringSourceAccessor::lstat(const CanonPath & path) { checkAccess(path); - return next->maybeLstat(prefix / path); + return next->lstat(prefix / path); } SourceAccessor::DirEntries FilteringSourceAccessor::readDirectory(const CanonPath & path) @@ -47,6 +61,13 @@ std::string FilteringSourceAccessor::showPath(const CanonPath & path) return displayPrefix + next->showPath(prefix / path) + displaySuffix; } +std::pair> FilteringSourceAccessor::getFingerprint(const CanonPath & path) +{ + if (fingerprint) + return {path, fingerprint}; + return next->getFingerprint(prefix / path); +} + void FilteringSourceAccessor::checkAccess(const CanonPath & path) { if (!isAllowed(path)) @@ -56,13 +77,13 @@ void FilteringSourceAccessor::checkAccess(const CanonPath & path) struct AllowListSourceAccessorImpl : AllowListSourceAccessor { - std::set allowedPrefixes; - std::unordered_set allowedPaths; + SharedSync> allowedPrefixes; + SharedSync> allowedPaths; AllowListSourceAccessorImpl( ref next, std::set && allowedPrefixes, - std::unordered_set && allowedPaths, + boost::unordered_flat_set && allowedPaths, MakeNotAllowedError && makeNotAllowedError) : AllowListSourceAccessor(SourcePath(next), std::move(makeNotAllowedError)) , allowedPrefixes(std::move(allowedPrefixes)) @@ -72,19 +93,19 @@ struct AllowListSourceAccessorImpl : AllowListSourceAccessor bool isAllowed(const CanonPath & path) override { - return allowedPaths.contains(path) || path.isAllowed(allowedPrefixes); + return allowedPaths.readLock()->contains(path) || path.isAllowed(*allowedPrefixes.readLock()); } void allowPrefix(CanonPath prefix) override { - allowedPrefixes.insert(std::move(prefix)); + allowedPrefixes.lock()->insert(std::move(prefix)); } }; ref AllowListSourceAccessor::create( ref next, std::set && allowedPrefixes, - std::unordered_set && allowedPaths, + boost::unordered_flat_set && allowedPaths, MakeNotAllowedError && makeNotAllowedError) { return make_ref( diff --git a/src/libfetchers/git-lfs-fetch.cc b/src/libfetchers/git-lfs-fetch.cc index a68cdf83230..9688daa4a71 100644 --- a/src/libfetchers/git-lfs-fetch.cc +++ b/src/libfetchers/git-lfs-fetch.cc @@ -25,7 +25,7 @@ static void downloadToSink( std::string sha256Expected, size_t sizeExpected) { - FileTransferRequest request(url); + FileTransferRequest request(parseURL(url)); Headers headers; if (authHeader.has_value()) headers.push_back({"Authorization", *authHeader}); @@ -69,7 +69,8 @@ static LfsApiInfo getLfsApi(const ParsedURL & url) args.push_back("--"); args.push_back("git-lfs-authenticate"); - args.push_back(url.path); + // FIXME %2F encode slashes? Does this command take/accept percent encoding? + args.push_back(url.renderPath(/*encode=*/false)); args.push_back("download"); auto [status, output] = runProgram({.program = "ssh", .args = args}); @@ -179,7 +180,7 @@ Fetch::Fetch(git_repository * repo, git_oid rev) const auto remoteUrl = lfs::getLfsEndpointUrl(repo); - this->url = nix::parseURL(nix::fixGitURL(remoteUrl)).canonicalise(); + this->url = nix::fixGitURL(remoteUrl).canonicalise(); } bool Fetch::shouldFetch(const CanonPath & path) const @@ -207,7 +208,7 @@ std::vector Fetch::fetchUrls(const std::vector & pointe auto api = lfs::getLfsApi(this->url); auto url = api.endpoint + "/objects/batch"; const auto & authHeader = api.authHeader; - FileTransferRequest request(url); + FileTransferRequest request(parseURL(url)); request.post = true; Headers headers; if (authHeader.has_value()) diff --git a/src/libfetchers/git-utils.cc b/src/libfetchers/git-utils.cc index b8d9b03cedc..603b8958741 100644 --- a/src/libfetchers/git-utils.cc +++ b/src/libfetchers/git-utils.cc @@ -9,6 +9,9 @@ #include "nix/util/users.hh" #include "nix/util/fs-sink.hh" #include "nix/util/sync.hh" +#include "nix/util/thread-pool.hh" +#include "nix/util/pool.hh" +#include "nix/util/executable-path.hh" #include #include @@ -30,8 +33,9 @@ #include #include +#include +#include #include -#include #include #include #include @@ -223,20 +227,28 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this { /** Location of the repository on disk. */ std::filesystem::path path; + + bool bare; + /** * libgit2 repository. Note that new objects are not written to disk, * because we are using a mempack backend. For writing to disk, see * `flush()`, which is also called by `GitFileSystemObjectSink::sync()`. */ Repository repo; + /** * In-memory object store for efficient batched writing to packfiles. * Owned by `repo`. */ git_odb_backend * mempack_backend; - GitRepoImpl(std::filesystem::path _path, bool create, bool bare) + bool useMempack; + + GitRepoImpl(std::filesystem::path _path, bool create, bool bare, bool useMempack = false) : path(std::move(_path)) + , bare(bare) + , useMempack(useMempack) { initLibGit2(); @@ -244,16 +256,18 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this if (git_repository_open(Setter(repo), path.string().c_str())) throw Error("opening Git repository %s: %s", path, git_error_last()->message); - ObjectDb odb; - if (git_repository_odb(Setter(odb), repo.get())) - throw Error("getting Git object database: %s", git_error_last()->message); + if (useMempack) { + ObjectDb odb; + if (git_repository_odb(Setter(odb), repo.get())) + throw Error("getting Git object database: %s", git_error_last()->message); - // mempack_backend will be owned by the repository, so we are not expected to free it ourselves. - if (git_mempack_new(&mempack_backend)) - throw Error("creating mempack backend: %s", git_error_last()->message); + // mempack_backend will be owned by the repository, so we are not expected to free it ourselves. + if (git_mempack_new(&mempack_backend)) + throw Error("creating mempack backend: %s", git_error_last()->message); - if (git_odb_add_backend(odb.get(), mempack_backend, 999)) - throw Error("adding mempack backend to Git object database: %s", git_error_last()->message); + if (git_odb_add_backend(odb.get(), mempack_backend, 999)) + throw Error("adding mempack backend to Git object database: %s", git_error_last()->message); + } } operator git_repository *() @@ -263,6 +277,9 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this void flush() override { + if (!useMempack) + return; + checkInterrupt(); git_buf buf = GIT_BUF_INIT; @@ -315,7 +332,7 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this uint64_t getRevCount(const Hash & rev) override { - std::unordered_set done; + boost::unordered_flat_set> done; std::queue todo; todo.push(peelObject(lookupObject(*this, hashToOID(rev)).get(), GIT_OBJECT_COMMIT)); @@ -544,47 +561,80 @@ struct GitRepoImpl : GitRepo, std::enable_shared_from_this // that) // then use code that was removed in this commit (see blame) - auto dir = this->path; - Strings gitArgs{"-C", dir.string(), "--git-dir", ".", "fetch", "--quiet", "--force"}; - if (shallow) - append(gitArgs, {"--depth", "1"}); - append(gitArgs, {std::string("--"), url, refspec}); - - auto [status, output] = runProgram( - RunOptions{ - .program = "git", - .lookupPath = true, - // FIXME: git stderr messes up our progress indicator, so - // we're using --quiet for now. Should process its stderr. - .args = gitArgs, - .input = {}, - .mergeStderrToStdout = true, - .isInteractive = true}); - - if (status > 0) { - throw Error("Failed to fetch git repository %s : %s", url, output); + if (ExecutablePath::load().findName("git")) { + auto dir = this->path; + Strings gitArgs{"-C", dir.string(), "--git-dir", ".", "fetch", "--quiet", "--force"}; + if (shallow) + append(gitArgs, {"--depth", "1"}); + append(gitArgs, {std::string("--"), url, refspec}); + + auto [status, output] = runProgram( + RunOptions{ + .program = "git", + .lookupPath = true, + // FIXME: git stderr messes up our progress indicator, so + // we're using --quiet for now. Should process its stderr. + .args = gitArgs, + .input = {}, + .mergeStderrToStdout = true, + .isInteractive = true}); + + if (status > 0) + throw Error("Failed to fetch git repository %s : %s", url, output); + } else { + // Fall back to using libgit2 for fetching. This does not + // support SSH very well. + Remote remote; + + if (git_remote_create_anonymous(Setter(remote), *this, url.c_str())) + throw Error("cannot create Git remote '%s': %s", url, git_error_last()->message); + + char * refspecs[] = {(char *) refspec.c_str()}; + git_strarray refspecs2{.strings = refspecs, .count = 1}; + + git_fetch_options opts = GIT_FETCH_OPTIONS_INIT; + // FIXME: for some reason, shallow fetching over ssh barfs + // with "could not read from remote repository". + opts.depth = shallow && parseURL(url).scheme != "ssh" ? 1 : GIT_FETCH_DEPTH_FULL; + opts.callbacks.payload = &act; + opts.callbacks.sideband_progress = sidebandProgressCallback; + opts.callbacks.transfer_progress = transferProgressCallback; + + if (git_remote_fetch(remote.get(), &refspecs2, &opts, nullptr)) + throw Error("fetching '%s' from '%s': %s", refspec, url, git_error_last()->message); } } void verifyCommit(const Hash & rev, const std::vector & publicKeys) override { + // Map of SSH key types to their internal OpenSSH representations + static const boost::unordered_flat_map keyTypeMap = { + {"ssh-dsa", "ssh-dsa"}, + {"ssh-ecdsa", "ssh-ecdsa"}, + {"ssh-ecdsa-sk", "sk-ecdsa-sha2-nistp256@openssh.com"}, + {"ssh-ed25519", "ssh-ed25519"}, + {"ssh-ed25519-sk", "sk-ssh-ed25519@openssh.com"}, + {"ssh-rsa", "ssh-rsa"}}; + // Create ad-hoc allowedSignersFile and populate it with publicKeys auto allowedSignersFile = createTempFile().second; std::string allowedSigners; + for (const fetchers::PublicKey & k : publicKeys) { - if (k.type != "ssh-dsa" && k.type != "ssh-ecdsa" && k.type != "ssh-ecdsa-sk" && k.type != "ssh-ed25519" - && k.type != "ssh-ed25519-sk" && k.type != "ssh-rsa") + auto it = keyTypeMap.find(k.type); + if (it == keyTypeMap.end()) { + std::string supportedTypes; + for (const auto & [type, _] : keyTypeMap) { + supportedTypes += fmt(" %s\n", type); + } throw Error( - "Unknown key type '%s'.\n" - "Please use one of\n" - "- ssh-dsa\n" - " ssh-ecdsa\n" - " ssh-ecdsa-sk\n" - " ssh-ed25519\n" - " ssh-ed25519-sk\n" - " ssh-rsa", - k.type); - allowedSigners += "* " + k.type + " " + k.key + "\n"; + "Invalid SSH key type '%s' in publicKeys.\n" + "Please use one of:\n%s", + k.type, + supportedTypes); + } + + allowedSigners += fmt("* %s %s\n", it->second, k.key); } writeFile(allowedSignersFile, allowedSigners); @@ -805,7 +855,7 @@ struct GitSourceAccessor : SourceAccessor return toHash(*git_tree_entry_id(entry)); } - std::unordered_map lookupCache; + boost::unordered_flat_map lookupCache; /* Recursively look up 'path' relative to the root. */ git_tree_entry * lookup(State & state, const CanonPath & path) @@ -1003,216 +1053,239 @@ struct GitFileSystemObjectSinkImpl : GitFileSystemObjectSink { ref repo; - struct PendingDir - { - std::string name; - TreeBuilder builder; - }; + bool useMempack = +// On macOS, mempack is beneficial. +#ifdef __linux__ + false +#else + true +#endif + ; - std::vector pendingDirs; + Pool repoPool; - void pushBuilder(std::string name) - { - const git_tree_entry * entry; - Tree prevTree = nullptr; - - if (!pendingDirs.empty() && (entry = git_treebuilder_get(pendingDirs.back().builder.get(), name.c_str()))) { - /* Clone a tree that we've already finished. This happens - if a tarball has directory entries that are not - contiguous. */ - if (git_tree_entry_type(entry) != GIT_OBJECT_TREE) - throw Error("parent of '%s' is not a directory", name); - - if (git_tree_entry_to_object((git_object **) (git_tree **) Setter(prevTree), *repo, entry)) - throw Error("looking up parent of '%s': %s", name, git_error_last()->message); - } + unsigned int concurrency = std::min(std::thread::hardware_concurrency(), 4U); - git_treebuilder * b; - if (git_treebuilder_new(&b, *repo, prevTree.get())) - throw Error("creating a tree builder: %s", git_error_last()->message); - pendingDirs.push_back({.name = std::move(name), .builder = TreeBuilder(b)}); - }; + ThreadPool workers{concurrency}; GitFileSystemObjectSinkImpl(ref repo) : repo(repo) + , repoPool(std::numeric_limits::max(), [repo, useMempack(useMempack)]() -> ref { + return make_ref(repo->path, false, repo->bare, useMempack); + }) { - pushBuilder(""); } - std::pair popBuilder() + struct Child; + + struct Directory { - assert(!pendingDirs.empty()); - auto pending = std::move(pendingDirs.back()); - git_oid oid; - if (git_treebuilder_write(&oid, pending.builder.get())) - throw Error("creating a tree object: %s", git_error_last()->message); - pendingDirs.pop_back(); - return {oid, pending.name}; + std::map children; + std::optional oid; + + Child & lookup(const CanonPath & path) + { + assert(!path.isRoot()); + auto parent = path.parent(); + auto cur = this; + for (auto & name : *parent) { + auto i = cur->children.find(std::string(name)); + if (i == cur->children.end()) + throw Error("path '%s' does not exist", path); + auto dir = std::get_if(&i->second.file); + if (!dir) + throw Error("path '%s' has a non-directory parent", path); + cur = dir; + } + + auto i = cur->children.find(std::string(*path.baseName())); + if (i == cur->children.end()) + throw Error("path '%s' does not exist", path); + return i->second; + } }; - void addToTree(const std::string & name, const git_oid & oid, git_filemode_t mode) + struct Child { - assert(!pendingDirs.empty()); - auto & pending = pendingDirs.back(); - if (git_treebuilder_insert(nullptr, pending.builder.get(), name.c_str(), &oid, mode)) - throw Error("adding a file to a tree builder: %s", git_error_last()->message); + git_filemode_t mode; + std::variant file; + + /// Sequential numbering of the file in the tarball. This is + /// used to make sure we only import the latest version of a + /// path. + size_t id{0}; }; - void updateBuilders(std::span names) + struct State { - // Find the common prefix of pendingDirs and names. - size_t prefixLen = 0; - for (; prefixLen < names.size() && prefixLen + 1 < pendingDirs.size(); ++prefixLen) - if (names[prefixLen] != pendingDirs[prefixLen + 1].name) - break; - - // Finish the builders that are not part of the common prefix. - for (auto n = pendingDirs.size(); n > prefixLen + 1; --n) { - auto [oid, name] = popBuilder(); - addToTree(name, oid, GIT_FILEMODE_TREE); - } - - // Create builders for the new directories. - for (auto n = prefixLen; n < names.size(); ++n) - pushBuilder(names[n]); + Directory root; }; - bool prepareDirs(const std::vector & pathComponents, bool isDir) + Sync _state; + + void addNode(State & state, const CanonPath & path, Child && child) { - std::span pathComponents2{pathComponents}; + assert(!path.isRoot()); + auto parent = path.parent(); - updateBuilders(isDir ? pathComponents2 : pathComponents2.first(pathComponents2.size() - 1)); + Directory * cur = &state.root; - return true; + for (auto & i : *parent) { + auto child = std::get_if( + &cur->children.emplace(std::string(i), Child{GIT_FILEMODE_TREE, {Directory()}}).first->second.file); + assert(child); + cur = child; + } + + std::string name(*path.baseName()); + + if (auto prev = cur->children.find(name); prev == cur->children.end() || prev->second.id < child.id) + cur->children.insert_or_assign(name, std::move(child)); } + size_t nextId = 0; + void createRegularFile(const CanonPath & path, std::function func) override { - auto pathComponents = tokenizeString>(path.rel(), "/"); - if (!prepareDirs(pathComponents, false)) - return; - - git_writestream * stream = nullptr; - if (git_blob_create_from_stream(&stream, *repo, nullptr)) - throw Error("creating a blob stream object: %s", git_error_last()->message); - struct CRF : CreateRegularFileSink { - const CanonPath & path; - GitFileSystemObjectSinkImpl & back; - git_writestream * stream; + std::string data; bool executable = false; - CRF(const CanonPath & path, GitFileSystemObjectSinkImpl & back, git_writestream * stream) - : path(path) - , back(back) - , stream(stream) - { - } - void operator()(std::string_view data) override { - if (stream->write(stream, data.data(), data.size())) - throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message); + this->data += data; } void isExecutable() override { executable = true; } - } crf{path, *this, stream}; + } crf; func(crf); - git_oid oid; - if (git_blob_create_from_stream_commit(&oid, stream)) - throw Error("creating a blob object for tarball member '%s': %s", path, git_error_last()->message); + workers.enqueue([this, path, data{std::move(crf.data)}, executable(crf.executable), id(nextId++)]() { + auto repo(repoPool.get()); - addToTree(*pathComponents.rbegin(), oid, crf.executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB); + git_writestream * stream = nullptr; + if (git_blob_create_from_stream(&stream, *repo, nullptr)) + throw Error("creating a blob stream object: %s", git_error_last()->message); + + if (stream->write(stream, data.data(), data.size())) + throw Error("writing a blob for tarball member '%s': %s", path, git_error_last()->message); + + git_oid oid; + if (git_blob_create_from_stream_commit(&oid, stream)) + throw Error("creating a blob object for tarball member '%s': %s", path, git_error_last()->message); + + auto state(_state.lock()); + addNode(*state, path, Child{executable ? GIT_FILEMODE_BLOB_EXECUTABLE : GIT_FILEMODE_BLOB, oid, id}); + }); } void createDirectory(const CanonPath & path) override { - auto pathComponents = tokenizeString>(path.rel(), "/"); - (void) prepareDirs(pathComponents, true); + if (path.isRoot()) + return; + auto state(_state.lock()); + addNode(*state, path, {GIT_FILEMODE_TREE, Directory()}); } void createSymlink(const CanonPath & path, const std::string & target) override { - auto pathComponents = tokenizeString>(path.rel(), "/"); - if (!prepareDirs(pathComponents, false)) - return; + workers.enqueue([this, path, target]() { + auto repo(repoPool.get()); - git_oid oid; - if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size())) - throw Error("creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message); + git_oid oid; + if (git_blob_create_from_buffer(&oid, *repo, target.c_str(), target.size())) + throw Error( + "creating a blob object for tarball symlink member '%s': %s", path, git_error_last()->message); - addToTree(*pathComponents.rbegin(), oid, GIT_FILEMODE_LINK); + auto state(_state.lock()); + addNode(*state, path, Child{GIT_FILEMODE_LINK, oid}); + }); } + std::map hardLinks; + void createHardlink(const CanonPath & path, const CanonPath & target) override { - std::vector pathComponents; - for (auto & c : path) - pathComponents.emplace_back(c); + hardLinks.insert_or_assign(path, target); + } - if (!prepareDirs(pathComponents, false)) - return; + Hash flush() override + { + workers.process(); - // We can't just look up the path from the start of the root, since - // some parent directories may not have finished yet, so we compute - // a relative path that helps us find the right git_tree_builder or object. - auto relTarget = CanonPath(path).parent()->makeRelative(target); - - auto dir = pendingDirs.rbegin(); - - // For each ../ component at the start, go up one directory. - // CanonPath::makeRelative() always puts all .. elements at the start, - // so they're all handled by this loop: - std::string_view relTargetLeft(relTarget); - while (hasPrefix(relTargetLeft, "../")) { - if (dir == pendingDirs.rend()) - throw Error("invalid hard link target '%s' for path '%s'", target, path); - ++dir; - relTargetLeft = relTargetLeft.substr(3); - } - if (dir == pendingDirs.rend()) - throw Error("invalid hard link target '%s' for path '%s'", target, path); - - // Look up the remainder of the target, starting at the - // top-most `git_treebuilder`. - std::variant curDir{dir->builder.get()}; - Object tree; // needed to keep `entry` alive - const git_tree_entry * entry = nullptr; - - for (auto & c : CanonPath(relTargetLeft)) { - if (auto builder = std::get_if(&curDir)) { - assert(*builder); - if (!(entry = git_treebuilder_get(*builder, std::string(c).c_str()))) - throw Error("cannot find hard link target '%s' for path '%s'", target, path); - curDir = *git_tree_entry_id(entry); - } else if (auto oid = std::get_if(&curDir)) { - tree = lookupObject(*repo, *oid, GIT_OBJECT_TREE); - if (!(entry = git_tree_entry_byname((const git_tree *) &*tree, std::string(c).c_str()))) - throw Error("cannot find hard link target '%s' for path '%s'", target, path); - curDir = *git_tree_entry_id(entry); + /* Create hard links. */ + { + auto state(_state.lock()); + for (auto & [path, target] : hardLinks) { + if (target.isRoot()) + continue; + try { + auto child = state->root.lookup(target); + auto oid = std::get_if(&child.file); + if (!oid) + throw Error("cannot create a hard link to a directory"); + addNode(*state, path, {child.mode, *oid}); + } catch (Error & e) { + e.addTrace(nullptr, "while creating a hard link from '%s' to '%s'", path, target); + throw; + } } } - assert(entry); + auto & root = _state.lock()->root; - addToTree(*pathComponents.rbegin(), *git_tree_entry_id(entry), git_tree_entry_filemode(entry)); - } + auto doFlush = [&]() { + auto repos = repoPool.clear(); + ThreadPool workers{repos.size()}; + for (auto & repo : repos) + workers.enqueue([repo]() { repo->flush(); }); + workers.process(); + }; - Hash flush() override - { - updateBuilders({}); + if (useMempack) + doFlush(); + + processGraph( + {&root}, + [&](Directory * const & node) -> std::set { + std::set edges; + for (auto & child : node->children) + if (auto dir = std::get_if(&child.second.file)) + edges.insert(dir); + return edges; + }, + [&](Directory * const & node) { + auto repo(repoPool.get()); + + git_treebuilder * b; + if (git_treebuilder_new(&b, *repo, nullptr)) + throw Error("creating a tree builder: %s", git_error_last()->message); + TreeBuilder builder(b); + + for (auto & [name, child] : node->children) { + auto oid_p = std::get_if(&child.file); + auto oid = oid_p ? *oid_p : std::get(child.file).oid.value(); + if (git_treebuilder_insert(nullptr, builder.get(), name.c_str(), &oid, child.mode)) + throw Error("adding a file to a tree builder: %s", git_error_last()->message); + } - auto [oid, _name] = popBuilder(); + git_oid oid; + if (git_treebuilder_write(&oid, builder.get())) + throw Error("creating a tree object: %s", git_error_last()->message); + node->oid = oid; + }, + true, + useMempack ? 1 : concurrency); - repo->flush(); + if (useMempack) + doFlush(); - return toHash(oid); + return toHash(root.oid.value()); } }; @@ -1242,13 +1315,12 @@ GitRepoImpl::getAccessor(const WorkdirInfo & wd, bool exportIgnore, MakeNotAllow makeFSSourceAccessor(path), std::set{wd.files}, // Always allow access to the root, but not its children. - std::unordered_set{CanonPath::root}, + boost::unordered_flat_set{CanonPath::root}, std::move(makeNotAllowedError)) .cast(); if (exportIgnore) - return make_ref(self, fileAccessor, std::nullopt); - else - return fileAccessor; + fileAccessor = make_ref(self, fileAccessor, std::nullopt); + return fileAccessor; } ref GitRepoImpl::getFileSystemObjectSink() @@ -1290,13 +1362,18 @@ std::vector> GitRepoImpl::getSubmodules return result; } -ref getTarballCache() -{ - static auto repoDir = std::filesystem::path(getCacheDir()) / "tarball-cache"; +namespace fetchers { - return GitRepo::openRepo(repoDir, true, true); +ref Settings::getTarballCache() const +{ + auto tarballCache(_tarballCache.lock()); + if (!*tarballCache) + *tarballCache = GitRepo::openRepo(std::filesystem::path(getCacheDir()) / "tarball-cache", true, true); + return ref(*tarballCache); } +} // namespace fetchers + GitRepo::WorkdirInfo GitRepo::getCachedWorkdirInfo(const std::filesystem::path & path) { static Sync> _cache; diff --git a/src/libfetchers/git.cc b/src/libfetchers/git.cc index bd1e1fffe99..727792db0a2 100644 --- a/src/libfetchers/git.cc +++ b/src/libfetchers/git.cc @@ -15,6 +15,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/util/json-utils.hh" #include "nix/util/archive.hh" +#include "nix/util/mounted-source-accessor.hh" #include #include @@ -163,8 +164,8 @@ struct GitInputScheme : InputScheme { std::optional inputFromURL(const Settings & settings, const ParsedURL & url, bool requireTree) const override { - if (url.scheme != "git" && url.scheme != "git+http" && url.scheme != "git+https" && url.scheme != "git+ssh" - && url.scheme != "git+file") + auto parsedScheme = parseUrlScheme(url.scheme); + if (parsedScheme.application != "git") return {}; auto url2(url); @@ -233,9 +234,7 @@ struct GitInputScheme : InputScheme Input input{settings}; input.attrs = attrs; - auto url = fixGitURL(getStrAttr(attrs, "url")); - parseURL(url); - input.attrs["url"] = url; + input.attrs["url"] = fixGitURL(getStrAttr(attrs, "url")).to_string(); getShallowAttr(input); getSubmodulesAttr(input); getAllRefsAttr(input); @@ -282,7 +281,7 @@ struct GitInputScheme : InputScheme return res; } - void clone(const Input & input, const Path & destDir) const override + void clone(ref store, const Input & input, const std::filesystem::path & destDir) const override { auto repoInfo = getRepoInfo(input); @@ -406,10 +405,10 @@ struct GitInputScheme : InputScheme { if (workdirInfo.isDirty) { if (!settings.allowDirty) - throw Error("Git tree '%s' is dirty", locationToArg()); + throw Error("Git tree '%s' has uncommitted changes", locationToArg()); if (settings.warnDirty) - warn("Git tree '%s' is dirty", locationToArg()); + warn("Git tree '%s' has uncommitted changes", locationToArg()); } } @@ -464,8 +463,8 @@ struct GitInputScheme : InputScheme // Why are we checking for bare repository? // well if it's a bare repository we want to force a git fetch rather than copying the folder - bool isBareRepository = url.scheme == "file" && pathExists(url.path) && !pathExists(url.path + "/.git"); - // + auto isBareRepository = [](PathView path) { return pathExists(path) && !pathExists(path + "/.git"); }; + // FIXME: here we turn a possibly relative path into an absolute path. // This allows relative git flake inputs to be resolved against the // **current working directory** (as in POSIX), which tends to work out @@ -474,8 +473,10 @@ struct GitInputScheme : InputScheme // // See: https://discourse.nixos.org/t/57783 and #9708 // - if (url.scheme == "file" && !forceHttp && !isBareRepository) { - if (!isAbsolute(url.path)) { + if (url.scheme == "file" && !forceHttp && !isBareRepository(renderUrlPathEnsureLegal(url.path))) { + auto path = renderUrlPathEnsureLegal(url.path); + + if (!isAbsolute(path)) { warn( "Fetching Git repository '%s', which uses a path relative to the current directory. " "This is not supported and will stop working in a future release. " @@ -485,10 +486,10 @@ struct GitInputScheme : InputScheme // If we don't check here for the path existence, then we can give libgit2 any directory // and it will initialize them as git directories. - if (!pathExists(url.path)) { - throw Error("The path '%s' does not exist.", url.path); + if (!pathExists(path)) { + throw Error("The path '%s' does not exist.", path); } - repoInfo.location = std::filesystem::absolute(url.path); + repoInfo.location = std::filesystem::absolute(path); } else { if (url.scheme == "file") /* Query parameters are meaningless for file://, but @@ -892,8 +893,7 @@ struct GitInputScheme : InputScheme return makeFingerprint(*rev); else { auto repoInfo = getRepoInfo(input); - if (auto repoPath = repoInfo.getPath(); - repoPath && repoInfo.workdirInfo.headRev && repoInfo.workdirInfo.submodules.empty()) { + if (auto repoPath = repoInfo.getPath(); repoPath && repoInfo.workdirInfo.submodules.empty()) { /* Calculate a fingerprint that takes into account the deleted and modified/added files. */ HashSink hashSink{HashAlgorithm::SHA512}; @@ -906,7 +906,7 @@ struct GitInputScheme : InputScheme writeString("deleted:", hashSink); writeString(file.abs(), hashSink); } - return makeFingerprint(*repoInfo.workdirInfo.headRev) + return makeFingerprint(repoInfo.workdirInfo.headRev.value_or(nullRev)) + ";d=" + hashSink.finish().hash.to_string(HashFormat::Base16, false); } return std::nullopt; diff --git a/src/libfetchers/github.cc b/src/libfetchers/github.cc index 841a9c2df1e..2f4eb9dac68 100644 --- a/src/libfetchers/github.cc +++ b/src/libfetchers/github.cc @@ -19,7 +19,7 @@ namespace nix::fetchers { struct DownloadUrl { - std::string url; + ParsedURL url; Headers headers; }; @@ -38,7 +38,8 @@ struct GitArchiveInputScheme : InputScheme if (url.scheme != schemeName()) return {}; - auto path = tokenizeString>(url.path, "/"); + /* This ignores empty path segments for back-compat. Older versions used a tokenizeString here. */ + auto path = url.pathSegments(/*skipEmpty=*/true) | std::ranges::to>(); std::optional rev; std::optional ref; @@ -139,12 +140,12 @@ struct GitArchiveInputScheme : InputScheme auto repo = getStrAttr(input.attrs, "repo"); auto ref = input.getRef(); auto rev = input.getRev(); - auto path = owner + "/" + repo; + std::vector path{owner, repo}; assert(!(ref && rev)); if (ref) - path += "/" + *ref; + path.push_back(*ref); if (rev) - path += "/" + rev->to_string(HashFormat::Base16, false); + path.push_back(rev->to_string(HashFormat::Base16, false)); auto url = ParsedURL{ .scheme = std::string{schemeName()}, .path = path, @@ -269,7 +270,7 @@ struct GitArchiveInputScheme : InputScheme if (auto lastModifiedAttrs = cache->lookup(lastModifiedKey)) { auto treeHash = getRevAttr(*treeHashAttrs, "treeHash"); auto lastModified = getIntAttr(*lastModifiedAttrs, "lastModified"); - if (getTarballCache()->hasObject(treeHash)) + if (input.settings->getTarballCache()->hasObject(treeHash)) return {std::move(input), TarballInfo{.treeHash = treeHash, .lastModified = (time_t) lastModified}}; else debug("Git tree with hash '%s' has disappeared from the cache, refetching...", treeHash.gitRev()); @@ -289,7 +290,7 @@ struct GitArchiveInputScheme : InputScheme *logger, lvlInfo, actUnknown, fmt("unpacking '%s' into the Git cache", input.to_string())); TarArchive archive{*source}; - auto tarballCache = getTarballCache(); + auto tarballCache = input.settings->getTarballCache(); auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); auto tree = parseSink->flush(); @@ -323,7 +324,15 @@ struct GitArchiveInputScheme : InputScheme #endif input.attrs.insert_or_assign("lastModified", uint64_t(tarballInfo.lastModified)); - auto accessor = getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»"); + auto accessor = + input.settings->getTarballCache()->getAccessor(tarballInfo.treeHash, false, "«" + input.to_string() + "»"); + + if (!input.settings->trustTarballsFromGitForges) + // FIXME: computing the NAR hash here is wasteful if + // copyInputToStore() is just going to hash/copy it as + // well. + input.attrs.insert_or_assign( + "narHash", accessor->hashPath(CanonPath::root).to_string(HashFormat::SRI, true)); return {accessor, input}; } @@ -338,11 +347,6 @@ struct GitArchiveInputScheme : InputScheme && (input.settings->trustTarballsFromGitForges || input.getNarHash().has_value()); } - std::optional experimentalFeature() const override - { - return Xp::Flakes; - } - std::optional getFingerprint(ref store, const Input & input) const override { if (auto rev = input.getRev()) @@ -397,8 +401,8 @@ struct GitHubInputScheme : GitArchiveInputScheme Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input); - auto json = nlohmann::json::parse( - readFile(store->toRealPath(downloadFile(store, *input.settings, url, "source", headers).storePath))); + auto downloadResult = downloadFile(store, *input.settings, url, "source", headers); + auto json = nlohmann::json::parse(store->getFSAccessor(downloadResult.storePath)->readFile(CanonPath::root)); return RefInfo{ .rev = Hash::parseAny(std::string{json["sha"]}, HashAlgorithm::SHA1), @@ -420,15 +424,15 @@ struct GitHubInputScheme : GitArchiveInputScheme const auto url = fmt(urlFmt, host, getOwner(input), getRepo(input), input.getRev()->to_string(HashFormat::Base16, false)); - return DownloadUrl{url, headers}; + return DownloadUrl{parseURL(url), headers}; } - void clone(const Input & input, const Path & destDir) const override + void clone(ref store, const Input & input, const std::filesystem::path & destDir) const override { auto host = getHost(input); Input::fromURL(*input.settings, fmt("git+https://%s/%s/%s.git", host, getOwner(input), getRepo(input))) .applyOverrides(input.getRef(), input.getRev()) - .clone(destDir); + .clone(store, destDir); } }; @@ -471,8 +475,8 @@ struct GitLabInputScheme : GitArchiveInputScheme Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input); - auto json = nlohmann::json::parse( - readFile(store->toRealPath(downloadFile(store, *input.settings, url, "source", headers).storePath))); + auto downloadResult = downloadFile(store, *input.settings, url, "source", headers); + auto json = nlohmann::json::parse(store->getFSAccessor(downloadResult.storePath)->readFile(CanonPath::root)); if (json.is_array() && json.size() >= 1 && json[0]["id"] != nullptr) { return RefInfo{.rev = Hash::parseAny(std::string(json[0]["id"]), HashAlgorithm::SHA1)}; @@ -500,10 +504,10 @@ struct GitLabInputScheme : GitArchiveInputScheme input.getRev()->to_string(HashFormat::Base16, false)); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input); - return DownloadUrl{url, headers}; + return DownloadUrl{parseURL(url), headers}; } - void clone(const Input & input, const Path & destDir) const override + void clone(ref store, const Input & input, const std::filesystem::path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("gitlab.com"); // FIXME: get username somewhere @@ -511,7 +515,7 @@ struct GitLabInputScheme : GitArchiveInputScheme *input.settings, fmt("git+https://%s/%s/%s.git", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) - .clone(destDir); + .clone(store, destDir); } }; @@ -592,17 +596,17 @@ struct SourceHutInputScheme : GitArchiveInputScheme input.getRev()->to_string(HashFormat::Base16, false)); Headers headers = makeHeadersWithAuthTokens(*input.settings, host, input); - return DownloadUrl{url, headers}; + return DownloadUrl{parseURL(url), headers}; } - void clone(const Input & input, const Path & destDir) const override + void clone(ref store, const Input & input, const std::filesystem::path & destDir) const override { auto host = maybeGetStrAttr(input.attrs, "host").value_or("git.sr.ht"); Input::fromURL( *input.settings, fmt("git+https://%s/%s/%s", host, getStrAttr(input.attrs, "owner"), getStrAttr(input.attrs, "repo"))) .applyOverrides(input.getRef(), input.getRev()) - .clone(destDir); + .clone(store, destDir); } }; diff --git a/src/libfetchers/include/nix/fetchers/cache.hh b/src/libfetchers/include/nix/fetchers/cache.hh index 7219635ec07..8cac076f1f2 100644 --- a/src/libfetchers/include/nix/fetchers/cache.hh +++ b/src/libfetchers/include/nix/fetchers/cache.hh @@ -67,9 +67,9 @@ struct Cache /** * Look up a store path in the cache. The returned store path will - * be valid, but it may be expired. + * be valid (unless `allowInvalid` is true), but it may be expired. */ - virtual std::optional lookupStorePath(Key key, Store & store) = 0; + virtual std::optional lookupStorePath(Key key, Store & store, bool allowInvalid = false) = 0; /** * Look up a store path in the cache. Return nothing if its TTL diff --git a/src/libfetchers/include/nix/fetchers/fetch-settings.hh b/src/libfetchers/include/nix/fetchers/fetch-settings.hh index 605b95e0d02..6c39bb2ea92 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-settings.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-settings.hh @@ -11,6 +11,12 @@ #include +namespace nix { + +struct GitRepo; + +} + namespace nix::fetchers { struct Cache; @@ -88,10 +94,7 @@ struct Settings : public Config are subsequently modified. Therefore lock files with dirty locks should generally only be used for local testing, and should not be pushed to other users. - )", - {}, - true, - Xp::Flakes}; + )"}; Setting trustTarballsFromGitForges{ this, @@ -118,15 +121,25 @@ struct Settings : public Config Path or URI of the global flake registry. When empty, disables the global flake registry. - )", - {}, - true, - Xp::Flakes}; + )"}; ref getCache() const; + ref getTarballCache() const; + private: mutable Sync> _cache; + + mutable Sync> _tarballCache; }; } // namespace nix::fetchers + +namespace nix { + +/** + * @todo Get rid of global setttings variables + */ +extern fetchers::Settings fetchSettings; + +} // namespace nix diff --git a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh index 3a223230235..e7f88072491 100644 --- a/src/libfetchers/include/nix/fetchers/fetch-to-store.hh +++ b/src/libfetchers/include/nix/fetchers/fetch-to-store.hh @@ -24,7 +24,17 @@ StorePath fetchToStore( PathFilter * filter = nullptr, RepairFlag repair = NoRepair); -fetchers::Cache::Key makeFetchToStoreCacheKey( - const std::string & name, const std::string & fingerprint, ContentAddressMethod method, const std::string & path); +std::pair fetchToStore2( + const fetchers::Settings & settings, + Store & store, + const SourcePath & path, + FetchMode mode, + std::string_view name = "source", + ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive, + PathFilter * filter = nullptr, + RepairFlag repair = NoRepair); + +fetchers::Cache::Key +makeSourcePathToHashCacheKey(const std::string & fingerprint, ContentAddressMethod method, const std::string & path); } // namespace nix diff --git a/src/libfetchers/include/nix/fetchers/fetchers.hh b/src/libfetchers/include/nix/fetchers/fetchers.hh index 9dcd365eae4..f2763afdc1d 100644 --- a/src/libfetchers/include/nix/fetchers/fetchers.hh +++ b/src/libfetchers/include/nix/fetchers/fetchers.hh @@ -120,7 +120,7 @@ public: * Fetch the entire input into the Nix store, returning the * location in the Nix store and the locked input. */ - std::pair fetchToStore(ref store) const; + std::tuple, Input> fetchToStore(ref store) const; /** * Check the locking attributes in `result` against @@ -150,7 +150,7 @@ public: Input applyOverrides(std::optional ref, std::optional rev) const; - void clone(const Path & destDir) const; + void clone(ref store, const std::filesystem::path & destDir) const; std::optional getSourcePath() const; @@ -223,7 +223,7 @@ struct InputScheme virtual Input applyOverrides(const Input & input, std::optional ref, std::optional rev) const; - virtual void clone(const Input & input, const Path & destDir) const; + virtual void clone(ref store, const Input & input, const std::filesystem::path & destDir) const; virtual std::optional getSourcePath(const Input & input) const; diff --git a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh index 70e837ff4db..5e98caa5816 100644 --- a/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh +++ b/src/libfetchers/include/nix/fetchers/filtering-source-accessor.hh @@ -2,7 +2,7 @@ #include "nix/util/source-path.hh" -#include +#include namespace nix { @@ -36,8 +36,12 @@ struct FilteringSourceAccessor : SourceAccessor std::string readFile(const CanonPath & path) override; + void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override; + bool pathExists(const CanonPath & path) override; + Stat lstat(const CanonPath & path) override; + std::optional maybeLstat(const CanonPath & path) override; DirEntries readDirectory(const CanonPath & path) override; @@ -46,6 +50,8 @@ struct FilteringSourceAccessor : SourceAccessor std::string showPath(const CanonPath & path) override; + std::pair> getFingerprint(const CanonPath & path) override; + /** * Call `makeNotAllowedError` to throw a `RestrictedPathError` * exception if `isAllowed()` returns `false` for `path`. @@ -72,7 +78,7 @@ struct AllowListSourceAccessor : public FilteringSourceAccessor static ref create( ref next, std::set && allowedPrefixes, - std::unordered_set && allowedPaths, + boost::unordered_flat_set && allowedPaths, MakeNotAllowedError && makeNotAllowedError); using FilteringSourceAccessor::FilteringSourceAccessor; diff --git a/src/libfetchers/include/nix/fetchers/git-utils.hh b/src/libfetchers/include/nix/fetchers/git-utils.hh index 07b9855417f..6656793f26d 100644 --- a/src/libfetchers/include/nix/fetchers/git-utils.hh +++ b/src/libfetchers/include/nix/fetchers/git-utils.hh @@ -120,8 +120,6 @@ struct GitRepo virtual Hash dereferenceSingletonDirectory(const Hash & oid) = 0; }; -ref getTarballCache(); - // A helper to ensure that the `git_*_free` functions get called. template struct Deleter diff --git a/src/libfetchers/include/nix/fetchers/input-cache.hh b/src/libfetchers/include/nix/fetchers/input-cache.hh index b2fc842458e..40241207150 100644 --- a/src/libfetchers/include/nix/fetchers/input-cache.hh +++ b/src/libfetchers/include/nix/fetchers/input-cache.hh @@ -11,6 +11,7 @@ struct InputCache ref accessor; Input resolvedInput; Input lockedInput; + Attrs extraAttrs; }; CachedResult getAccessor(ref store, const Input & originalInput, UseRegistries useRegistries); @@ -19,6 +20,7 @@ struct InputCache { Input lockedInput; ref accessor; + Attrs extraAttrs; }; virtual std::optional lookup(const Input & originalInput) const = 0; diff --git a/src/libfetchers/include/nix/fetchers/meson.build b/src/libfetchers/include/nix/fetchers/meson.build index fcd446a6d8b..a313b1e0bc0 100644 --- a/src/libfetchers/include/nix/fetchers/meson.build +++ b/src/libfetchers/include/nix/fetchers/meson.build @@ -11,6 +11,5 @@ headers = files( 'git-utils.hh', 'input-cache.hh', 'registry.hh', - 'store-path-accessor.hh', 'tarball.hh', ) diff --git a/src/libfetchers/include/nix/fetchers/store-path-accessor.hh b/src/libfetchers/include/nix/fetchers/store-path-accessor.hh deleted file mode 100644 index a107293f822..00000000000 --- a/src/libfetchers/include/nix/fetchers/store-path-accessor.hh +++ /dev/null @@ -1,14 +0,0 @@ -#pragma once - -#include "nix/util/source-path.hh" - -namespace nix { - -class StorePath; -class Store; - -ref makeStorePathAccessor(ref store, const StorePath & storePath); - -SourcePath getUnfilteredRootPath(CanonPath path); - -} // namespace nix diff --git a/src/libfetchers/indirect.cc b/src/libfetchers/indirect.cc index 4bd4d890df8..c8539298cc7 100644 --- a/src/libfetchers/indirect.cc +++ b/src/libfetchers/indirect.cc @@ -14,7 +14,8 @@ struct IndirectInputScheme : InputScheme if (url.scheme != "flake") return {}; - auto path = tokenizeString>(url.path, "/"); + /* This ignores empty path segments for back-compat. Older versions used a tokenizeString here. */ + auto path = url.pathSegments(/*skipEmpty=*/true) | std::ranges::to>(); std::optional rev; std::optional ref; @@ -82,16 +83,15 @@ struct IndirectInputScheme : InputScheme ParsedURL toURL(const Input & input) const override { - ParsedURL url; - url.scheme = "flake"; - url.path = getStrAttr(input.attrs, "id"); + ParsedURL url{ + .scheme = "flake", + .path = {getStrAttr(input.attrs, "id")}, + }; if (auto ref = input.getRef()) { - url.path += '/'; - url.path += *ref; + url.path.push_back(*ref); }; if (auto rev = input.getRev()) { - url.path += '/'; - url.path += rev->gitRev(); + url.path.push_back(rev->gitRev()); }; return url; } @@ -111,11 +111,6 @@ struct IndirectInputScheme : InputScheme throw Error("indirect input '%s' cannot be fetched directly", input.to_string()); } - std::optional experimentalFeature() const override - { - return Xp::Flakes; - } - bool isDirect(const Input & input) const override { return false; diff --git a/src/libfetchers/input-cache.cc b/src/libfetchers/input-cache.cc index 1422c1d9a20..c44f1a236b4 100644 --- a/src/libfetchers/input-cache.cc +++ b/src/libfetchers/input-cache.cc @@ -22,7 +22,8 @@ InputCache::getAccessor(ref store, const Input & originalInput, UseRegist fetched = lookup(resolvedInput); if (!fetched) { auto [accessor, lockedInput] = resolvedInput.getAccessor(store); - fetched.emplace(CachedInput{.lockedInput = lockedInput, .accessor = accessor}); + fetched.emplace( + CachedInput{.lockedInput = lockedInput, .accessor = accessor, .extraAttrs = extraAttrs}); } upsert(resolvedInput, *fetched); } else { @@ -36,7 +37,7 @@ InputCache::getAccessor(ref store, const Input & originalInput, UseRegist debug("got tree '%s' from '%s'", fetched->accessor, fetched->lockedInput.to_string()); - return {fetched->accessor, resolvedInput, fetched->lockedInput}; + return {fetched->accessor, resolvedInput, fetched->lockedInput, fetched->extraAttrs}; } struct InputCacheImpl : InputCache diff --git a/src/libfetchers/mercurial.cc b/src/libfetchers/mercurial.cc index 9b17d675ef3..bf460d9c6fb 100644 --- a/src/libfetchers/mercurial.cc +++ b/src/libfetchers/mercurial.cc @@ -6,7 +6,6 @@ #include "nix/util/tarfile.hh" #include "nix/store/store-api.hh" #include "nix/util/url-parts.hh" -#include "nix/fetchers/store-path-accessor.hh" #include "nix/fetchers/fetch-settings.hh" #include @@ -120,7 +119,7 @@ struct MercurialInputScheme : InputScheme { auto url = parseURL(getStrAttr(input.attrs, "url")); if (url.scheme == "file" && !input.getRef() && !input.getRev()) - return url.path; + return renderUrlPathEnsureLegal(url.path); return {}; } @@ -152,7 +151,7 @@ struct MercurialInputScheme : InputScheme { auto url = parseURL(getStrAttr(input.attrs, "url")); bool isLocal = url.scheme == "file"; - return {isLocal, isLocal ? url.path : url.to_string()}; + return {isLocal, isLocal ? renderUrlPathEnsureLegal(url.path) : url.to_string()}; } StorePath fetchToStore(ref store, Input & input) const @@ -331,7 +330,8 @@ struct MercurialInputScheme : InputScheme auto storePath = fetchToStore(store, input); - auto accessor = makeStorePathAccessor(store, storePath); + // We just added it, it should be there. + auto accessor = ref{store->getFSAccessor(storePath)}; accessor->setPathDisplay("«" + input.to_string() + "»"); diff --git a/src/libfetchers/meson.build b/src/libfetchers/meson.build index 922a2c49199..25d4d7821d7 100644 --- a/src/libfetchers/meson.build +++ b/src/libfetchers/meson.build @@ -32,9 +32,11 @@ libgit2 = dependency('libgit2', version : '>= 1.9') deps_private += libgit2 subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'attrs.cc', + 'builtin.cc', 'cache.cc', 'fetch-settings.cc', 'fetch-to-store.cc', @@ -49,7 +51,6 @@ sources = files( 'mercurial.cc', 'path.cc', 'registry.cc', - 'store-path-accessor.cc', 'tarball.cc', ) @@ -61,6 +62,7 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nixfetchers', sources, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, diff --git a/src/libfetchers/package.nix b/src/libfetchers/package.nix index 14592087999..b6b061e2d9f 100644 --- a/src/libfetchers/package.nix +++ b/src/libfetchers/package.nix @@ -17,7 +17,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-fetchers"; + pname = "determinate-nix-fetchers"; inherit version; workDir = ./.; diff --git a/src/libfetchers/path.cc b/src/libfetchers/path.cc index e5635ee75c7..4721a82d23f 100644 --- a/src/libfetchers/path.cc +++ b/src/libfetchers/path.cc @@ -1,7 +1,6 @@ #include "nix/fetchers/fetchers.hh" #include "nix/store/store-api.hh" #include "nix/util/archive.hh" -#include "nix/fetchers/store-path-accessor.hh" #include "nix/fetchers/cache.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/fetchers/fetch-settings.hh" @@ -20,7 +19,7 @@ struct PathInputScheme : InputScheme Input input{settings}; input.attrs.insert_or_assign("type", "path"); - input.attrs.insert_or_assign("path", url.path); + input.attrs.insert_or_assign("path", renderUrlPathEnsureLegal(url.path)); for (auto & [name, value] : url.query) if (name == "rev" || name == "narHash") @@ -74,7 +73,7 @@ struct PathInputScheme : InputScheme query.erase("__final"); return ParsedURL{ .scheme = "path", - .path = getStrAttr(input.attrs, "path"), + .path = splitString>(getStrAttr(input.attrs, "path"), "/"), .query = query, }; } @@ -124,8 +123,6 @@ struct PathInputScheme : InputScheme auto absPath = getAbsPath(input); - Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath)); - // FIXME: check whether access to 'path' is allowed. auto storePath = store->maybeParseStorePath(absPath.string()); @@ -134,48 +131,30 @@ struct PathInputScheme : InputScheme time_t mtime = 0; if (!storePath || storePath->name() != "source" || !store->isValidPath(*storePath)) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("copying %s to the store", absPath)); // FIXME: try to substitute storePath. auto src = sinkToSource( [&](Sink & sink) { mtime = dumpPathAndGetMtime(absPath.string(), sink, defaultPathFilter); }); storePath = store->addToStoreFromDump(*src, "source"); } - // To avoid copying the path again to the /nix/store, we need to add a cache entry. - ContentAddressMethod method = ContentAddressMethod::Raw::NixArchive; - auto fp = getFingerprint(store, input); - if (fp) { - auto cacheKey = makeFetchToStoreCacheKey(input.getName(), *fp, method, "/"); - input.settings->getCache()->upsert(cacheKey, *store, {}, *storePath); - } + auto accessor = ref{store->getFSAccessor(*storePath)}; + + // To prevent `fetchToStore()` copying the path again to Nix + // store, pre-create an entry in the fetcher cache. + auto info = store->queryPathInfo(*storePath); + accessor->fingerprint = + fmt("path:%s", store->queryPathInfo(*storePath)->narHash.to_string(HashFormat::SRI, true)); + input.settings->getCache()->upsert( + makeSourcePathToHashCacheKey(*accessor->fingerprint, ContentAddressMethod::Raw::NixArchive, "/"), + {{"hash", info->narHash.to_string(HashFormat::SRI, true)}}); /* Trust the lastModified value supplied by the user, if any. It's not a "secure" attribute so we don't care. */ if (!input.getLastModified()) input.attrs.insert_or_assign("lastModified", uint64_t(mtime)); - return {makeStorePathAccessor(store, *storePath), std::move(input)}; - } - - std::optional getFingerprint(ref store, const Input & input) const override - { - if (isRelative(input)) - return std::nullopt; - - /* If this path is in the Nix store, use the hash of the - store object and the subpath. */ - auto path = getAbsPath(input); - try { - auto [storePath, subPath] = store->toStorePath(path.string()); - auto info = store->queryPathInfo(storePath); - return fmt("path:%s:%s", info->narHash.to_string(HashFormat::Base16, false), subPath); - } catch (Error &) { - return std::nullopt; - } - } - - std::optional experimentalFeature() const override - { - return Xp::Flakes; + return {accessor, std::move(input)}; } }; diff --git a/src/libfetchers/store-path-accessor.cc b/src/libfetchers/store-path-accessor.cc deleted file mode 100644 index 65160e311b3..00000000000 --- a/src/libfetchers/store-path-accessor.cc +++ /dev/null @@ -1,11 +0,0 @@ -#include "nix/fetchers/store-path-accessor.hh" -#include "nix/store/store-api.hh" - -namespace nix { - -ref makeStorePathAccessor(ref store, const StorePath & storePath) -{ - return projectSubdirSourceAccessor(store->getFSAccessor(), storePath.to_string()); -} - -} // namespace nix diff --git a/src/libfetchers/tarball.cc b/src/libfetchers/tarball.cc index 309bbaf5a3d..76cf3fd3267 100644 --- a/src/libfetchers/tarball.cc +++ b/src/libfetchers/tarball.cc @@ -6,7 +6,6 @@ #include "nix/util/archive.hh" #include "nix/util/tarfile.hh" #include "nix/util/types.hh" -#include "nix/fetchers/store-path-accessor.hh" #include "nix/store/store-api.hh" #include "nix/fetchers/git-utils.hh" #include "nix/fetchers/fetch-settings.hh" @@ -43,7 +42,7 @@ DownloadFileResult downloadFile( if (cached && !cached->expired) return useCached(); - FileTransferRequest request(url); + FileTransferRequest request(VerbatimURL{url}); request.headers = headers; if (cached) request.expectedETag = getStrAttr(cached->value, "etag"); @@ -74,7 +73,7 @@ DownloadFileResult downloadFile( StringSink sink; dumpString(res.data, sink); auto hash = hashString(HashAlgorithm::SHA256, res.data); - ValidPathInfo info{ + auto info = ValidPathInfo::makeFromCA( *store, name, FixedOutputInfo{ @@ -82,8 +81,7 @@ DownloadFileResult downloadFile( .hash = hash, .references = {}, }, - hashString(HashAlgorithm::SHA256, sink.s), - }; + hashString(HashAlgorithm::SHA256, sink.s)); info.narSize = sink.s.size(); auto source = StringSource{sink.s}; store->addToStore(info, source, NoRepair, NoCheckSigs); @@ -107,20 +105,20 @@ DownloadFileResult downloadFile( } static DownloadTarballResult downloadTarball_( - const Settings & settings, const std::string & url, const Headers & headers, const std::string & displayPrefix) + const Settings & settings, const std::string & urlS, const Headers & headers, const std::string & displayPrefix) { + ParsedURL url = parseURL(urlS); // Some friendly error messages for common mistakes. // Namely lets catch when the url is a local file path, but // it is not in fact a tarball. - if (url.rfind("file://", 0) == 0) { - // Remove "file://" prefix to get the local file path - std::string localPath = url.substr(7); - if (!std::filesystem::exists(localPath)) { + if (url.scheme == "file") { + std::filesystem::path localPath = renderUrlPathEnsureLegal(url.path); + if (!exists(localPath)) { throw Error("tarball '%s' does not exist.", localPath); } - if (std::filesystem::is_directory(localPath)) { - if (std::filesystem::exists(localPath + "/.git")) { + if (is_directory(localPath)) { + if (exists(localPath / ".git")) { throw Error( "tarball '%s' is a git repository, not a tarball. Please use `git+file` as the scheme.", localPath); } @@ -128,7 +126,7 @@ static DownloadTarballResult downloadTarball_( } } - Cache::Key cacheKey{"tarball", {{"url", url}}}; + Cache::Key cacheKey{"tarball", {{"url", urlS}}}; auto cached = settings.getCache()->lookupExpired(cacheKey); @@ -138,11 +136,11 @@ static DownloadTarballResult downloadTarball_( .treeHash = treeHash, .lastModified = (time_t) getIntAttr(infoAttrs, "lastModified"), .immutableUrl = maybeGetStrAttr(infoAttrs, "immutableUrl"), - .accessor = getTarballCache()->getAccessor(treeHash, false, displayPrefix), + .accessor = settings.getTarballCache()->getAccessor(treeHash, false, displayPrefix), }; }; - if (cached && !getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash"))) + if (cached && !settings.getTarballCache()->hasObject(getRevAttr(cached->value, "treeHash"))) cached.reset(); if (cached && !cached->expired) @@ -166,7 +164,7 @@ static DownloadTarballResult downloadTarball_( /* Note: if the download is cached, `importTarball()` will receive no data, which causes it to import an empty tarball. */ - auto archive = hasSuffix(toLower(parseURL(url).path), ".zip") ? ({ + auto archive = !url.path.empty() && hasSuffix(toLower(url.path.back()), ".zip") ? ({ /* In streaming mode, libarchive doesn't handle symlinks in zip files correctly (#10649). So write the entire file to disk so libarchive can access it @@ -180,8 +178,8 @@ static DownloadTarballResult downloadTarball_( } TarArchive{path}; }) - : TarArchive{*source}; - auto tarballCache = getTarballCache(); + : TarArchive{*source}; + auto tarballCache = settings.getTarballCache(); auto parseSink = tarballCache->getFileSystemObjectSink(); auto lastModified = unpackTarfileToSink(archive, *parseSink); auto tree = parseSink->flush(); @@ -234,8 +232,11 @@ struct CurlInputScheme : InputScheme { const StringSet transportUrlSchemes = {"file", "http", "https"}; - bool hasTarballExtension(std::string_view path) const + bool hasTarballExtension(const ParsedURL & url) const { + if (url.path.empty()) + return false; + const auto & path = url.path.back(); return hasSuffix(path, ".zip") || hasSuffix(path, ".tar") || hasSuffix(path, ".tgz") || hasSuffix(path, ".tar.gz") || hasSuffix(path, ".tar.xz") || hasSuffix(path, ".tar.bz2") || hasSuffix(path, ".tar.zst"); @@ -336,7 +337,7 @@ struct FileInputScheme : CurlInputScheme auto parsedUrlScheme = parseUrlScheme(url.scheme); return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application ? parsedUrlScheme.application.value() == schemeName() - : (!requireTree && !hasTarballExtension(url.path))); + : (!requireTree && !hasTarballExtension(url))); } std::pair, Input> getAccessor(ref store, const Input & _input) const override @@ -352,7 +353,7 @@ struct FileInputScheme : CurlInputScheme auto narHash = store->queryPathInfo(file.storePath)->narHash; input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - auto accessor = makeStorePathAccessor(store, file.storePath); + auto accessor = ref{store->getFSAccessor(file.storePath)}; accessor->setPathDisplay("«" + input.to_string() + "»"); @@ -373,7 +374,7 @@ struct TarballInputScheme : CurlInputScheme return transportUrlSchemes.count(std::string(parsedUrlScheme.transport)) && (parsedUrlScheme.application ? parsedUrlScheme.application.value() == schemeName() - : (requireTree || hasTarballExtension(url.path))); + : (requireTree || hasTarballExtension(url))); } std::pair, Input> getAccessor(ref store, const Input & _input) const override @@ -397,7 +398,9 @@ struct TarballInputScheme : CurlInputScheme input.attrs.insert_or_assign( "narHash", - getTarballCache()->treeHashToNarHash(*input.settings, result.treeHash).to_string(HashFormat::SRI, true)); + input.settings->getTarballCache() + ->treeHashToNarHash(*input.settings, result.treeHash) + .to_string(HashFormat::SRI, true)); return {result.accessor, input}; } diff --git a/src/libflake-c/meson.build b/src/libflake-c/meson.build index 933e06d9037..d0d45cfa813 100644 --- a/src/libflake-c/meson.build +++ b/src/libflake-c/meson.build @@ -32,6 +32,7 @@ deps_public_maybe_subproject = [ subdir('nix-meson-build-support/subprojects') subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'nix_api_flake.cc', @@ -53,6 +54,7 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nixflakec', sources, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, diff --git a/src/libflake-c/nix_api_flake.cc b/src/libflake-c/nix_api_flake.cc index ad8f0bf4ec4..2de0e667ec3 100644 --- a/src/libflake-c/nix_api_flake.cc +++ b/src/libflake-c/nix_api_flake.cc @@ -10,6 +10,8 @@ #include "nix/flake/flake.hh" +extern "C" { + nix_flake_settings * nix_flake_settings_new(nix_c_context * context) { nix_clear_err(context); @@ -203,3 +205,5 @@ nix_value * nix_locked_flake_get_output_attrs( } NIXC_CATCH_ERRS_NULL } + +} // extern "C" diff --git a/src/libflake-c/package.nix b/src/libflake-c/package.nix index 8c6883d9cf9..9ae3ec69515 100644 --- a/src/libflake-c/package.nix +++ b/src/libflake-c/package.nix @@ -17,7 +17,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-flake-c"; + pname = "determinate-nix-flake-c"; inherit version; workDir = ./.; diff --git a/src/libflake-tests/flakeref.cc b/src/libflake-tests/flakeref.cc index 404d7590a6a..0ba25c98a2f 100644 --- a/src/libflake-tests/flakeref.cc +++ b/src/libflake-tests/flakeref.cc @@ -2,6 +2,7 @@ #include "nix/fetchers/fetch-settings.hh" #include "nix/flake/flakeref.hh" +#include "nix/fetchers/attrs.hh" namespace nix { @@ -9,8 +10,6 @@ namespace nix { TEST(parseFlakeRef, path) { - experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); - fetchers::Settings fetchSettings; { @@ -59,8 +58,6 @@ TEST(parseFlakeRef, path) TEST(parseFlakeRef, GitArchiveInput) { - experimentalFeatureSettings.experimentalFeatures.get().insert(Xp::Flakes); - fetchers::Settings fetchSettings; { @@ -90,6 +87,157 @@ TEST(parseFlakeRef, GitArchiveInput) } } +struct InputFromURLTestCase +{ + std::string url; + fetchers::Attrs attrs; + std::string description; + std::string expectedUrl = url; +}; + +class InputFromURLTest : public ::testing::WithParamInterface, public ::testing::Test +{}; + +TEST_P(InputFromURLTest, attrsAreCorrectAndRoundTrips) +{ + fetchers::Settings fetchSettings; + + const auto & testCase = GetParam(); + + auto flakeref = parseFlakeRef(fetchSettings, testCase.url); + + EXPECT_EQ(flakeref.toAttrs(), testCase.attrs); + EXPECT_EQ(flakeref.to_string(), testCase.expectedUrl); + + auto input = fetchers::Input::fromURL(fetchSettings, flakeref.to_string()); + + EXPECT_EQ(input.toURLString(), testCase.expectedUrl); + EXPECT_EQ(input.toAttrs(), testCase.attrs); + + // Round-trip check. + auto input2 = fetchers::Input::fromURL(fetchSettings, input.toURLString()); + EXPECT_EQ(input, input2); + EXPECT_EQ(input.toURLString(), input2.toURLString()); +} + +using fetchers::Attr; + +INSTANTIATE_TEST_SUITE_P( + InputFromURL, + InputFromURLTest, + ::testing::Values( + InputFromURLTestCase{ + .url = "flake:nixpkgs", + .attrs = + { + {"id", Attr("nixpkgs")}, + {"type", Attr("indirect")}, + }, + .description = "basic_indirect", + }, + InputFromURLTestCase{ + .url = "flake:nixpkgs/branch", + .attrs = + { + {"id", Attr("nixpkgs")}, + {"type", Attr("indirect")}, + {"ref", Attr("branch")}, + }, + .description = "basic_indirect_branch", + }, + InputFromURLTestCase{ + .url = "nixpkgs/branch", + .attrs = + { + {"id", Attr("nixpkgs")}, + {"type", Attr("indirect")}, + {"ref", Attr("branch")}, + }, + .description = "flake_id_ref_branch", + .expectedUrl = "flake:nixpkgs/branch", + }, + InputFromURLTestCase{ + .url = "nixpkgs/branch/2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", + .attrs = + { + {"id", Attr("nixpkgs")}, + {"type", Attr("indirect")}, + {"ref", Attr("branch")}, + {"rev", Attr("2aae6c35c94fcfb415dbe95f408b9ce91ee846ed")}, + }, + .description = "flake_id_ref_branch_trailing_slash", + .expectedUrl = "flake:nixpkgs/branch/2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", + }, + // The following tests are for back-compat with lax parsers in older versions + // that used `tokenizeString` for splitting path segments, which ignores empty + // strings. + InputFromURLTestCase{ + .url = "nixpkgs/branch////", + .attrs = + { + {"id", Attr("nixpkgs")}, + {"type", Attr("indirect")}, + {"ref", Attr("branch")}, + }, + .description = "flake_id_ref_branch_ignore_empty_trailing_segments", + .expectedUrl = "flake:nixpkgs/branch", + }, + InputFromURLTestCase{ + .url = "nixpkgs/branch///2aae6c35c94fcfb415dbe95f408b9ce91ee846ed///", + .attrs = + { + {"id", Attr("nixpkgs")}, + {"type", Attr("indirect")}, + {"ref", Attr("branch")}, + {"rev", Attr("2aae6c35c94fcfb415dbe95f408b9ce91ee846ed")}, + }, + .description = "flake_id_ref_branch_ignore_empty_segments_ref_rev", + .expectedUrl = "flake:nixpkgs/branch/2aae6c35c94fcfb415dbe95f408b9ce91ee846ed", + }, + InputFromURLTestCase{ + // Note that this is different from above because the "flake id" shorthand + // doesn't allow this. + .url = "flake:/nixpkgs///branch////", + .attrs = + { + {"id", Attr("nixpkgs")}, + {"type", Attr("indirect")}, + {"ref", Attr("branch")}, + }, + .description = "indirect_branch_empty_segments_everywhere", + .expectedUrl = "flake:nixpkgs/branch", + }, + InputFromURLTestCase{ + // TODO: Technically this has an empty authority, but it's ignored + // for now. Yes, this is what all versions going back to at least + // 2.18 did and yes, this should not be allowed. + .url = "github://////owner%42/////repo%41///branch%43////", + .attrs = + { + {"type", Attr("github")}, + {"owner", Attr("ownerB")}, + {"repo", Attr("repoA")}, + {"ref", Attr("branchC")}, + }, + .description = "github_ref_slashes_in_path_everywhere", + .expectedUrl = "github:ownerB/repoA/branchC", + }, + InputFromURLTestCase{ + // FIXME: Subgroups in gitlab URLs are busted. This double-encoding + // behavior exists since 2.18. See issue #9161 and PR #8845. + .url = "gitlab:/owner%252Fsubgroup/////repo%41///branch%43////", + .attrs = + { + {"type", Attr("gitlab")}, + {"owner", Attr("owner%2Fsubgroup")}, + {"repo", Attr("repoA")}, + {"ref", Attr("branchC")}, + }, + .description = "gitlab_ref_slashes_in_path_everywhere_with_pct_encoding", + .expectedUrl = "gitlab:owner%252Fsubgroup/repoA/branchC", + }), + [](const ::testing::TestParamInfo & info) { return info.param.description; }); + TEST(to_string, doesntReencodeUrl) { fetchers::Settings fetchSettings; diff --git a/src/libflake-tests/meson.build b/src/libflake-tests/meson.build index 59094abe866..28f9257a30f 100644 --- a/src/libflake-tests/meson.build +++ b/src/libflake-tests/meson.build @@ -34,6 +34,7 @@ gtest = dependency('gtest', main : true) deps_private += gtest subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'flakeref.cc', @@ -58,9 +59,8 @@ this_exe = executable( test( meson.project_name(), this_exe, - env : { + env : asan_test_options_env + { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', - 'NIX_CONFIG' : 'extra-experimental-features = flakes', 'HOME' : meson.current_build_dir() / 'test-home', }, protocol : 'gtest', diff --git a/src/libflake-tests/package.nix b/src/libflake-tests/package.nix index 397ef419244..49eaed151c3 100644 --- a/src/libflake-tests/package.nix +++ b/src/libflake-tests/package.nix @@ -59,8 +59,8 @@ mkMesonExecutable (finalAttrs: { buildInputs = [ writableTmpDirAsHomeHook ]; } ('' + export ASAN_OPTIONS=abort_on_error=1:print_summary=1:detect_leaks=0 export _NIX_TEST_UNIT_DATA=${resolvePath ./data} - export NIX_CONFIG="extra-experimental-features = flakes" ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out ''); diff --git a/src/libflake/call-flake.nix b/src/libflake/call-flake.nix index ed7947e0601..6db35269705 100644 --- a/src/libflake/call-flake.nix +++ b/src/libflake/call-flake.nix @@ -45,7 +45,17 @@ let parentNode = allNodes.${getInputByPath lockFile.root node.parent}; sourceInfo = - if hasOverride then + if node.buildTime or false then + derivation { + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = node.locked; + outputHashMode = "recursive"; + outputHash = node.locked.narHash; + } + else if hasOverride then overrides.${key}.sourceInfo else if isRelative then parentNode.sourceInfo @@ -93,6 +103,7 @@ let result = if node.flake or true then assert builtins.isFunction flake.outputs; + assert !(node.buildTime or false); result else sourceInfo // { inherit sourceInfo outPath; }; diff --git a/src/libflake/flake-primops.cc b/src/libflake/flake-primops.cc index 7c5ce01b269..703463141f3 100644 --- a/src/libflake/flake-primops.cc +++ b/src/libflake/flake-primops.cc @@ -52,7 +52,6 @@ PrimOp getFlake(const Settings & settings) ``` )", .fun = prim_getFlake, - .experimentalFeature = Xp::Flakes, }; } @@ -94,7 +93,6 @@ nix::PrimOp parseFlakeRef({ ``` )", .fun = prim_parseFlakeRef, - .experimentalFeature = Xp::Flakes, }); static void prim_flakeRefToString(EvalState & state, const PosIdx pos, Value ** args, Value & v) @@ -154,7 +152,6 @@ nix::PrimOp flakeRefToString({ ``` )", .fun = prim_flakeRefToString, - .experimentalFeature = Xp::Flakes, }); } // namespace nix::flake::primops diff --git a/src/libflake/flake.cc b/src/libflake/flake.cc index b31bef21103..c510872b37d 100644 --- a/src/libflake/flake.cc +++ b/src/libflake/flake.cc @@ -14,6 +14,7 @@ #include "nix/store/local-fs-store.hh" #include "nix/fetchers/fetch-to-store.hh" #include "nix/util/memory-source-accessor.hh" +#include "nix/util/mounted-source-accessor.hh" #include "nix/fetchers/input-cache.hh" #include @@ -21,35 +22,22 @@ namespace nix { using namespace flake; +using namespace fetchers; namespace flake { -static StorePath copyInputToStore( - EvalState & state, fetchers::Input & input, const fetchers::Input & originalInput, ref accessor) -{ - auto storePath = fetchToStore(*input.settings, *state.store, accessor, FetchMode::Copy, input.getName()); - - state.allowPath(storePath); - - auto narHash = state.store->queryPathInfo(storePath)->narHash; - input.attrs.insert_or_assign("narHash", narHash.to_string(HashFormat::SRI, true)); - - assert(!originalInput.getNarHash() || storePath == originalInput.computeStorePath(*state.store)); - - return storePath; -} - static void forceTrivialValue(EvalState & state, Value & value, const PosIdx pos) { - if (value.isThunk() && value.isTrivial()) + if (value.isTrivial()) state.forceValue(value, pos); } static void expectType(EvalState & state, ValueType type, Value & value, const PosIdx pos) { forceTrivialValue(state, value, pos); - if (value.type() != type) - throw Error("expected %s but got %s at %s", showType(type), showType(value.type()), state.positions[pos]); + auto t = value.type(); + if (t != type) + throw Error("expected %s but got %s at %s", showType(type), showType(t), state.positions[pos]); } static std::pair, fetchers::Attrs> parseFlakeInputs( @@ -60,7 +48,7 @@ static std::pair, fetchers::Attrs> parseFlakeInput const SourcePath & flakeDir, bool allowSelf); -static void parseFlakeInputAttr(EvalState & state, const Attr & attr, fetchers::Attrs & attrs) +static void parseFlakeInputAttr(EvalState & state, const nix::Attr & attr, fetchers::Attrs & attrs) { // Allow selecting a subset of enum values #pragma GCC diagnostic push @@ -114,6 +102,7 @@ static FlakeInput parseFlakeInput( auto sUrl = state.symbols.create("url"); auto sFlake = state.symbols.create("flake"); auto sFollows = state.symbols.create("follows"); + auto sBuildTime = state.symbols.create("buildTime"); fetchers::Attrs attrs; std::optional url; @@ -142,6 +131,11 @@ static FlakeInput parseFlakeInput( } else if (attr.name == sFlake) { expectType(state, nBool, *attr.value, attr.pos); input.isFlake = attr.value->boolean(); + } else if (attr.name == sBuildTime) { + expectType(state, nBool, *attr.value, attr.pos); + input.buildTime = attr.value->boolean(); + if (input.buildTime) + experimentalFeatureSettings.require(Xp::BuildTimeFetchTree); } else if (attr.name == sInputs) { input.overrides = parseFlakeInputs(state, attr.value, attr.pos, lockRootAttrPath, flakeDir, false).first; @@ -232,7 +226,7 @@ static Flake readFlake( .path = flakePath, }; - if (auto description = vInfo.attrs()->get(state.sDescription)) { + if (auto description = vInfo.attrs()->get(state.s.description)) { expectType(state, nString, *description->value, description->pos); flake.description = description->value->c_str(); } @@ -253,7 +247,7 @@ static Flake readFlake( if (outputs->value->isLambda() && outputs->value->lambda().fun->hasFormals()) { for (auto & formal : outputs->value->lambda().fun->formals->formals) { - if (formal.name != state.sSelf) + if (formal.name != state.s.self) flake.inputs.emplace( state.symbols[formal.name], FlakeInput{.ref = parseFlakeRef(state.fetchSettings, std::string(state.symbols[formal.name]))}); @@ -305,7 +299,8 @@ static Flake readFlake( } for (auto & attr : *vInfo.attrs()) { - if (attr.name != state.sDescription && attr.name != sInputs && attr.name != sOutputs && attr.name != sNixConfig) + if (attr.name != state.s.description && attr.name != sInputs && attr.name != sOutputs + && attr.name != sNixConfig) throw Error( "flake '%s' has an unsupported attribute '%s', at %s", resolvedRef, @@ -335,13 +330,15 @@ static Flake getFlake( EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries, - const InputAttrPath & lockRootAttrPath) + const InputAttrPath & lockRootAttrPath, + bool requireLockable) { // Fetch a lazy tree first. auto cachedInput = state.inputCache->getAccessor(state.store, originalRef.input, useRegistries); - auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), originalRef.subdir); - auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), originalRef.subdir); + auto subdir = fetchers::maybeGetStrAttr(cachedInput.extraAttrs, "dir").value_or(originalRef.subdir); + auto resolvedRef = FlakeRef(std::move(cachedInput.resolvedInput), subdir); + auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), subdir); // Parse/eval flake.nix to get at the input.self attributes. auto flake = readFlake(state, originalRef, resolvedRef, lockedRef, {cachedInput.accessor}, lockRootAttrPath); @@ -358,16 +355,20 @@ static Flake getFlake( lockedRef = FlakeRef(std::move(cachedInput2.lockedInput), newLockedRef.subdir); } - // Copy the tree to the store. - auto storePath = copyInputToStore(state, lockedRef.input, originalRef.input, cachedInput.accessor); - // Re-parse flake.nix from the store. - return readFlake(state, originalRef, resolvedRef, lockedRef, state.storePath(storePath), lockRootAttrPath); + return readFlake( + state, + originalRef, + resolvedRef, + lockedRef, + state.storePath(state.mountInput(lockedRef.input, originalRef.input, cachedInput.accessor, requireLockable)), + lockRootAttrPath); } -Flake getFlake(EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries) +Flake getFlake( + EvalState & state, const FlakeRef & originalRef, fetchers::UseRegistries useRegistries, bool requireLockable) { - return getFlake(state, originalRef, useRegistries, {}); + return getFlake(state, originalRef, useRegistries, {}, requireLockable); } static LockFile readLockFile(const fetchers::Settings & fetchSettings, const SourcePath & lockFilePath) @@ -381,13 +382,11 @@ static LockFile readLockFile(const fetchers::Settings & fetchSettings, const Sou LockedFlake lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, const LockFlags & lockFlags) { - experimentalFeatureSettings.require(Xp::Flakes); - auto useRegistries = lockFlags.useRegistries.value_or(settings.useRegistries); auto useRegistriesTop = useRegistries ? fetchers::UseRegistries::All : fetchers::UseRegistries::No; auto useRegistriesInputs = useRegistries ? fetchers::UseRegistries::Limited : fetchers::UseRegistries::No; - auto flake = getFlake(state, topRef, useRegistriesTop, {}); + auto flake = getFlake(state, topRef, useRegistriesTop, {}, lockFlags.requireLockable); if (lockFlags.applyNixConfig) { flake.config.apply(settings); @@ -568,7 +567,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, if (auto resolvedPath = resolveRelativePath()) { return readFlake(state, ref, ref, ref, *resolvedPath, inputAttrPath); } else { - return getFlake(state, ref, useRegistries, inputAttrPath); + return getFlake(state, ref, useRegistriesInputs, inputAttrPath, true); } }; @@ -591,7 +590,11 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, didn't change and there is no override from a higher level flake. */ auto childNode = make_ref( - oldLock->lockedRef, oldLock->originalRef, oldLock->isFlake, oldLock->parentInputAttrPath); + oldLock->lockedRef, + oldLock->originalRef, + oldLock->isFlake, + oldLock->buildTime, + oldLock->parentInputAttrPath); node->inputs.insert_or_assign(id, childNode); @@ -679,12 +682,34 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, auto inputIsOverride = explicitCliOverrides.contains(inputAttrPath); auto ref = (input2.ref && inputIsOverride) ? *input2.ref : *input.ref; + /* Warn against the use of indirect flakerefs + (but only at top-level since we don't want + to annoy users about flakes that are not + under their control). */ + auto warnRegistry = [&](const FlakeRef & resolvedRef) { + if (inputAttrPath.size() == 1 && !input.ref->input.isDirect()) { + std::ostringstream s; + printLiteralString(s, resolvedRef.to_string()); + warn( + "Flake input '%1%' uses the flake registry. " + "Using the registry in flake inputs is deprecated in Determinate Nix. " + "To make your flake future-proof, add the following to '%2%':\n" + "\n" + " inputs.%1%.url = %3%;\n" + "\n" + "For more information, see: https://github.com/DeterminateSystems/nix-src/issues/37", + inputAttrPathS, + flake.path, + s.str()); + } + }; + if (input.isFlake) { auto inputFlake = getInputFlake( *input.ref, inputIsOverride ? fetchers::UseRegistries::All : useRegistriesInputs); - auto childNode = - make_ref(inputFlake.lockedRef, ref, true, overriddenParentPath); + auto childNode = make_ref( + inputFlake.lockedRef, ref, true, input.buildTime, overriddenParentPath); node->inputs.insert_or_assign(id, childNode); @@ -706,6 +731,8 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, inputAttrPath, inputFlake.path, false); + + warnRegistry(inputFlake.resolvedRef); } else { @@ -714,20 +741,24 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, if (auto resolvedPath = resolveRelativePath()) { return {*resolvedPath, *input.ref}; } else { - auto cachedInput = state.inputCache->getAccessor( - state.store, input.ref->input, useRegistriesInputs); + auto cachedInput = + state.inputCache->getAccessor(state.store, input.ref->input, useRegistriesTop); + auto resolvedRef = + FlakeRef(std::move(cachedInput.resolvedInput), input.ref->subdir); auto lockedRef = FlakeRef(std::move(cachedInput.lockedInput), input.ref->subdir); - // FIXME: allow input to be lazy. - auto storePath = copyInputToStore( - state, lockedRef.input, input.ref->input, cachedInput.accessor); + warnRegistry(resolvedRef); - return {state.storePath(storePath), lockedRef}; + return { + state.storePath(state.mountInput( + lockedRef.input, input.ref->input, cachedInput.accessor, true, true)), + lockedRef}; } }(); - auto childNode = make_ref(lockedRef, ref, false, overriddenParentPath); + auto childNode = + make_ref(lockedRef, ref, false, input.buildTime, overriddenParentPath); nodePaths.emplace(childNode, path); @@ -843,7 +874,7 @@ lockFlake(const Settings & settings, EvalState & state, const FlakeRef & topRef, repo, so we should re-read it. FIXME: we could also just clear the 'rev' field... */ auto prevLockedRef = flake.lockedRef; - flake = getFlake(state, topRef, useRegistriesTop); + flake = getFlake(state, topRef, useRegistriesTop, lockFlags.requireLockable); if (lockFlags.commitLockFile && flake.lockedRef.input.getRev() && prevLockedRef.input.getRev() != flake.lockedRef.input.getRev()) @@ -890,8 +921,6 @@ static Value * requireInternalFile(EvalState & state, CanonPath path) void callFlake(EvalState & state, const LockedFlake & lockedFlake, Value & vRes) { - experimentalFeatureSettings.require(Xp::Flakes); - auto [lockFileStr, keyMap] = lockedFlake.lockFile.to_string(); auto overrides = state.buildBindings(lockedFlake.nodePaths.size()); diff --git a/src/libflake/flakeref.cc b/src/libflake/flakeref.cc index 070f4e48391..38979783d5e 100644 --- a/src/libflake/flakeref.cc +++ b/src/libflake/flakeref.cc @@ -143,7 +143,7 @@ std::pair parsePathFlakeRefWithFragment( auto parsedURL = ParsedURL{ .scheme = "git+file", .authority = ParsedURL::Authority{}, - .path = flakeRoot, + .path = splitString>(flakeRoot, "/"), .query = query, .fragment = fragment, }; @@ -172,7 +172,13 @@ std::pair parsePathFlakeRefWithFragment( return fromParsedURL( fetchSettings, - {.scheme = "path", .authority = ParsedURL::Authority{}, .path = path, .query = query, .fragment = fragment}, + { + .scheme = "path", + .authority = ParsedURL::Authority{}, + .path = splitString>(path, "/"), + .query = query, + .fragment = fragment, + }, isFlake); } @@ -192,8 +198,8 @@ parseFlakeIdRef(const fetchers::Settings & fetchSettings, const std::string & ur if (std::regex_match(url, match, flakeRegex)) { auto parsedURL = ParsedURL{ .scheme = "flake", - .authority = ParsedURL::Authority{}, - .path = match[1], + .authority = std::nullopt, + .path = splitString>(match[1].str(), "/"), }; return std::make_pair( @@ -211,8 +217,12 @@ std::optional> parseURLFlakeRef( { try { auto parsed = parseURL(url, /*lenient=*/true); - if (baseDir && (parsed.scheme == "path" || parsed.scheme == "git+file") && !isAbsolute(parsed.path)) - parsed.path = absPath(parsed.path, *baseDir); + if (baseDir && (parsed.scheme == "path" || parsed.scheme == "git+file")) { + /* Here we know that the path must not contain encoded '/' or NUL bytes. */ + auto path = renderUrlPathEnsureLegal(parsed.path); + if (!isAbsolute(path)) + parsed.path = splitString>(absPath(path, *baseDir), "/"); + } return fromParsedURL(fetchSettings, std::move(parsed), isFlake); } catch (BadURL &) { return std::nullopt; diff --git a/src/libflake/include/nix/flake/flake.hh b/src/libflake/include/nix/flake/flake.hh index 13002b47c05..3c8acb2b72d 100644 --- a/src/libflake/include/nix/flake/flake.hh +++ b/src/libflake/include/nix/flake/flake.hh @@ -43,12 +43,18 @@ typedef std::map FlakeInputs; struct FlakeInput { std::optional ref; + /** - * true = process flake to get outputs - * - * false = (fetched) static source path + * Whether to call the `flake.nix` file in this input to get its outputs. */ bool isFlake = true; + + /** + * Whether to fetch this input at evaluation time or at build + * time. + */ + bool buildTime = false; + std::optional follows; FlakeInputs overrides; }; @@ -115,7 +121,8 @@ struct Flake } }; -Flake getFlake(EvalState & state, const FlakeRef & flakeRef, fetchers::UseRegistries useRegistries); +Flake getFlake( + EvalState & state, const FlakeRef & flakeRef, fetchers::UseRegistries useRegistries, bool requireLockable = true); /** * Fingerprint of a locked flake; used as a cache key. @@ -211,6 +218,11 @@ struct LockFlags * for those inputs will be ignored. */ std::set inputUpdates; + + /** + * Whether to require a locked input. + */ + bool requireLockable = true; }; LockedFlake diff --git a/src/libflake/include/nix/flake/lockfile.hh b/src/libflake/include/nix/flake/lockfile.hh index c5740a2f114..1ca7cc3dd30 100644 --- a/src/libflake/include/nix/flake/lockfile.hh +++ b/src/libflake/include/nix/flake/lockfile.hh @@ -37,6 +37,7 @@ struct LockedNode : Node { FlakeRef lockedRef, originalRef; bool isFlake = true; + bool buildTime = false; /* The node relative to which relative source paths (e.g. 'path:../foo') are interpreted. */ @@ -46,10 +47,12 @@ struct LockedNode : Node const FlakeRef & lockedRef, const FlakeRef & originalRef, bool isFlake = true, + bool buildTime = false, std::optional parentInputAttrPath = {}) : lockedRef(std::move(lockedRef)) , originalRef(std::move(originalRef)) , isFlake(isFlake) + , buildTime(buildTime) , parentInputAttrPath(std::move(parentInputAttrPath)) { } diff --git a/src/libflake/include/nix/flake/settings.hh b/src/libflake/include/nix/flake/settings.hh index 618ed4d38ef..d8ed4a91a75 100644 --- a/src/libflake/include/nix/flake/settings.hh +++ b/src/libflake/include/nix/flake/settings.hh @@ -20,13 +20,7 @@ struct Settings : public Config void configureEvalSettings(nix::EvalSettings & evalSettings) const; Setting useRegistries{ - this, - true, - "use-registries", - "Whether to use flake registries to resolve flake references.", - {}, - true, - Xp::Flakes}; + this, true, "use-registries", "Whether to use flake registries to resolve flake references.", {}, true}; Setting acceptFlakeConfig{ this, @@ -34,8 +28,7 @@ struct Settings : public Config "accept-flake-config", "Whether to accept Nix configuration settings from a flake without prompting.", {}, - true, - Xp::Flakes}; + true}; Setting commitLockFileSummary{ this, @@ -46,8 +39,7 @@ struct Settings : public Config empty, the summary is generated based on the action performed. )", {"commit-lockfile-summary"}, - true, - Xp::Flakes}; + true}; }; } // namespace nix::flake diff --git a/src/libflake/lockfile.cc b/src/libflake/lockfile.cc index 94e7f11f1a6..9d3aee93f42 100644 --- a/src/libflake/lockfile.cc +++ b/src/libflake/lockfile.cc @@ -1,5 +1,3 @@ -#include - #include "nix/fetchers/fetch-settings.hh" #include "nix/flake/settings.hh" #include "nix/flake/lockfile.hh" @@ -9,6 +7,7 @@ #include #include +#include #include #include @@ -38,6 +37,7 @@ LockedNode::LockedNode(const fetchers::Settings & fetchSettings, const nlohmann: : lockedRef(getFlakeRef(fetchSettings, json, "locked", "info")) // FIXME: remove "info" , originalRef(getFlakeRef(fetchSettings, json, "original", nullptr)) , isFlake(json.find("flake") != json.end() ? (bool) json["flake"] : true) + , buildTime(json.find("buildTime") != json.end() ? (bool) json["buildTime"] : false) , parentInputAttrPath( json.find("parent") != json.end() ? (std::optional) json["parent"] : std::nullopt) { @@ -162,7 +162,7 @@ std::pair LockFile::toJSON() const { nlohmann::json nodes; KeyMap nodeKeys; - std::unordered_set keys; + boost::unordered_flat_set keys; std::function node)> dumpNode; @@ -210,6 +210,8 @@ std::pair LockFile::toJSON() const n["locked"].erase("__final"); if (!lockedNode->isFlake) n["flake"] = false; + if (lockedNode->buildTime) + n["buildTime"] = true; if (lockedNode->parentInputAttrPath) n["parent"] = *lockedNode->parentInputAttrPath; } diff --git a/src/libflake/meson.build b/src/libflake/meson.build index 191d8f0680c..3bd04fcf415 100644 --- a/src/libflake/meson.build +++ b/src/libflake/meson.build @@ -29,6 +29,7 @@ nlohmann_json = dependency('nlohmann_json', version : '>= 3.9') deps_public += nlohmann_json subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') subdir('nix-meson-build-support/generate-header') @@ -58,6 +59,7 @@ this_library = library( 'nixflake', sources, generated_headers, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, diff --git a/src/libflake/package.nix b/src/libflake/package.nix index dd442a44ec9..2b0c827a09c 100644 --- a/src/libflake/package.nix +++ b/src/libflake/package.nix @@ -18,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-flake"; + pname = "determinate-nix-flake"; inherit version; workDir = ./.; diff --git a/src/libflake/url-name.cc b/src/libflake/url-name.cc index b3eeca26a96..3bba3692eb1 100644 --- a/src/libflake/url-name.cc +++ b/src/libflake/url-name.cc @@ -27,16 +27,21 @@ std::optional getNameFromURL(const ParsedURL & url) return match.str(2); } + /* This is not right, because special chars like slashes within the + path fragments should be percent encoded, but I don't think any + of the regexes above care. */ + auto path = concatStringsSep("/", url.path); + /* If this is a github/gitlab/sourcehut flake, use the repo name */ - if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(url.path, match, secondPathSegmentRegex)) + if (std::regex_match(url.scheme, gitProviderRegex) && std::regex_match(path, match, secondPathSegmentRegex)) return match.str(1); /* If it is a regular git flake, use the directory name */ - if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(url.path, match, lastPathSegmentRegex)) + if (std::regex_match(url.scheme, gitSchemeRegex) && std::regex_match(path, match, lastPathSegmentRegex)) return match.str(1); /* If there is no fragment, take the last element of the path */ - if (std::regex_match(url.path, match, lastPathSegmentRegex)) + if (std::regex_match(path, match, lastPathSegmentRegex)) return match.str(1); /* If even that didn't work, the URL does not contain enough info to determine a useful name */ diff --git a/src/libmain-c/meson.build b/src/libmain-c/meson.build index 9e26ad8adf3..2ac2b799bca 100644 --- a/src/libmain-c/meson.build +++ b/src/libmain-c/meson.build @@ -28,6 +28,7 @@ deps_public_maybe_subproject = [ subdir('nix-meson-build-support/subprojects') subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'nix_api_main.cc', @@ -45,6 +46,7 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nixmainc', sources, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, diff --git a/src/libmain-c/nix_api_main.cc b/src/libmain-c/nix_api_main.cc index eacb804554c..0ee965dc82e 100644 --- a/src/libmain-c/nix_api_main.cc +++ b/src/libmain-c/nix_api_main.cc @@ -4,6 +4,9 @@ #include "nix_api_util_internal.h" #include "nix/main/plugin.hh" +#include "nix/main/loggers.hh" + +extern "C" { nix_err nix_init_plugins(nix_c_context * context) { @@ -14,3 +17,17 @@ nix_err nix_init_plugins(nix_c_context * context) } NIXC_CATCH_ERRS } + +nix_err nix_set_log_format(nix_c_context * context, const char * format) +{ + if (context) + context->last_err_code = NIX_OK; + if (format == nullptr) + return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Log format is null"); + try { + nix::setLogFormat(format); + } + NIXC_CATCH_ERRS +} + +} // extern "C" diff --git a/src/libmain-c/nix_api_main.h b/src/libmain-c/nix_api_main.h index 3957b992fd3..3d5d12c1559 100644 --- a/src/libmain-c/nix_api_main.h +++ b/src/libmain-c/nix_api_main.h @@ -30,6 +30,14 @@ extern "C" { */ nix_err nix_init_plugins(nix_c_context * context); +/** + * @brief Sets the log format + * + * @param[out] context Optional, stores error information + * @param[in] format The string name of the format. + */ +nix_err nix_set_log_format(nix_c_context * context, const char * format); + // cffi end #ifdef __cplusplus } diff --git a/src/libmain-c/package.nix b/src/libmain-c/package.nix index f019a917d36..17858d56f2e 100644 --- a/src/libmain-c/package.nix +++ b/src/libmain-c/package.nix @@ -17,7 +17,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-main-c"; + pname = "determinate-nix-main-c"; inherit version; workDir = ./.; diff --git a/src/libmain/include/nix/main/shared.hh b/src/libmain/include/nix/main/shared.hh index 47d08a05042..8b84ae47bd6 100644 --- a/src/libmain/include/nix/main/shared.hh +++ b/src/libmain/include/nix/main/shared.hh @@ -29,6 +29,8 @@ void parseCmdLine( const Strings & args, std::function parseArg); +std::string version(); + void printVersion(const std::string & programName); /** diff --git a/src/libmain/meson.build b/src/libmain/meson.build index 4a90d2d83b6..21bfbea3e24 100644 --- a/src/libmain/meson.build +++ b/src/libmain/meson.build @@ -53,6 +53,7 @@ config_priv_h = configure_file( ) subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'common-args.cc', @@ -77,6 +78,7 @@ this_library = library( 'nixmain', sources, config_priv_h, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, diff --git a/src/libmain/package.nix b/src/libmain/package.nix index 7b0a4dee7da..119e1f1aca5 100644 --- a/src/libmain/package.nix +++ b/src/libmain/package.nix @@ -18,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-main"; + pname = "determinate-nix-main"; inherit version; workDir = ./.; diff --git a/src/libmain/progress-bar.cc b/src/libmain/progress-bar.cc index c00f5d86b4d..edec8460de2 100644 --- a/src/libmain/progress-bar.cc +++ b/src/libmain/progress-bar.cc @@ -183,7 +183,7 @@ class ProgressBar : public Logger std::ostringstream oss; showErrorInfo(oss, ei, loggerSettings.showTrace.get()); - log(*state, ei.level, toView(oss)); + log(*state, ei.level, oss.view()); } void log(State & state, Verbosity lvl, std::string_view s) diff --git a/src/libmain/shared.cc b/src/libmain/shared.cc index 7187e972059..5de4d778d7e 100644 --- a/src/libmain/shared.cc +++ b/src/libmain/shared.cc @@ -290,9 +290,14 @@ void parseCmdLine( LegacyArgs(programName, parseArg).parseCmdline(args); } +std::string version() +{ + return fmt("(Determinate Nix %s) %s", determinateNixVersion, nixVersion); +} + void printVersion(const std::string & programName) { - std::cout << fmt("%1% (Nix) %2%", programName, nixVersion) << std::endl; + std::cout << fmt("%s %s", programName, version()) << std::endl; if (verbosity > lvlInfo) { Strings cfg; #if NIX_USE_BOEHMGC @@ -334,7 +339,7 @@ int handleExceptions(const std::string & programName, std::function fun) return e.status; } catch (UsageError & e) { logError(e.info()); - printError("Try '%1% --help' for more information.", programName); + printError("\nTry '%1% --help' for more information.", programName); return 1; } catch (BaseError & e) { logError(e.info()); diff --git a/src/libstore-c/meson.build b/src/libstore-c/meson.build index f8eaef80395..a92771efc1d 100644 --- a/src/libstore-c/meson.build +++ b/src/libstore-c/meson.build @@ -26,6 +26,7 @@ deps_public_maybe_subproject = [ subdir('nix-meson-build-support/subprojects') subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'nix_api_store.cc', @@ -46,6 +47,7 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nixstorec', sources, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, diff --git a/src/libstore-c/nix_api_store.cc b/src/libstore-c/nix_api_store.cc index 4f91f533254..180573e6416 100644 --- a/src/libstore-c/nix_api_store.cc +++ b/src/libstore-c/nix_api_store.cc @@ -10,6 +10,8 @@ #include "nix/store/globals.hh" +extern "C" { + nix_err nix_libstore_init(nix_c_context * context) { if (context) @@ -91,7 +93,7 @@ nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_cal NIXC_CATCH_ERRS } -bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path) +bool nix_store_is_valid_path(nix_c_context * context, Store * store, const StorePath * path) { if (context) context->last_err_code = NIX_OK; @@ -129,7 +131,7 @@ nix_err nix_store_realise( Store * store, StorePath * path, void * userdata, - void (*callback)(void * userdata, const char *, const char *)) + void (*callback)(void * userdata, const char *, const StorePath *)) { if (context) context->last_err_code = NIX_OK; @@ -143,9 +145,11 @@ nix_err nix_store_realise( if (callback) { for (const auto & result : results) { - for (const auto & [outputName, realisation] : result.builtOutputs) { - auto op = store->ptr->printStorePath(realisation.outPath); - callback(userdata, outputName.c_str(), op.c_str()); + if (auto * success = result.tryGetSuccess()) { + for (const auto & [outputName, realisation] : success->builtOutputs) { + StorePath p{realisation.outPath}; + callback(userdata, outputName.c_str(), &p); + } } } } @@ -164,11 +168,74 @@ void nix_store_path_free(StorePath * sp) delete sp; } +void nix_derivation_free(nix_derivation * drv) +{ + delete drv; +} + StorePath * nix_store_path_clone(const StorePath * p) { return new StorePath{p->path}; } +nix_derivation * nix_derivation_clone(const nix_derivation * d) +{ + return new nix_derivation{d->drv, d->store}; +} + +nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store, const char * json) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto drv = static_cast(nlohmann::json::parse(json)); + + auto drvPath = nix::writeDerivation(*store->ptr, drv, nix::NoRepair, /* read only */ true); + + drv.checkInvariants(*store->ptr, drvPath); + + return new nix_derivation{drv}; + } + NIXC_CATCH_ERRS_NULL +} + +nix_err nix_derivation_make_outputs( + nix_c_context * context, + Store * store, + const char * json, + void (*callback)(void * userdata, const char * output_name, const char * path), + void * userdata) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto drv = nix::Derivation::fromJSON(nlohmann::json::parse(json)); + auto hashesModulo = hashDerivationModulo(*store->ptr, drv, true); + + for (auto & output : drv.outputs) { + nix::Hash h = hashesModulo.hashes.at(output.first); + auto outPath = store->ptr->makeOutputPath(output.first, h, drv.name); + + if (callback) { + callback(userdata, output.first.c_str(), store->ptr->printStorePath(outPath).c_str()); + } + } + } + NIXC_CATCH_ERRS +} + +StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto ret = nix::writeDerivation(*store->ptr, derivation->drv, nix::NoRepair); + + return new StorePath{ret}; + } + NIXC_CATCH_ERRS_NULL +} + nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store * dstStore, StorePath * path) { if (context) @@ -180,3 +247,139 @@ nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store } NIXC_CATCH_ERRS } + +nix_err nix_store_get_fs_closure( + nix_c_context * context, + Store * store, + const StorePath * store_path, + bool flip_direction, + bool include_outputs, + bool include_derivers, + void * userdata, + void (*callback)(void * userdata, const StorePath * store_path)) +{ + if (context) + context->last_err_code = NIX_OK; + try { + const auto nixStore = store->ptr; + + nix::StorePathSet set; + nixStore->computeFSClosure(store_path->path, set, flip_direction, include_outputs, include_derivers); + + if (callback) { + for (const auto & path : set) { + const StorePath tmp{path}; + callback(userdata, &tmp); + } + } + } + NIXC_CATCH_ERRS +} + +nix_err nix_store_drv_from_path( + nix_c_context * context, + Store * store, + const StorePath * path, + void (*callback)(void * userdata, const nix_derivation * drv), + void * userdata) +{ + if (context) + context->last_err_code = NIX_OK; + try { + nix::Derivation drv = store->ptr->derivationFromPath(path->path); + if (callback) { + const nix_derivation tmp{drv, store}; + callback(userdata, &tmp); + } + } + NIXC_CATCH_ERRS +} + +nix_err nix_store_query_path_info( + nix_c_context * context, + Store * store, + const StorePath * store_path, + void * userdata, + nix_get_string_callback callback) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto info = store->ptr->queryPathInfo(store_path->path); + if (callback) { + auto result = info->toJSON(store->ptr->config, true, nix::HashFormat::Nix32).dump(); + callback(result.data(), result.size(), userdata); + } + } + NIXC_CATCH_ERRS +} + +nix_err nix_store_build_paths( + nix_c_context * context, + Store * store, + const StorePath ** store_paths, + unsigned int num_store_paths, + void (*callback)(void * userdata, const char * path, const char * result), + void * userdata) +{ + if (context) + context->last_err_code = NIX_OK; + try { + std::vector derived_paths; + for (size_t i = 0; i < num_store_paths; i++) { + const StorePath * store_path = store_paths[i]; + derived_paths.push_back(nix::SingleDerivedPath::Opaque{store_path->path}); + } + + auto results = store->ptr->buildPathsWithResults(derived_paths); + for (auto & result : results) { + if (callback) { + nlohmann::json json; + nix::to_json(json, result); + callback(userdata, result.path.to_string(store->ptr->config).c_str(), json.dump().c_str()); + } + } + } + NIXC_CATCH_ERRS +} + +nix_err nix_derivation_get_outputs_and_optpaths( + nix_c_context * context, + const nix_derivation * drv, + const Store * store, + void (*callback)(void * userdata, const char * name, const StorePath * path), + void * userdata) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto value = drv->drv.outputsAndOptPaths(store->ptr->config); + if (callback) { + for (const auto & [name, result] : value) { + if (auto store_path = result.second) { + const StorePath tmp_path{*store_path}; + callback(userdata, name.c_str(), &tmp_path); + } else { + callback(userdata, name.c_str(), nullptr); + } + } + } + } + NIXC_CATCH_ERRS +} + +nix_err nix_derivation_to_json( + nix_c_context * context, const nix_derivation * drv, nix_get_string_callback callback, void * userdata) +{ + if (context) + context->last_err_code = NIX_OK; + try { + auto result = drv->drv.toJSON().dump(); + if (callback) { + callback(result.data(), result.size(), userdata); + } + } + NIXC_CATCH_ERRS +} + +} // extern "C" diff --git a/src/libstore-c/nix_api_store.h b/src/libstore-c/nix_api_store.h index ad3d7b22a84..f59b62f9078 100644 --- a/src/libstore-c/nix_api_store.h +++ b/src/libstore-c/nix_api_store.h @@ -23,6 +23,8 @@ extern "C" { typedef struct Store Store; /** @brief Nix store path */ typedef struct StorePath StorePath; +/** @brief Nix Derivation */ +typedef struct nix_derivation nix_derivation; /** * @brief Initializes the Nix store library @@ -148,7 +150,7 @@ void nix_store_path_free(StorePath * p); * @param[in] path Path to check * @return true or false, error info in context */ -bool nix_store_is_valid_path(nix_c_context * context, Store * store, StorePath * path); +bool nix_store_is_valid_path(nix_c_context * context, Store * store, const StorePath * path); /** * @brief Get the physical location of a store path @@ -190,7 +192,7 @@ nix_err nix_store_realise( Store * store, StorePath * path, void * userdata, - void (*callback)(void * userdata, const char * outname, const char * out)); + void (*callback)(void * userdata, const char * outname, const StorePath * out)); /** * @brief get the version of a nix store. @@ -207,6 +209,56 @@ nix_err nix_store_realise( nix_err nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_callback callback, void * user_data); +/** + * @brief Create a `nix_derivation` from a JSON representation of that derivation. + * + * @param[out] context Optional, stores error information. + * @param[in] store nix store reference. + * @param[in] json JSON of the derivation as a string. + */ +nix_derivation * nix_derivation_from_json(nix_c_context * context, Store * store, const char * json); + +/** + * @brief Hashes the derivation and gives the output paths + * + * @param[in] context Optional, stores error information. + * @param[in] store nix store reference. + * @param[in] json JSON of the derivation as a string. + * @param[in] callback Called for every output to provide the output path. + * @param[in] userdata User data to pass to the callback. + */ +nix_err nix_derivation_make_outputs( + nix_c_context * context, + Store * store, + const char * json, + void (*callback)(void * userdata, const char * output_name, const char * path), + void * userdata); + +/** + * @brief Add the given `nix_derivation` to the given store + * + * @param[out] context Optional, stores error information. + * @param[in] store nix store reference. The derivation will be inserted here. + * @param[in] derivation nix_derivation to insert into the given store. + */ +StorePath * nix_add_derivation(nix_c_context * context, Store * store, nix_derivation * derivation); + +/** + * @brief Deallocate a `nix_derivation` + * + * Does not fail. + * @param[in] drv the derivation to free + */ +void nix_derivation_free(nix_derivation * drv); + +/** + * @brief Copy a `nix_derivation` + * + * @param[in] d the derivation to copy + * @return a new `nix_derivation` + */ +nix_derivation * nix_derivation_clone(const nix_derivation * d); + /** * @brief Copy the closure of `path` from `srcStore` to `dstStore`. * @@ -217,6 +269,114 @@ nix_store_get_version(nix_c_context * context, Store * store, nix_get_string_cal */ nix_err nix_store_copy_closure(nix_c_context * context, Store * srcStore, Store * dstStore, StorePath * path); +/** + * @brief Gets the closure of a specific store path + * + * @note The callback borrows each StorePath only for the duration of the call. + * + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] store_path The path to compute from + * @param[in] flip_direction + * @param[in] include_outputs + * @param[in] include_derivers + * @param[in] callback The function to call for every store path + * @param[in] userdata The userdata to pass to the callback + */ +nix_err nix_store_get_fs_closure( + nix_c_context * context, + Store * store, + const StorePath * store_path, + bool flip_direction, + bool include_outputs, + bool include_derivers, + void * userdata, + void (*callback)(void * userdata, const StorePath * store_path)); + +/** + * @brief Returns the derivation associated with the store path + * + * @note The callback borrows the Derivation only for the duration of the call. + * + * @param[out] context Optional, stores error information + * @param[in] store The nix store + * @param[in] path The nix store path + * @param[in] callback The callback to call + * @param[in] userdata The userdata to pass to the callback + */ +nix_err nix_store_drv_from_path( + nix_c_context * context, + Store * store, + const StorePath * path, + void (*callback)(void * userdata, const nix_derivation * drv), + void * userdata); + +/** + * @brief Queries for the nix store path info. + * + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] store_path A store path + * @param[in] userdata The data to pass to the callback + * @param[in] callback Called for when the path info is resolved + */ +nix_err nix_store_query_path_info( + nix_c_context * context, + Store * store, + const StorePath * store_path, + void * userdata, + nix_get_string_callback callback); + +/** + * @brief Builds the paths, if they are a derivation then they get built. + * + * @note Path and result for the callback only exist for the lifetime of + * the call. Result is a string containing the build result in JSON. + * + * @param[out] context Optional, stores error information + * @param[in] store nix store reference + * @param[in] store_paths Pointer to list of nix store paths + * @param[in] num_store_paths Number of nix store paths + * @param[in] callback The callback to trigger for build results + * @param[in] userdata User data to pass to the callback + */ +nix_err nix_store_build_paths( + nix_c_context * context, + Store * store, + const StorePath ** store_paths, + unsigned int num_store_paths, + void (*callback)(void * userdata, const char * path, const char * result), + void * userdata); + +/** + * @brief Iterate and get all of the store paths for each output. + * + * @note The callback borrows the StorePath only for the duration of the call. + * + * @param[out] context Optional, stores error information + * @param[in] drv The derivation + * @param[in] store The nix store + * @param[in] callback The function to call on every output and store path + * @param[in] userdata The userdata to pass to the callback + */ +nix_err nix_derivation_get_outputs_and_optpaths( + nix_c_context * context, + const nix_derivation * drv, + const Store * store, + void (*callback)(void * userdata, const char * name, const StorePath * path), + void * userdata); + +/** + * @brief Gets the derivation as a JSON string + * + * @param[out] context Optional, stores error information + * @param[in] drv The derivation + * @param[in] callback Called with the JSON string + * @param[in] userdata Arbitrary data passed to the callback + */ +nix_err nix_derivation_to_json( + nix_c_context * context, const nix_derivation * drv, nix_get_string_callback callback, void * userdata); + // cffi end #ifdef __cplusplus } diff --git a/src/libstore-c/nix_api_store_internal.h b/src/libstore-c/nix_api_store_internal.h index b0194bfd3ad..0199628da8a 100644 --- a/src/libstore-c/nix_api_store_internal.h +++ b/src/libstore-c/nix_api_store_internal.h @@ -1,6 +1,9 @@ #ifndef NIX_API_STORE_INTERNAL_H #define NIX_API_STORE_INTERNAL_H #include "nix/store/store-api.hh" +#include "nix/store/derivations.hh" + +extern "C" { struct Store { @@ -12,4 +15,12 @@ struct StorePath nix::StorePath path; }; +struct nix_derivation +{ + nix::Derivation drv; + Store * store; +}; + +} // extern "C" + #endif diff --git a/src/libstore-c/package.nix b/src/libstore-c/package.nix index fde17c78e01..0ce37e44c01 100644 --- a/src/libstore-c/package.nix +++ b/src/libstore-c/package.nix @@ -15,7 +15,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-store-c"; + pname = "determinate-nix-store-c"; inherit version; workDir = ./.; diff --git a/src/libstore-test-support/include/nix/store/tests/libstore.hh b/src/libstore-test-support/include/nix/store/tests/libstore.hh index 28b29fa315a..d79b5531232 100644 --- a/src/libstore-test-support/include/nix/store/tests/libstore.hh +++ b/src/libstore-test-support/include/nix/store/tests/libstore.hh @@ -19,14 +19,13 @@ public: } protected: + LibStoreTest(ref store) + : store(std::move(store)) + { + } + LibStoreTest() - : store(openStore({ - .variant = - StoreReference::Specified{ - .scheme = "dummy", - }, - .params = {}, - })) + : LibStoreTest(openStore("dummy://")) { } diff --git a/src/libstore-test-support/include/nix/store/tests/meson.build b/src/libstore-test-support/include/nix/store/tests/meson.build index f79769d4102..33524de3851 100644 --- a/src/libstore-test-support/include/nix/store/tests/meson.build +++ b/src/libstore-test-support/include/nix/store/tests/meson.build @@ -9,4 +9,5 @@ headers = files( 'outputs-spec.hh', 'path.hh', 'protocol.hh', + 'test-main.hh', ) diff --git a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh index 608aa63d65e..7ecc5603b6a 100644 --- a/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh +++ b/src/libstore-test-support/include/nix/store/tests/nix_api_store.hh @@ -12,33 +12,32 @@ #include namespace nixC { -class nix_api_store_test : public nix_api_util_context + +class nix_api_store_test_base : public nix_api_util_context { public: - nix_api_store_test() + nix_api_store_test_base() { nix_libstore_init(ctx); - init_local_store(); }; - ~nix_api_store_test() override + ~nix_api_store_test_base() override { - nix_store_free(store); - - for (auto & path : std::filesystem::recursive_directory_iterator(nixDir)) { - std::filesystem::permissions(path, std::filesystem::perms::owner_all); + if (exists(std::filesystem::path{nixDir})) { + for (auto & path : std::filesystem::recursive_directory_iterator(nixDir)) { + std::filesystem::permissions(path, std::filesystem::perms::owner_all); + } + std::filesystem::remove_all(nixDir); } - std::filesystem::remove_all(nixDir); } - Store * store; std::string nixDir; std::string nixStoreDir; std::string nixStateDir; std::string nixLogDir; protected: - void init_local_store() + Store * open_local_store() { #ifdef _WIN32 // no `mkdtemp` with MinGW @@ -66,11 +65,37 @@ protected: const char ** params[] = {p1, p2, p3, nullptr}; - store = nix_store_open(ctx, "local", params); + auto * store = nix_store_open(ctx, "local", params); if (!store) { std::string errMsg = nix_err_msg(nullptr, ctx, nullptr); - ASSERT_NE(store, nullptr) << "Could not open store: " << errMsg; + EXPECT_NE(store, nullptr) << "Could not open store: " << errMsg; + assert(store); }; + return store; } }; + +class nix_api_store_test : public nix_api_store_test_base +{ +public: + nix_api_store_test() + : nix_api_store_test_base{} + { + init_local_store(); + }; + + ~nix_api_store_test() override + { + nix_store_free(store); + } + + Store * store; + +protected: + void init_local_store() + { + store = open_local_store(); + } +}; + } // namespace nixC diff --git a/src/libstore-test-support/include/nix/store/tests/test-main.hh b/src/libstore-test-support/include/nix/store/tests/test-main.hh new file mode 100644 index 00000000000..3a1897469de --- /dev/null +++ b/src/libstore-test-support/include/nix/store/tests/test-main.hh @@ -0,0 +1,13 @@ +#pragma once + +///@file + +namespace nix { + +/** + * Call this for a GTest test suite that will including performing Nix + * builds, before running tests. + */ +int testMainForBuidingPre(int argc, char ** argv); + +} // namespace nix diff --git a/src/libstore-test-support/meson.build b/src/libstore-test-support/meson.build index b2977941f86..e929ae2b499 100644 --- a/src/libstore-test-support/meson.build +++ b/src/libstore-test-support/meson.build @@ -29,11 +29,13 @@ rapidcheck = dependency('rapidcheck') deps_public += rapidcheck subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'derived-path.cc', 'outputs-spec.cc', 'path.cc', + 'test-main.cc', ) subdir('include/nix/store/tests') @@ -44,6 +46,7 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-store-test-support', sources, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 diff --git a/src/libstore-test-support/package.nix b/src/libstore-test-support/package.nix index 391ddeefda2..2561dd791eb 100644 --- a/src/libstore-test-support/package.nix +++ b/src/libstore-test-support/package.nix @@ -18,7 +18,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-store-test-support"; + pname = "determinate-nix-store-test-support"; inherit version; workDir = ./.; diff --git a/src/libstore-test-support/test-main.cc b/src/libstore-test-support/test-main.cc new file mode 100644 index 00000000000..0b9072dc08f --- /dev/null +++ b/src/libstore-test-support/test-main.cc @@ -0,0 +1,47 @@ +#include + +#include "nix/store/globals.hh" +#include "nix/util/logging.hh" + +#include "nix/store/tests/test-main.hh" + +namespace nix { + +int testMainForBuidingPre(int argc, char ** argv) +{ + if (argc > 1 && std::string_view(argv[1]) == "__build-remote") { + printError("test-build-remote: not supported in libexpr unit tests"); + return EXIT_FAILURE; + } + + // Disable build hook. We won't be testing remote builds in these unit tests. If we do, fix the above build hook. + settings.buildHook = {}; + + // No substituters, unless a test specifically requests. + settings.substituters = {}; + +#ifdef __linux__ // should match the conditional around sandboxBuildDir declaration. + + // When building and testing nix within the host's Nix sandbox, our store dir will be located in the host's + // sandboxBuildDir, e.g.: Host + // storeDir = /nix/store + // sandboxBuildDir = /build + // This process + // storeDir = /build/foo/bar/store + // sandboxBuildDir = /build + // However, we have a rule that the store dir must not be inside the storeDir, so we need to pick a different + // sandboxBuildDir. + settings.sandboxBuildDir = "/test-build-dir-instead-of-usual-build-dir"; +#endif + +#ifdef __APPLE__ + // Avoid this error, when already running in a sandbox: + // sandbox-exec: sandbox_apply: Operation not permitted + settings.sandboxMode = smDisabled; + setEnv("_NIX_TEST_NO_SANDBOX", "1"); +#endif + + return EXIT_SUCCESS; +} + +} // namespace nix diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json index bc67236b54f..eb4bd4f3de6 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-defaults.json @@ -21,5 +21,6 @@ "method": "nar" } }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json index 183148b29b3..3a4a3079b45 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs-defaults.json @@ -32,5 +32,6 @@ ], "system": "my-system" }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json index ec044d77877..b10355af711 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes-structured-attrs.json @@ -10,14 +10,14 @@ "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9" }, "inputDrvs": { - "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { "dynamicOutputs": {}, "outputs": [ "dev", "out" ] }, - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { "dynamicOutputs": {}, "outputs": [ "dev", @@ -26,7 +26,7 @@ } }, "inputSrcs": [ - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" ], "name": "advanced-attributes-structured-attrs", "outputs": { @@ -100,5 +100,6 @@ ], "system": "my-system" }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ca/advanced-attributes.json b/src/libstore-tests/data/derivation/ca/advanced-attributes.json index 0ac0a9c5c1c..d6688203660 100644 --- a/src/libstore-tests/data/derivation/ca/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ca/advanced-attributes.json @@ -26,14 +26,14 @@ "system": "my-system" }, "inputDrvs": { - "/nix/store/j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { + "j56sf12rxpcv5swr14vsjn5cwm6bj03h-foo.drv": { "dynamicOutputs": {}, "outputs": [ "dev", "out" ] }, - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv": { "dynamicOutputs": {}, "outputs": [ "dev", @@ -42,7 +42,7 @@ } }, "inputSrcs": [ - "/nix/store/qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" + "qnml92yh97a6fbrs2m5qg5cqlc8vni58-bar.drv" ], "name": "advanced-attributes", "outputs": { @@ -51,5 +51,6 @@ "method": "nar" } }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ca/self-contained.json b/src/libstore-tests/data/derivation/ca/self-contained.json new file mode 100644 index 00000000000..331beb7be26 --- /dev/null +++ b/src/libstore-tests/data/derivation/ca/self-contained.json @@ -0,0 +1,24 @@ +{ + "args": [ + "-c", + "echo $name foo > $out" + ], + "builder": "/bin/sh", + "env": { + "builder": "/bin/sh", + "name": "myname", + "out": "/1rz4g4znpzjwh1xymhjpm42vipw92pr73vdgl6xs1hycac8kf2n9", + "system": "x86_64-linux" + }, + "inputDrvs": {}, + "inputSrcs": [], + "name": "myname", + "outputs": { + "out": { + "hashAlgo": "sha256", + "method": "nar" + } + }, + "system": "x86_64-linux", + "version": 3 +} diff --git a/src/libstore-tests/data/derivation/dynDerivationDeps.drv b/src/libstore-tests/data/derivation/dyn-dep-derivation.drv similarity index 100% rename from src/libstore-tests/data/derivation/dynDerivationDeps.drv rename to src/libstore-tests/data/derivation/dyn-dep-derivation.drv diff --git a/src/libstore-tests/data/derivation/dynDerivationDeps.json b/src/libstore-tests/data/derivation/dyn-dep-derivation.json similarity index 78% rename from src/libstore-tests/data/derivation/dynDerivationDeps.json rename to src/libstore-tests/data/derivation/dyn-dep-derivation.json index 9dbeb1f15af..1a9f54c5304 100644 --- a/src/libstore-tests/data/derivation/dynDerivationDeps.json +++ b/src/libstore-tests/data/derivation/dyn-dep-derivation.json @@ -8,7 +8,7 @@ "BIG_BAD": "WOLF" }, "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { "dynamicOutputs": { "cat": { "dynamicOutputs": {}, @@ -30,9 +30,10 @@ } }, "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], "name": "dyn-dep-derivation", "outputs": {}, - "system": "wasm-sel4" + "system": "wasm-sel4", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json index d58e7d5b586..0fa543f214a 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-defaults.json @@ -15,8 +15,9 @@ "name": "advanced-attributes-defaults", "outputs": { "out": { - "path": "/nix/store/1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults" + "path": "1qsc7svv43m4dw2prh6mvyf7cai5czji-advanced-attributes-defaults" } }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json index f5349e6c311..e02392ea131 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs-defaults.json @@ -13,10 +13,10 @@ "name": "advanced-attributes-structured-attrs-defaults", "outputs": { "dev": { - "path": "/nix/store/8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev" + "path": "8bazivnbipbyi569623skw5zm91z6kc2-advanced-attributes-structured-attrs-defaults-dev" }, "out": { - "path": "/nix/store/f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" + "path": "f8f8nvnx32bxvyxyx2ff7akbvwhwd9dw-advanced-attributes-structured-attrs-defaults" } }, "structuredAttrs": { @@ -28,5 +28,6 @@ ], "system": "my-system" }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json index b8d56646275..9230b06b629 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes-structured-attrs.json @@ -10,14 +10,14 @@ "out": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" }, "inputDrvs": { - "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { "dynamicOutputs": {}, "outputs": [ "dev", "out" ] }, - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { "dynamicOutputs": {}, "outputs": [ "dev", @@ -26,18 +26,18 @@ } }, "inputSrcs": [ - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" ], "name": "advanced-attributes-structured-attrs", "outputs": { "bin": { - "path": "/nix/store/33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin" + "path": "33qms3h55wlaspzba3brlzlrm8m2239g-advanced-attributes-structured-attrs-bin" }, "dev": { - "path": "/nix/store/wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev" + "path": "wyfgwsdi8rs851wmy1xfzdxy7y5vrg5l-advanced-attributes-structured-attrs-dev" }, "out": { - "path": "/nix/store/7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" + "path": "7cxy4zx1vqc885r4jl2l64pymqbdmhii-advanced-attributes-structured-attrs" } }, "structuredAttrs": { @@ -95,5 +95,6 @@ ], "system": "my-system" }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/ia/advanced-attributes.json b/src/libstore-tests/data/derivation/ia/advanced-attributes.json index 20ce5e1c2bb..ba5911c911a 100644 --- a/src/libstore-tests/data/derivation/ia/advanced-attributes.json +++ b/src/libstore-tests/data/derivation/ia/advanced-attributes.json @@ -24,14 +24,14 @@ "system": "my-system" }, "inputDrvs": { - "/nix/store/afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { + "afc3vbjbzql750v2lp8gxgaxsajphzih-foo.drv": { "dynamicOutputs": {}, "outputs": [ "dev", "out" ] }, - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv": { "dynamicOutputs": {}, "outputs": [ "dev", @@ -40,13 +40,14 @@ } }, "inputSrcs": [ - "/nix/store/vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" + "vj2i49jm2868j2fmqvxm70vlzmzvgv14-bar.drv" ], "name": "advanced-attributes", "outputs": { "out": { - "path": "/nix/store/wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes" + "path": "wyhpwd748pns4k7svh48wdrc8kvjk0ra-advanced-attributes" } }, - "system": "my-system" + "system": "my-system", + "version": 3 } diff --git a/src/libstore-tests/data/derivation/output-caFixedFlat.json b/src/libstore-tests/data/derivation/output-caFixedFlat.json index 7001ea0a9fb..e6a0123f65c 100644 --- a/src/libstore-tests/data/derivation/output-caFixedFlat.json +++ b/src/libstore-tests/data/derivation/output-caFixedFlat.json @@ -1,6 +1,5 @@ { "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hashAlgo": "sha256", - "method": "flat", - "path": "/nix/store/rhcg9h16sqvlbpsa6dqm57sbr2al6nzg-drv-name-output-name" + "method": "flat" } diff --git a/src/libstore-tests/data/derivation/output-caFixedNAR.json b/src/libstore-tests/data/derivation/output-caFixedNAR.json index 54eb306e672..b57e065a934 100644 --- a/src/libstore-tests/data/derivation/output-caFixedNAR.json +++ b/src/libstore-tests/data/derivation/output-caFixedNAR.json @@ -1,6 +1,5 @@ { "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hashAlgo": "sha256", - "method": "nar", - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + "method": "nar" } diff --git a/src/libstore-tests/data/derivation/output-caFixedText.json b/src/libstore-tests/data/derivation/output-caFixedText.json index e8a65186049..84778509ee2 100644 --- a/src/libstore-tests/data/derivation/output-caFixedText.json +++ b/src/libstore-tests/data/derivation/output-caFixedText.json @@ -1,6 +1,5 @@ { "hash": "894517c9163c896ec31a2adbd33c0681fd5f45b2c0ef08a64c92a03fb97f390f", "hashAlgo": "sha256", - "method": "text", - "path": "/nix/store/6s1zwabh956jvhv4w9xcdb5jiyanyxg1-drv-name-output-name" + "method": "text" } diff --git a/src/libstore-tests/data/derivation/output-inputAddressed.json b/src/libstore-tests/data/derivation/output-inputAddressed.json index 86c7f3a05ce..04491ffdec3 100644 --- a/src/libstore-tests/data/derivation/output-inputAddressed.json +++ b/src/libstore-tests/data/derivation/output-inputAddressed.json @@ -1,3 +1,3 @@ { - "path": "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" + "path": "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name" } diff --git a/src/libstore-tests/data/derivation/simple.drv b/src/libstore-tests/data/derivation/simple-derivation.drv similarity index 100% rename from src/libstore-tests/data/derivation/simple.drv rename to src/libstore-tests/data/derivation/simple-derivation.drv diff --git a/src/libstore-tests/data/derivation/simple.json b/src/libstore-tests/data/derivation/simple-derivation.json similarity index 66% rename from src/libstore-tests/data/derivation/simple.json rename to src/libstore-tests/data/derivation/simple-derivation.json index 20d0f8933e6..41a049aef77 100644 --- a/src/libstore-tests/data/derivation/simple.json +++ b/src/libstore-tests/data/derivation/simple-derivation.json @@ -8,7 +8,7 @@ "BIG_BAD": "WOLF" }, "inputDrvs": { - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv": { "dynamicOutputs": {}, "outputs": [ "cat", @@ -17,9 +17,10 @@ } }, "inputSrcs": [ - "/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" + "c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1" ], "name": "simple-derivation", "outputs": {}, - "system": "wasm-sel4" + "system": "wasm-sel4", + "version": 3 } diff --git a/src/libstore-tests/data/derived-path/multi_built_built.json b/src/libstore-tests/data/derived-path/multi_built_built.json new file mode 100644 index 00000000000..561d0485092 --- /dev/null +++ b/src/libstore-tests/data/derived-path/multi_built_built.json @@ -0,0 +1,10 @@ +{ + "drvPath": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "output": "bar" + }, + "outputs": [ + "baz", + "quux" + ] +} diff --git a/src/libstore-tests/data/derived-path/multi_built_built_wildcard.json b/src/libstore-tests/data/derived-path/multi_built_built_wildcard.json new file mode 100644 index 00000000000..da1f9d996ac --- /dev/null +++ b/src/libstore-tests/data/derived-path/multi_built_built_wildcard.json @@ -0,0 +1,9 @@ +{ + "drvPath": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "output": "bar" + }, + "outputs": [ + "*" + ] +} diff --git a/src/libstore-tests/data/derived-path/multi_opaque.json b/src/libstore-tests/data/derived-path/multi_opaque.json new file mode 100644 index 00000000000..9bedb882bca --- /dev/null +++ b/src/libstore-tests/data/derived-path/multi_opaque.json @@ -0,0 +1 @@ +"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" diff --git a/src/libstore-tests/data/derived-path/mutli_built.json b/src/libstore-tests/data/derived-path/mutli_built.json new file mode 100644 index 00000000000..d7bcff53d49 --- /dev/null +++ b/src/libstore-tests/data/derived-path/mutli_built.json @@ -0,0 +1,7 @@ +{ + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "outputs": [ + "bar", + "baz" + ] +} diff --git a/src/libstore-tests/data/derived-path/single_built.json b/src/libstore-tests/data/derived-path/single_built.json new file mode 100644 index 00000000000..64110a364eb --- /dev/null +++ b/src/libstore-tests/data/derived-path/single_built.json @@ -0,0 +1,4 @@ +{ + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "output": "bar" +} diff --git a/src/libstore-tests/data/derived-path/single_built_built.json b/src/libstore-tests/data/derived-path/single_built_built.json new file mode 100644 index 00000000000..66faa668cd6 --- /dev/null +++ b/src/libstore-tests/data/derived-path/single_built_built.json @@ -0,0 +1,7 @@ +{ + "drvPath": { + "drvPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "output": "bar" + }, + "output": "baz" +} diff --git a/src/libstore-tests/data/derived-path/single_opaque.json b/src/libstore-tests/data/derived-path/single_opaque.json new file mode 100644 index 00000000000..9bedb882bca --- /dev/null +++ b/src/libstore-tests/data/derived-path/single_opaque.json @@ -0,0 +1 @@ +"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" diff --git a/src/libstore-tests/data/outputs-spec/all.json b/src/libstore-tests/data/outputs-spec/all.json new file mode 100644 index 00000000000..1449203e9ff --- /dev/null +++ b/src/libstore-tests/data/outputs-spec/all.json @@ -0,0 +1,3 @@ +[ + "*" +] diff --git a/src/libstore-tests/data/outputs-spec/extended/all.json b/src/libstore-tests/data/outputs-spec/extended/all.json new file mode 100644 index 00000000000..1449203e9ff --- /dev/null +++ b/src/libstore-tests/data/outputs-spec/extended/all.json @@ -0,0 +1,3 @@ +[ + "*" +] diff --git a/src/libstore-tests/data/outputs-spec/extended/def.json b/src/libstore-tests/data/outputs-spec/extended/def.json new file mode 100644 index 00000000000..19765bd501b --- /dev/null +++ b/src/libstore-tests/data/outputs-spec/extended/def.json @@ -0,0 +1 @@ +null diff --git a/src/libstore-tests/data/outputs-spec/extended/name.json b/src/libstore-tests/data/outputs-spec/extended/name.json new file mode 100644 index 00000000000..0ede90fb485 --- /dev/null +++ b/src/libstore-tests/data/outputs-spec/extended/name.json @@ -0,0 +1,3 @@ +[ + "a" +] diff --git a/src/libstore-tests/data/outputs-spec/extended/names.json b/src/libstore-tests/data/outputs-spec/extended/names.json new file mode 100644 index 00000000000..517c9d68edb --- /dev/null +++ b/src/libstore-tests/data/outputs-spec/extended/names.json @@ -0,0 +1,4 @@ +[ + "a", + "b" +] diff --git a/src/libstore-tests/data/outputs-spec/name.json b/src/libstore-tests/data/outputs-spec/name.json new file mode 100644 index 00000000000..0ede90fb485 --- /dev/null +++ b/src/libstore-tests/data/outputs-spec/name.json @@ -0,0 +1,3 @@ +[ + "a" +] diff --git a/src/libstore-tests/data/outputs-spec/names.json b/src/libstore-tests/data/outputs-spec/names.json new file mode 100644 index 00000000000..517c9d68edb --- /dev/null +++ b/src/libstore-tests/data/outputs-spec/names.json @@ -0,0 +1,4 @@ +[ + "a", + "b" +] diff --git a/src/libstore-tests/data/realisation/simple.json b/src/libstore-tests/data/realisation/simple.json new file mode 100644 index 00000000000..2ccb1e72119 --- /dev/null +++ b/src/libstore-tests/data/realisation/simple.json @@ -0,0 +1,6 @@ +{ + "dependentRealisations": {}, + "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [] +} diff --git a/src/libstore-tests/data/realisation/with-dependent-realisations.json b/src/libstore-tests/data/realisation/with-dependent-realisations.json new file mode 100644 index 00000000000..a58e0d7fe1c --- /dev/null +++ b/src/libstore-tests/data/realisation/with-dependent-realisations.json @@ -0,0 +1,8 @@ +{ + "dependentRealisations": { + "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" + }, + "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [] +} diff --git a/src/libstore-tests/data/realisation/with-signature.json b/src/libstore-tests/data/realisation/with-signature.json new file mode 100644 index 00000000000..a28848cb02b --- /dev/null +++ b/src/libstore-tests/data/realisation/with-signature.json @@ -0,0 +1,8 @@ +{ + "dependentRealisations": {}, + "id": "sha256:ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad!foo", + "outPath": "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv", + "signatures": [ + "asdfasdfasdf" + ] +} diff --git a/src/libstore-tests/data/serve-protocol/handshake-to-client.bin b/src/libstore-tests/data/serve-protocol/handshake-to-client.bin index 15ba4b5e3d9..465daa532c4 100644 Binary files a/src/libstore-tests/data/serve-protocol/handshake-to-client.bin and b/src/libstore-tests/data/serve-protocol/handshake-to-client.bin differ diff --git a/src/libstore-tests/data/store-path/simple.json b/src/libstore-tests/data/store-path/simple.json new file mode 100644 index 00000000000..9bedb882bca --- /dev/null +++ b/src/libstore-tests/data/store-path/simple.json @@ -0,0 +1 @@ +"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv" diff --git a/src/libstore-tests/data/store-reference/daemon_shorthand.txt b/src/libstore-tests/data/store-reference/daemon_shorthand.txt new file mode 100644 index 00000000000..bd8c0f8c41e --- /dev/null +++ b/src/libstore-tests/data/store-reference/daemon_shorthand.txt @@ -0,0 +1 @@ +daemon \ No newline at end of file diff --git a/src/libstore-tests/data/store-reference/local_shorthand_3.txt b/src/libstore-tests/data/store-reference/local_shorthand_3.txt new file mode 100644 index 00000000000..c2c027fec1a --- /dev/null +++ b/src/libstore-tests/data/store-reference/local_shorthand_3.txt @@ -0,0 +1 @@ +local \ No newline at end of file diff --git a/src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_1.txt b/src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_1.txt new file mode 100644 index 00000000000..861b5bb3515 --- /dev/null +++ b/src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_1.txt @@ -0,0 +1 @@ +ssh://::1 \ No newline at end of file diff --git a/src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_2.txt b/src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_2.txt new file mode 100644 index 00000000000..952d5a55d31 --- /dev/null +++ b/src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_2.txt @@ -0,0 +1 @@ +ssh://userinfo@fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e \ No newline at end of file diff --git a/src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_3.txt b/src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_3.txt new file mode 100644 index 00000000000..d1f17adac15 --- /dev/null +++ b/src/libstore-tests/data/store-reference/ssh_unbracketed_ipv6_3.txt @@ -0,0 +1 @@ +ssh://userinfo@fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e?a=b&c=d \ No newline at end of file diff --git a/src/libstore-tests/derivation-advanced-attrs.cc b/src/libstore-tests/derivation-advanced-attrs.cc index 37b422421a0..9c13bf04830 100644 --- a/src/libstore-tests/derivation-advanced-attrs.cc +++ b/src/libstore-tests/derivation-advanced-attrs.cc @@ -51,45 +51,44 @@ using BothFixtures = ::testing::TypesreadTest(NAME ".json", [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - /* Use DRV file instead of C++ literal as source of truth. */ \ - auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ - auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \ - Derivation got = Derivation::fromJSON(*this->store, encoded, this->mockXpSettings); \ - EXPECT_EQ(got, expected); \ - }); \ - } \ - \ - TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \ - { \ - this->writeTest( \ - NAME ".json", \ - [&]() -> json { \ - /* Use DRV file instead of C++ literal as source of truth. */ \ - auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ - return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings) \ - .toJSON(*this->store); \ - }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ - } \ - \ - TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \ - { \ - this->readTest(NAME ".drv", [&](auto encoded) { \ - /* Use JSON file instead of C++ literal as source of truth. */ \ - auto json = json::parse(readFile(this->goldenMaster(NAME ".json"))); \ - auto expected = Derivation::fromJSON(*this->store, json, this->mockXpSettings); \ - auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \ - EXPECT_EQ(got.toJSON(*this->store), expected.toJSON(*this->store)); \ - EXPECT_EQ(got, expected); \ - }); \ - } \ - \ +#define TEST_ATERM_JSON(STEM, NAME) \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_json) \ + { \ + this->readTest(NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ + auto expected = parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings); \ + Derivation got = Derivation::fromJSON(encoded, this->mockXpSettings); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_to_json) \ + { \ + this->writeTest( \ + NAME ".json", \ + [&]() -> json { \ + /* Use DRV file instead of C++ literal as source of truth. */ \ + auto aterm = readFile(this->goldenMaster(NAME ".drv")); \ + return parseDerivation(*this->store, std::move(aterm), NAME, this->mockXpSettings).toJSON(); \ + }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } \ + \ + TYPED_TEST(DerivationAdvancedAttrsBothTest, Derivation_##STEM##_from_aterm) \ + { \ + this->readTest(NAME ".drv", [&](auto encoded) { \ + /* Use JSON file instead of C++ literal as source of truth. */ \ + auto json = json::parse(readFile(this->goldenMaster(NAME ".json"))); \ + auto expected = Derivation::fromJSON(json, this->mockXpSettings); \ + auto got = parseDerivation(*this->store, std::move(encoded), NAME, this->mockXpSettings); \ + EXPECT_EQ(got.toJSON(), expected.toJSON()); \ + EXPECT_EQ(got, expected); \ + }); \ + } \ + \ /* No corresponding write test, because we need to read the drv to write the json file */ TEST_ATERM_JSON(advancedAttributes, "advanced-attributes-defaults"); diff --git a/src/libstore-tests/derivation-parser-bench.cc b/src/libstore-tests/derivation-parser-bench.cc index ef698b20555..1709eed1cdb 100644 --- a/src/libstore-tests/derivation-parser-bench.cc +++ b/src/libstore-tests/derivation-parser-bench.cc @@ -28,12 +28,33 @@ static void BM_ParseRealDerivationFile(benchmark::State & state, const std::stri state.SetBytesProcessed(state.iterations() * content.size()); } +// Benchmark unparsing real derivation files +static void BM_UnparseRealDerivationFile(benchmark::State & state, const std::string & filename) +{ + // Read the file once + std::ifstream file(filename); + std::stringstream buffer; + buffer << file.rdbuf(); + std::string content = buffer.str(); + + auto store = openStore("dummy://"); + ExperimentalFeatureSettings xpSettings; + auto drv = parseDerivation(*store, std::string(content), "test", xpSettings); + + for (auto _ : state) { + auto unparsed = drv.unparse(*store, /*maskOutputs=*/false); + benchmark::DoNotOptimize(unparsed); + assert(unparsed.size() == content.size()); + } + state.SetBytesProcessed(state.iterations() * content.size()); +} + // Register benchmarks for actual test derivation files if they exist BENCHMARK_CAPTURE( - BM_ParseRealDerivationFile, - hello, - getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value_or(NIX_UNIT_TEST_DATA) + "/derivation/hello.drv"); + BM_ParseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv"); +BENCHMARK_CAPTURE( + BM_ParseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv"); +BENCHMARK_CAPTURE( + BM_UnparseRealDerivationFile, hello, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/hello.drv"); BENCHMARK_CAPTURE( - BM_ParseRealDerivationFile, - firefox, - getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value_or(NIX_UNIT_TEST_DATA) + "/derivation/firefox.drv"); + BM_UnparseRealDerivationFile, firefox, getEnvNonEmpty("_NIX_TEST_UNIT_DATA").value() + "/derivation/firefox.drv"); diff --git a/src/libstore-tests/derivation.cc b/src/libstore-tests/derivation.cc index 812e1d01b58..65a5d011d70 100644 --- a/src/libstore-tests/derivation.cc +++ b/src/libstore-tests/derivation.cc @@ -5,13 +5,13 @@ #include "nix/store/derivations.hh" #include "nix/store/tests/libstore.hh" -#include "nix/util/tests/characterization.hh" +#include "nix/util/tests/json-characterization.hh" namespace nix { using nlohmann::json; -class DerivationTest : public CharacterizationTest, public LibStoreTest +class DerivationTest : public virtual CharacterizationTest, public LibStoreTest { std::filesystem::path unitTestData = getUnitTestData() / "derivation"; @@ -66,146 +66,183 @@ TEST_F(DynDerivationTest, BadATerm_oldVersionDynDeps) FormatError); } -#define TEST_JSON(FIXTURE, NAME, VAL, DRV_NAME, OUTPUT_NAME) \ - TEST_F(FIXTURE, DerivationOutput_##NAME##_from_json) \ - { \ - readTest("output-" #NAME ".json", [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - DerivationOutput got = DerivationOutput::fromJSON(*store, DRV_NAME, OUTPUT_NAME, encoded, mockXpSettings); \ - DerivationOutput expected{VAL}; \ - ASSERT_EQ(got, expected); \ - }); \ - } \ - \ - TEST_F(FIXTURE, DerivationOutput_##NAME##_to_json) \ - { \ - writeTest( \ - "output-" #NAME ".json", \ - [&]() -> json { return DerivationOutput{(VAL)}.toJSON(*store, (DRV_NAME), (OUTPUT_NAME)); }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ +#define MAKE_OUTPUT_JSON_TEST_P(FIXTURE) \ + TEST_P(FIXTURE, from_json) \ + { \ + const auto & [name, expected] = GetParam(); \ + /* Don't use readJsonTest because we want to check experimental \ + features. */ \ + readTest(Path{"output-"} + name + ".json", [&](const auto & encoded_) { \ + json j = json::parse(encoded_); \ + DerivationOutput got = DerivationOutput::fromJSON(j, mockXpSettings); \ + ASSERT_EQ(got, expected); \ + }); \ + } \ + \ + TEST_P(FIXTURE, to_json) \ + { \ + const auto & [name, value] = GetParam(); \ + writeJsonTest("output-" + name, value); \ } -TEST_JSON( - DerivationTest, - inputAddressed, - (DerivationOutput::InputAddressed{ - .path = store->parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"), - }), - "drv-name", - "output-name") - -TEST_JSON( - DerivationTest, - caFixedFlat, - (DerivationOutput::CAFixed{ - .ca = - { - .method = ContentAddressMethod::Raw::Flat, - .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), - }, - }), - "drv-name", - "output-name") - -TEST_JSON( - DerivationTest, - caFixedNAR, - (DerivationOutput::CAFixed{ - .ca = - { +struct DerivationOutputJsonTest : DerivationTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +MAKE_OUTPUT_JSON_TEST_P(DerivationOutputJsonTest) + +INSTANTIATE_TEST_SUITE_P( + DerivationOutputJSON, + DerivationOutputJsonTest, + ::testing::Values( + std::pair{ + "inputAddressed", + DerivationOutput{DerivationOutput::InputAddressed{ + .path = StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-drv-name-output-name"}, + }}, + }, + std::pair{ + "caFixedFlat", + DerivationOutput{DerivationOutput::CAFixed{ + .ca = + { + .method = ContentAddressMethod::Raw::Flat, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }}, + }, + std::pair{ + "caFixedNAR", + DerivationOutput{DerivationOutput::CAFixed{ + .ca = + { + .method = ContentAddressMethod::Raw::NixArchive, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }}, + }, + std::pair{ + "deferred", + DerivationOutput{DerivationOutput::Deferred{}}, + })); + +struct DynDerivationOutputJsonTest : DynDerivationTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +MAKE_OUTPUT_JSON_TEST_P(DynDerivationOutputJsonTest); + +INSTANTIATE_TEST_SUITE_P( + DynDerivationOutputJSON, + DynDerivationOutputJsonTest, + ::testing::Values( + std::pair{ + "caFixedText", + DerivationOutput{DerivationOutput::CAFixed{ + .ca = + { + .method = ContentAddressMethod::Raw::Text, + .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), + }, + }}, + })); + +struct CaDerivationOutputJsonTest : CaDerivationTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +MAKE_OUTPUT_JSON_TEST_P(CaDerivationOutputJsonTest); + +INSTANTIATE_TEST_SUITE_P( + CaDerivationOutputJSON, + CaDerivationOutputJsonTest, + ::testing::Values( + std::pair{ + "caFloating", + DerivationOutput{DerivationOutput::CAFloating{ .method = ContentAddressMethod::Raw::NixArchive, - .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), - }, - }), - "drv-name", - "output-name") - -TEST_JSON( - DynDerivationTest, - caFixedText, - (DerivationOutput::CAFixed{ - .ca = - { - .method = ContentAddressMethod::Raw::Text, - .hash = Hash::parseAnyPrefixed("sha256-iUUXyRY8iW7DGirb0zwGgf1fRbLA7wimTJKgP7l/OQ8="), - }, - }), - "drv-name", - "output-name") - -TEST_JSON( - CaDerivationTest, - caFloating, - (DerivationOutput::CAFloating{ - .method = ContentAddressMethod::Raw::NixArchive, - .hashAlgo = HashAlgorithm::SHA256, - }), - "drv-name", - "output-name") - -TEST_JSON(DerivationTest, deferred, DerivationOutput::Deferred{}, "drv-name", "output-name") - -TEST_JSON( - ImpureDerivationTest, - impure, - (DerivationOutput::Impure{ - .method = ContentAddressMethod::Raw::NixArchive, - .hashAlgo = HashAlgorithm::SHA256, - }), - "drv-name", - "output-name") - -#undef TEST_JSON - -#define TEST_JSON(FIXTURE, NAME, VAL) \ - TEST_F(FIXTURE, Derivation_##NAME##_from_json) \ - { \ - readTest(#NAME ".json", [&](const auto & encoded_) { \ - auto encoded = json::parse(encoded_); \ - Derivation expected{VAL}; \ - Derivation got = Derivation::fromJSON(*store, encoded, mockXpSettings); \ - ASSERT_EQ(got, expected); \ - }); \ - } \ - \ - TEST_F(FIXTURE, Derivation_##NAME##_to_json) \ - { \ - writeTest( \ - #NAME ".json", \ - [&]() -> json { return Derivation{VAL}.toJSON(*store); }, \ - [](const auto & file) { return json::parse(readFile(file)); }, \ - [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ - } + .hashAlgo = HashAlgorithm::SHA256, + }}, + })); + +struct ImpureDerivationOutputJsonTest : ImpureDerivationTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; -#define TEST_ATERM(FIXTURE, NAME, VAL, DRV_NAME) \ - TEST_F(FIXTURE, Derivation_##NAME##_from_aterm) \ - { \ - readTest(#NAME ".drv", [&](auto encoded) { \ - Derivation expected{VAL}; \ - auto got = parseDerivation(*store, std::move(encoded), DRV_NAME, mockXpSettings); \ - ASSERT_EQ(got.toJSON(*store), expected.toJSON(*store)); \ - ASSERT_EQ(got, expected); \ - }); \ - } \ - \ - TEST_F(FIXTURE, Derivation_##NAME##_to_aterm) \ - { \ - writeTest(#NAME ".drv", [&]() -> std::string { return (VAL).unparse(*store, false); }); \ +MAKE_OUTPUT_JSON_TEST_P(ImpureDerivationOutputJsonTest); + +INSTANTIATE_TEST_SUITE_P( + ImpureDerivationOutputJSON, + ImpureDerivationOutputJsonTest, + ::testing::Values( + std::pair{ + "impure", + DerivationOutput{DerivationOutput::Impure{ + .method = ContentAddressMethod::Raw::NixArchive, + .hashAlgo = HashAlgorithm::SHA256, + }}, + })); + +#undef MAKE_OUTPUT_JSON_TEST_P + +#define MAKE_TEST_P(FIXTURE) \ + TEST_P(FIXTURE, from_json) \ + { \ + const auto & drv = GetParam(); \ + /* Don't use readJsonTest because we want to check experimental \ + features. */ \ + readTest(drv.name + ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + Derivation got = Derivation::fromJSON(encoded, mockXpSettings); \ + ASSERT_EQ(got, drv); \ + }); \ + } \ + \ + TEST_P(FIXTURE, to_json) \ + { \ + const auto & drv = GetParam(); \ + writeJsonTest(drv.name, drv); \ + } \ + \ + TEST_P(FIXTURE, from_aterm) \ + { \ + const auto & drv = GetParam(); \ + readTest(drv.name + ".drv", [&](auto encoded) { \ + auto got = parseDerivation(*store, std::move(encoded), drv.name, mockXpSettings); \ + ASSERT_EQ(got.toJSON(), drv.toJSON()); \ + ASSERT_EQ(got, drv); \ + }); \ + } \ + \ + TEST_P(FIXTURE, to_aterm) \ + { \ + const auto & drv = GetParam(); \ + writeTest(drv.name + ".drv", [&]() -> std::string { return drv.unparse(*store, false); }); \ } -Derivation makeSimpleDrv(const Store & store) +struct DerivationJsonAtermTest : DerivationTest, + JsonCharacterizationTest, + ::testing::WithParamInterface +{}; + +MAKE_TEST_P(DerivationJsonAtermTest); + +Derivation makeSimpleDrv() { Derivation drv; drv.name = "simple-derivation"; drv.inputSrcs = { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), }; drv.inputDrvs = { .map = { { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + StorePath("c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), { .value = { @@ -231,22 +268,27 @@ Derivation makeSimpleDrv(const Store & store) return drv; } -TEST_JSON(DerivationTest, simple, makeSimpleDrv(*store)) +INSTANTIATE_TEST_SUITE_P(DerivationJSONATerm, DerivationJsonAtermTest, ::testing::Values(makeSimpleDrv())); -TEST_ATERM(DerivationTest, simple, makeSimpleDrv(*store), "simple-derivation") +struct DynDerivationJsonAtermTest : DynDerivationTest, + JsonCharacterizationTest, + ::testing::WithParamInterface +{}; -Derivation makeDynDepDerivation(const Store & store) +MAKE_TEST_P(DynDerivationJsonAtermTest); + +Derivation makeDynDepDerivation() { Derivation drv; drv.name = "dyn-dep-derivation"; drv.inputSrcs = { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"), + StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep1"}, }; drv.inputDrvs = { .map = { { - store.parseStorePath("/nix/store/c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"), + StorePath{"c015dhfh5l0lp6wxyvdn7bmwhbbr6hr9-dep2.drv"}, DerivedPathMap::ChildNode{ .value = { @@ -293,11 +335,8 @@ Derivation makeDynDepDerivation(const Store & store) return drv; } -TEST_JSON(DynDerivationTest, dynDerivationDeps, makeDynDepDerivation(*store)) - -TEST_ATERM(DynDerivationTest, dynDerivationDeps, makeDynDepDerivation(*store), "dyn-dep-derivation") +INSTANTIATE_TEST_SUITE_P(DynDerivationJSONATerm, DynDerivationJsonAtermTest, ::testing::Values(makeDynDepDerivation())); -#undef TEST_JSON -#undef TEST_ATERM +#undef MAKE_TEST_P } // namespace nix diff --git a/src/libstore-tests/derived-path.cc b/src/libstore-tests/derived-path.cc index c7d2c58172e..6e7648f2589 100644 --- a/src/libstore-tests/derived-path.cc +++ b/src/libstore-tests/derived-path.cc @@ -3,13 +3,23 @@ #include #include +#include "nix/util/tests/characterization.hh" #include "nix/store/tests/derived-path.hh" #include "nix/store/tests/libstore.hh" namespace nix { -class DerivedPathTest : public LibStoreTest -{}; +class DerivedPathTest : public CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "derived-path"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; /** * Round trip (string <-> data structure) test for @@ -107,4 +117,90 @@ RC_GTEST_FIXTURE_PROP(DerivedPathTest, prop_round_rip, (const DerivedPath & o)) #endif +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +using nlohmann::json; + +#define TEST_JSON(TYPE, NAME, VAL) \ + static const TYPE NAME = VAL; \ + \ + TEST_F(DerivedPathTest, NAME##_from_json) \ + { \ + readTest(#NAME ".json", [&](const auto & encoded_) { \ + auto encoded = json::parse(encoded_); \ + TYPE got = static_cast(encoded); \ + ASSERT_EQ(got, NAME); \ + }); \ + } \ + \ + TEST_F(DerivedPathTest, NAME##_to_json) \ + { \ + writeTest( \ + #NAME ".json", \ + [&]() -> json { return static_cast(NAME); }, \ + [](const auto & file) { return json::parse(readFile(file)); }, \ + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); \ + } + +TEST_JSON( + SingleDerivedPath, single_opaque, SingleDerivedPath::Opaque{StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}); + +TEST_JSON( + SingleDerivedPath, + single_built, + (SingleDerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Opaque{ + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}), + .output = "bar", + })); + +TEST_JSON( + SingleDerivedPath, + single_built_built, + (SingleDerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Opaque{ + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}), + .output = "bar", + }), + .output = "baz", + })); + +TEST_JSON(DerivedPath, multi_opaque, DerivedPath::Opaque{StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}); + +TEST_JSON( + DerivedPath, + mutli_built, + (DerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Opaque{ + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}), + .outputs = OutputsSpec::Names{"bar", "baz"}, + })); + +TEST_JSON( + DerivedPath, + multi_built_built, + (DerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Opaque{ + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}), + .output = "bar", + }), + .outputs = OutputsSpec::Names{"baz", "quux"}, + })); + +TEST_JSON( + DerivedPath, + multi_built_built_wildcard, + (DerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Built{ + .drvPath = make_ref(SingleDerivedPath::Opaque{ + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}}), + .output = "bar", + }), + .outputs = OutputsSpec::All{}, + })); + } // namespace nix diff --git a/src/libstore-tests/dummy-store.cc b/src/libstore-tests/dummy-store.cc new file mode 100644 index 00000000000..b841d789002 --- /dev/null +++ b/src/libstore-tests/dummy-store.cc @@ -0,0 +1,27 @@ +#include + +#include "nix/store/dummy-store.hh" +#include "nix/store/globals.hh" +#include "nix/store/realisation.hh" + +namespace nix { + +TEST(DummyStore, realisation_read) +{ + initLibStore(/*loadConfig=*/false); + + auto store = [] { + auto cfg = make_ref(StoreReference::Params{}); + cfg->readOnly = false; + return cfg->openStore(); + }(); + + auto drvHash = Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", HashAlgorithm::SHA256, HashFormat::Base16); + + auto outputName = "foo"; + + EXPECT_EQ(store->queryRealisation({drvHash, outputName}), nullptr); +} + +} // namespace nix diff --git a/src/libstore-tests/http-binary-cache-store.cc b/src/libstore-tests/http-binary-cache-store.cc index 0e3be4cedb5..4b3754a1fe4 100644 --- a/src/libstore-tests/http-binary-cache-store.cc +++ b/src/libstore-tests/http-binary-cache-store.cc @@ -18,4 +18,20 @@ TEST(HttpBinaryCacheStore, constructConfigNoTrailingSlash) EXPECT_EQ(config.cacheUri.to_string(), "https://foo.bar.baz/a/b"); } +TEST(HttpBinaryCacheStore, constructConfigWithParams) +{ + StoreConfig::Params params{{"compression", "xz"}}; + HttpBinaryCacheStoreConfig config{"https", "foo.bar.baz/a/b/", params}; + EXPECT_EQ(config.cacheUri.to_string(), "https://foo.bar.baz/a/b"); + EXPECT_EQ(config.getReference().params, params); +} + +TEST(HttpBinaryCacheStore, constructConfigWithParamsAndUrlWithParams) +{ + StoreConfig::Params params{{"compression", "xz"}}; + HttpBinaryCacheStoreConfig config{"https", "foo.bar.baz/a/b?some-param=some-value", params}; + EXPECT_EQ(config.cacheUri.to_string(), "https://foo.bar.baz/a/b?some-param=some-value"); + EXPECT_EQ(config.getReference().params, params); +} + } // namespace nix diff --git a/src/libstore-tests/local-store.cc b/src/libstore-tests/local-store.cc index cdbc29b0319..d008888974b 100644 --- a/src/libstore-tests/local-store.cc +++ b/src/libstore-tests/local-store.cc @@ -33,4 +33,10 @@ TEST(LocalStore, constructConfig_rootPath) EXPECT_EQ(config.rootDir.get(), std::optional{"/foo/bar"}); } +TEST(LocalStore, constructConfig_to_string) +{ + LocalStoreConfig config{"local", "", {}}; + EXPECT_EQ(config.getReference().to_string(), "local"); +} + } // namespace nix diff --git a/src/libstore-tests/main.cc b/src/libstore-tests/main.cc new file mode 100644 index 00000000000..ffe9816134f --- /dev/null +++ b/src/libstore-tests/main.cc @@ -0,0 +1,15 @@ +#include + +#include "nix/store/tests/test-main.hh" + +using namespace nix; + +int main(int argc, char ** argv) +{ + auto res = testMainForBuidingPre(argc, argv); + if (res) + return res; + + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} diff --git a/src/libstore-tests/meson.build b/src/libstore-tests/meson.build index fced202696e..2784b31dce8 100644 --- a/src/libstore-tests/meson.build +++ b/src/libstore-tests/meson.build @@ -52,6 +52,7 @@ gtest = dependency('gmock') deps_private += gtest subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'common-protocol.cc', @@ -60,21 +61,24 @@ sources = files( 'derivation.cc', 'derived-path.cc', 'downstream-placeholder.cc', + 'dummy-store.cc', 'http-binary-cache-store.cc', 'legacy-ssh-store.cc', 'local-binary-cache-store.cc', 'local-overlay-store.cc', 'local-store.cc', 'machines.cc', + 'main.cc', 'nar-info-disk-cache.cc', 'nar-info.cc', 'nix_api_store.cc', 'outputs-spec.cc', 'path-info.cc', 'path.cc', + 'realisation.cc', 'references.cc', 's3-binary-cache-store.cc', - 's3.cc', + 's3-url.cc', 'serve-protocol.cc', 'ssh-store.cc', 'store-reference.cc', @@ -101,7 +105,7 @@ this_exe = executable( test( meson.project_name(), this_exe, - env : { + env : asan_test_options_env + { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', 'HOME' : meson.current_build_dir() / 'test-home', 'NIX_REMOTE' : meson.current_build_dir() / 'test-home' / 'store', @@ -130,10 +134,13 @@ if get_option('benchmarks') link_args : linker_export_flags, install : true, cpp_pch : do_pch ? [ 'pch/precompiled-headers.hh' ] : [], - cpp_args : [ - '-DNIX_UNIT_TEST_DATA="' + meson.current_source_dir() + '/data"', - ], ) - benchmark('nix-store-benchmarks', benchmark_exe) + benchmark( + 'nix-store-benchmarks', + benchmark_exe, + env : asan_test_options_env + { + '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', + }, + ) endif diff --git a/src/libstore-tests/nar-info.cc b/src/libstore-tests/nar-info.cc index a73df119051..751c5e305bb 100644 --- a/src/libstore-tests/nar-info.cc +++ b/src/libstore-tests/nar-info.cc @@ -23,7 +23,7 @@ class NarInfoTest : public CharacterizationTest, public LibStoreTest static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) { - NarInfo info = ValidPathInfo{ + auto info = NarInfo::makeFromCA( store, "foo", FixedOutputInfo{ @@ -41,8 +41,7 @@ static NarInfo makeNarInfo(const Store & store, bool includeImpureInfo) .self = true, }, }, - Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - }; + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=")); info.narSize = 34878; if (includeImpureInfo) { info.deriver = StorePath{ diff --git a/src/libstore-tests/nix_api_store.cc b/src/libstore-tests/nix_api_store.cc index c7146f977a5..dfd554ec160 100644 --- a/src/libstore-tests/nix_api_store.cc +++ b/src/libstore-tests/nix_api_store.cc @@ -1,9 +1,10 @@ +#include + #include "nix_api_util.h" -#include "nix_api_util_internal.h" #include "nix_api_store.h" -#include "nix_api_store_internal.h" #include "nix/store/tests/nix_api_store.hh" +#include "nix/store/globals.hh" #include "nix/util/tests/string_callback.hh" #include "nix/util/url.hh" @@ -65,7 +66,7 @@ TEST_F(nix_api_store_test, nix_store_get_storedir) TEST_F(nix_api_store_test, InvalidPathFails) { nix_store_parse_path(ctx, store, "invalid-path"); - ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); + ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR); } TEST_F(nix_api_store_test, ReturnsValidStorePath) @@ -80,7 +81,7 @@ TEST_F(nix_api_store_test, ReturnsValidStorePath) TEST_F(nix_api_store_test, SetsLastErrCodeToNixOk) { StorePath * path = nix_store_parse_path(ctx, store, (nixStoreDir + PATH_SUFFIX).c_str()); - ASSERT_EQ(ctx->last_err_code, NIX_OK); + ASSERT_EQ(nix_err_code(ctx), NIX_OK); nix_store_path_free(path); } @@ -103,7 +104,7 @@ TEST_F(nix_api_util_context, nix_store_open_dummy) { nix_libstore_init(ctx); Store * store = nix_store_open(ctx, "dummy://", nullptr); - ASSERT_EQ(NIX_OK, ctx->last_err_code); + ASSERT_EQ(NIX_OK, nix_err_code(ctx)); ASSERT_STREQ("dummy://", store->ptr->config.getReference().render(/*withParams=*/true).c_str()); std::string str; @@ -117,7 +118,7 @@ TEST_F(nix_api_util_context, nix_store_open_invalid) { nix_libstore_init(ctx); Store * store = nix_store_open(ctx, "invalid://", nullptr); - ASSERT_EQ(NIX_ERR_NIX_ERROR, ctx->last_err_code); + ASSERT_EQ(NIX_ERR_NIX_ERROR, nix_err_code(ctx)); ASSERT_EQ(nullptr, store); nix_store_free(store); } @@ -199,4 +200,60 @@ TEST_F(nix_api_util_context, nix_store_real_path_binary_cache) ASSERT_STREQ(path_raw.c_str(), rp.c_str()); } +template +struct LambdaAdapter +{ + F fun; + + template + static inline auto call(LambdaAdapter * ths, Args... args) + { + return ths->fun(args...); + } + + template + static auto call_void(void * ths, Args... args) + { + return call(static_cast *>(ths), args...); + } +}; + +TEST_F(nix_api_store_test_base, build_from_json) +{ + // FIXME get rid of these + nix::experimentalFeatureSettings.set("extra-experimental-features", "ca-derivations"); + nix::settings.substituters = {}; + + auto * store = open_local_store(); + + std::filesystem::path unitTestData{getenv("_NIX_TEST_UNIT_DATA")}; + + std::ifstream t{unitTestData / "derivation/ca/self-contained.json"}; + std::stringstream buffer; + buffer << t.rdbuf(); + + auto * drv = nix_derivation_from_json(ctx, store, buffer.str().c_str()); + assert_ctx_ok(); + ASSERT_NE(drv, nullptr); + + auto * drvPath = nix_add_derivation(ctx, store, drv); + assert_ctx_ok(); + ASSERT_NE(drv, nullptr); + + auto cb = LambdaAdapter{.fun = [&](const char * outname, const StorePath * outPath) { + auto is_valid_path = nix_store_is_valid_path(ctx, store, outPath); + ASSERT_EQ(is_valid_path, true); + }}; + + auto ret = nix_store_realise( + ctx, store, drvPath, static_cast(&cb), decltype(cb)::call_void); + assert_ctx_ok(); + ASSERT_EQ(ret, NIX_OK); + + // Clean up + nix_store_path_free(drvPath); + nix_derivation_free(drv); + nix_store_free(store); +} + } // namespace nixC diff --git a/src/libstore-tests/outputs-spec.cc b/src/libstore-tests/outputs-spec.cc index b0b80e7c407..1fac222fccb 100644 --- a/src/libstore-tests/outputs-spec.cc +++ b/src/libstore-tests/outputs-spec.cc @@ -1,18 +1,43 @@ -#include "nix/store/tests/outputs-spec.hh" - #include #include #include +#include "nix/store/tests/outputs-spec.hh" +#include "nix/util/tests/json-characterization.hh" + namespace nix { -TEST(OutputsSpec, no_empty_names) +class OutputsSpecTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "outputs-spec"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +class ExtendedOutputsSpecTest : public virtual CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "outputs-spec" / "extended"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +TEST_F(OutputsSpecTest, no_empty_names) { ASSERT_DEATH(OutputsSpec::Names{StringSet{}}, ""); } #define TEST_DONT_PARSE(NAME, STR) \ - TEST(OutputsSpec, bad_##NAME) \ + TEST_F(OutputsSpecTest, bad_##NAME) \ { \ std::optional OutputsSpecOpt = OutputsSpec::parseOpt(STR); \ ASSERT_FALSE(OutputsSpecOpt); \ @@ -26,7 +51,7 @@ TEST_DONT_PARSE(star_second, "foo,*") #undef TEST_DONT_PARSE -TEST(OutputsSpec, all) +TEST_F(OutputsSpecTest, all) { std::string_view str = "*"; OutputsSpec expected = OutputsSpec::All{}; @@ -34,7 +59,7 @@ TEST(OutputsSpec, all) ASSERT_EQ(expected.to_string(), str); } -TEST(OutputsSpec, names_out) +TEST_F(OutputsSpecTest, names_out) { std::string_view str = "out"; OutputsSpec expected = OutputsSpec::Names{"out"}; @@ -42,7 +67,7 @@ TEST(OutputsSpec, names_out) ASSERT_EQ(expected.to_string(), str); } -TEST(OutputsSpec, names_underscore) +TEST_F(OutputsSpecTest, names_underscore) { std::string_view str = "a_b"; OutputsSpec expected = OutputsSpec::Names{"a_b"}; @@ -50,7 +75,7 @@ TEST(OutputsSpec, names_underscore) ASSERT_EQ(expected.to_string(), str); } -TEST(OutputsSpec, names_numeric) +TEST_F(OutputsSpecTest, names_numeric) { std::string_view str = "01"; OutputsSpec expected = OutputsSpec::Names{"01"}; @@ -58,7 +83,7 @@ TEST(OutputsSpec, names_numeric) ASSERT_EQ(expected.to_string(), str); } -TEST(OutputsSpec, names_out_bin) +TEST_F(OutputsSpecTest, names_out_bin) { OutputsSpec expected = OutputsSpec::Names{"out", "bin"}; ASSERT_EQ(OutputsSpec::parse("out,bin"), expected); @@ -68,32 +93,32 @@ TEST(OutputsSpec, names_out_bin) #define TEST_SUBSET(X, THIS, THAT) X((OutputsSpec{THIS}).isSubsetOf(THAT)); -TEST(OutputsSpec, subsets_all_all) +TEST_F(OutputsSpecTest, subsets_all_all) { TEST_SUBSET(ASSERT_TRUE, OutputsSpec::All{}, OutputsSpec::All{}); } -TEST(OutputsSpec, subsets_names_all) +TEST_F(OutputsSpecTest, subsets_names_all) { TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names{"a"}, OutputsSpec::All{}); } -TEST(OutputsSpec, subsets_names_names_eq) +TEST_F(OutputsSpecTest, subsets_names_names_eq) { TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names{"a"}, OutputsSpec::Names{"a"}); } -TEST(OutputsSpec, subsets_names_names_noneq) +TEST_F(OutputsSpecTest, subsets_names_names_noneq) { TEST_SUBSET(ASSERT_TRUE, OutputsSpec::Names{"a"}, (OutputsSpec::Names{"a", "b"})); } -TEST(OutputsSpec, not_subsets_all_names) +TEST_F(OutputsSpecTest, not_subsets_all_names) { TEST_SUBSET(ASSERT_FALSE, OutputsSpec::All{}, OutputsSpec::Names{"a"}); } -TEST(OutputsSpec, not_subsets_names_names) +TEST_F(OutputsSpecTest, not_subsets_names_names) { TEST_SUBSET(ASSERT_FALSE, (OutputsSpec::Names{"a", "b"}), (OutputsSpec::Names{"a"})); } @@ -102,22 +127,22 @@ TEST(OutputsSpec, not_subsets_names_names) #define TEST_UNION(RES, THIS, THAT) ASSERT_EQ(OutputsSpec{RES}, (OutputsSpec{THIS}).union_(THAT)); -TEST(OutputsSpec, union_all_all) +TEST_F(OutputsSpecTest, union_all_all) { TEST_UNION(OutputsSpec::All{}, OutputsSpec::All{}, OutputsSpec::All{}); } -TEST(OutputsSpec, union_all_names) +TEST_F(OutputsSpecTest, union_all_names) { TEST_UNION(OutputsSpec::All{}, OutputsSpec::All{}, OutputsSpec::Names{"a"}); } -TEST(OutputsSpec, union_names_all) +TEST_F(OutputsSpecTest, union_names_all) { TEST_UNION(OutputsSpec::All{}, OutputsSpec::Names{"a"}, OutputsSpec::All{}); } -TEST(OutputsSpec, union_names_names) +TEST_F(OutputsSpecTest, union_names_names) { TEST_UNION((OutputsSpec::Names{"a", "b"}), OutputsSpec::Names{"a"}, OutputsSpec::Names{"b"}); } @@ -125,7 +150,7 @@ TEST(OutputsSpec, union_names_names) #undef TEST_UNION #define TEST_DONT_PARSE(NAME, STR) \ - TEST(ExtendedOutputsSpec, bad_##NAME) \ + TEST_F(ExtendedOutputsSpecTest, bad_##NAME) \ { \ std::optional extendedOutputsSpecOpt = ExtendedOutputsSpec::parseOpt(STR); \ ASSERT_FALSE(extendedOutputsSpecOpt); \ @@ -140,7 +165,7 @@ TEST_DONT_PARSE(star_second, "^foo,*") #undef TEST_DONT_PARSE -TEST(ExtendedOutputsSpec, default) +TEST_F(ExtendedOutputsSpecTest, default) { std::string_view str = "foo"; auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); @@ -150,7 +175,7 @@ TEST(ExtendedOutputsSpec, default) ASSERT_EQ(std::string{prefix} + expected.to_string(), str); } -TEST(ExtendedOutputsSpec, all) +TEST_F(ExtendedOutputsSpecTest, all) { std::string_view str = "foo^*"; auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); @@ -160,7 +185,7 @@ TEST(ExtendedOutputsSpec, all) ASSERT_EQ(std::string{prefix} + expected.to_string(), str); } -TEST(ExtendedOutputsSpec, out) +TEST_F(ExtendedOutputsSpecTest, out) { std::string_view str = "foo^out"; auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse(str); @@ -170,7 +195,7 @@ TEST(ExtendedOutputsSpec, out) ASSERT_EQ(std::string{prefix} + expected.to_string(), str); } -TEST(ExtendedOutputsSpec, out_bin) +TEST_F(ExtendedOutputsSpecTest, out_bin) { auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^out,bin"); ASSERT_EQ(prefix, "foo"); @@ -179,7 +204,7 @@ TEST(ExtendedOutputsSpec, out_bin) ASSERT_EQ(std::string{prefix} + expected.to_string(), "foo^bin,out"); } -TEST(ExtendedOutputsSpec, many_carrot) +TEST_F(ExtendedOutputsSpecTest, many_carrot) { auto [prefix, extendedOutputsSpec] = ExtendedOutputsSpec::parse("foo^bar^out,bin"); ASSERT_EQ(prefix, "foo^bar"); @@ -188,28 +213,49 @@ TEST(ExtendedOutputsSpec, many_carrot) ASSERT_EQ(std::string{prefix} + expected.to_string(), "foo^bar^bin,out"); } -#define TEST_JSON(TYPE, NAME, STR, VAL) \ - \ - TEST(TYPE, NAME##_to_json) \ - { \ - using nlohmann::literals::operator"" _json; \ - ASSERT_EQ(STR##_json, ((nlohmann::json) TYPE{VAL})); \ - } \ - \ - TEST(TYPE, NAME##_from_json) \ - { \ - using nlohmann::literals::operator"" _json; \ - ASSERT_EQ(TYPE{VAL}, (STR##_json).get()); \ +#define MAKE_TEST_P(FIXTURE, TYPE) \ + TEST_P(FIXTURE, from_json) \ + { \ + const auto & [name, value] = GetParam(); \ + readJsonTest(name, value); \ + } \ + \ + TEST_P(FIXTURE, to_json) \ + { \ + const auto & [name, value] = GetParam(); \ + writeJsonTest(name, value); \ } -TEST_JSON(OutputsSpec, all, R"(["*"])", OutputsSpec::All{}) -TEST_JSON(OutputsSpec, name, R"(["a"])", OutputsSpec::Names{"a"}) -TEST_JSON(OutputsSpec, names, R"(["a","b"])", (OutputsSpec::Names{"a", "b"})) - -TEST_JSON(ExtendedOutputsSpec, def, R"(null)", ExtendedOutputsSpec::Default{}) -TEST_JSON(ExtendedOutputsSpec, all, R"(["*"])", ExtendedOutputsSpec::Explicit{OutputsSpec::All{}}) -TEST_JSON(ExtendedOutputsSpec, name, R"(["a"])", ExtendedOutputsSpec::Explicit{OutputsSpec::Names{"a"}}) -TEST_JSON(ExtendedOutputsSpec, names, R"(["a","b"])", (ExtendedOutputsSpec::Explicit{OutputsSpec::Names{"a", "b"}})) +struct OutputsSpecJsonTest : OutputsSpecTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +MAKE_TEST_P(OutputsSpecJsonTest, OutputsSpec); + +INSTANTIATE_TEST_SUITE_P( + OutputsSpecJSON, + OutputsSpecJsonTest, + ::testing::Values( + std::pair{"all", OutputsSpec{OutputsSpec::All{}}}, + std::pair{"name", OutputsSpec{OutputsSpec::Names{"a"}}}, + std::pair{"names", OutputsSpec{OutputsSpec::Names{"a", "b"}}})); + +struct ExtendedOutputsSpecJsonTest : ExtendedOutputsSpecTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +MAKE_TEST_P(ExtendedOutputsSpecJsonTest, ExtendedOutputsSpec); + +INSTANTIATE_TEST_SUITE_P( + ExtendedOutputsSpecJSON, + ExtendedOutputsSpecJsonTest, + ::testing::Values( + std::pair{"def", ExtendedOutputsSpec{ExtendedOutputsSpec::Default{}}}, + std::pair{"all", ExtendedOutputsSpec{ExtendedOutputsSpec::Explicit{OutputsSpec::All{}}}}, + std::pair{"name", ExtendedOutputsSpec{ExtendedOutputsSpec::Explicit{OutputsSpec::Names{"a"}}}}, + std::pair{"names", ExtendedOutputsSpec{ExtendedOutputsSpec::Explicit{OutputsSpec::Names{"a", "b"}}}})); #undef TEST_JSON diff --git a/src/libstore-tests/package.nix b/src/libstore-tests/package.nix index 90e6af519bf..d5255f4f988 100644 --- a/src/libstore-tests/package.nix +++ b/src/libstore-tests/package.nix @@ -83,6 +83,7 @@ mkMesonExecutable (finalAttrs: { } ( '' + export ASAN_OPTIONS=abort_on_error=1:print_summary=1:detect_leaks=0 export _NIX_TEST_UNIT_DATA=${data + "/src/libstore-tests/data"} export NIX_REMOTE=$HOME/store ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} diff --git a/src/libstore-tests/path-info.cc b/src/libstore-tests/path-info.cc index de5c9515083..63310c1c391 100644 --- a/src/libstore-tests/path-info.cc +++ b/src/libstore-tests/path-info.cc @@ -29,7 +29,7 @@ static UnkeyedValidPathInfo makeEmpty() static ValidPathInfo makeFullKeyed(const Store & store, bool includeImpureInfo) { - ValidPathInfo info = ValidPathInfo{ + auto info = ValidPathInfo::makeFromCA( store, "foo", FixedOutputInfo{ @@ -47,8 +47,7 @@ static ValidPathInfo makeFullKeyed(const Store & store, bool includeImpureInfo) .self = true, }, }, - Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - }; + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=")); info.narSize = 34878; if (includeImpureInfo) { info.deriver = StorePath{ diff --git a/src/libstore-tests/path.cc b/src/libstore-tests/path.cc index 01d1ca792a9..eb860a34dab 100644 --- a/src/libstore-tests/path.cc +++ b/src/libstore-tests/path.cc @@ -7,7 +7,7 @@ #include "nix/store/path-regex.hh" #include "nix/store/store-api.hh" -#include "nix/util/tests/hash.hh" +#include "nix/util/tests/json-characterization.hh" #include "nix/store/tests/libstore.hh" #include "nix/store/tests/path.hh" @@ -16,8 +16,17 @@ namespace nix { #define STORE_DIR "/nix/store/" #define HASH_PART "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q" -class StorePathTest : public LibStoreTest -{}; +class StorePathTest : public virtual CharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "store-path"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; static std::regex nameRegex{std::string{nameRegexStr}}; @@ -134,4 +143,36 @@ RC_GTEST_FIXTURE_PROP(StorePathTest, prop_check_regex_eq_parse, ()) #endif +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +using nlohmann::json; + +struct StorePathJsonTest : StorePathTest, + JsonCharacterizationTest, + ::testing::WithParamInterface> +{}; + +TEST_P(StorePathJsonTest, from_json) +{ + auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} + +TEST_P(StorePathJsonTest, to_json) +{ + auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P( + StorePathJSON, + StorePathJsonTest, + ::testing::Values( + std::pair{ + "simple", + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + })); + } // namespace nix diff --git a/src/libstore-tests/realisation.cc b/src/libstore-tests/realisation.cc new file mode 100644 index 00000000000..a5a5bee508a --- /dev/null +++ b/src/libstore-tests/realisation.cc @@ -0,0 +1,97 @@ +#include + +#include +#include +#include + +#include "nix/store/store-api.hh" + +#include "nix/util/tests/json-characterization.hh" +#include "nix/store/tests/libstore.hh" + +namespace nix { + +class RealisationTest : public JsonCharacterizationTest, public LibStoreTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "realisation"; + +public: + + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; + +/* ---------------------------------------------------------------------------- + * JSON + * --------------------------------------------------------------------------*/ + +using nlohmann::json; + +struct RealisationJsonTest : RealisationTest, ::testing::WithParamInterface> +{}; + +TEST_P(RealisationJsonTest, from_json) +{ + const auto & [name, expected] = GetParam(); + readJsonTest(name, expected); +} + +TEST_P(RealisationJsonTest, to_json) +{ + const auto & [name, value] = GetParam(); + writeJsonTest(name, value); +} + +INSTANTIATE_TEST_SUITE_P( + RealisationJSON, + RealisationJsonTest, + ([] { + Realisation simple{ + + .id = + { + .drvHash = Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + HashAlgorithm::SHA256, + HashFormat::Base16), + .outputName = "foo", + }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + }; + return ::testing::Values( + std::pair{ + "simple", + simple, + }, + std::pair{ + "with-signature", + [&] { + auto r = simple; + // FIXME actually sign properly + r.signatures = {"asdfasdfasdf"}; + return r; + }()}, + std::pair{ + "with-dependent-realisations", + [&] { + auto r = simple; + r.dependentRealisations = {{ + { + .drvHash = Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + HashAlgorithm::SHA256, + HashFormat::Base16), + .outputName = "foo", + }, + StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo.drv"}, + }}; + return r; + }(), + }); + } + + ())); + +} // namespace nix diff --git a/src/libstore-tests/s3-url.cc b/src/libstore-tests/s3-url.cc new file mode 100644 index 00000000000..60652dd9cab --- /dev/null +++ b/src/libstore-tests/s3-url.cc @@ -0,0 +1,232 @@ +#include "nix/store/s3-url.hh" +#include "nix/util/tests/gmock-matchers.hh" + +#if NIX_WITH_S3_SUPPORT || NIX_WITH_CURL_S3 + +# include +# include + +namespace nix { + +// ============================================================================= +// ParsedS3URL Tests +// ============================================================================= + +struct ParsedS3URLTestCase +{ + std::string url; + ParsedS3URL expected; + std::string description; +}; + +class ParsedS3URLTest : public ::testing::WithParamInterface, public ::testing::Test +{}; + +TEST_P(ParsedS3URLTest, parseS3URLSuccessfully) +{ + const auto & testCase = GetParam(); + auto parsed = ParsedS3URL::parse(parseURL(testCase.url)); + ASSERT_EQ(parsed, testCase.expected); +} + +INSTANTIATE_TEST_SUITE_P( + QueryParams, + ParsedS3URLTest, + ::testing::Values( + ParsedS3URLTestCase{ + "s3://my-bucket/my-key.txt", + { + .bucket = "my-bucket", + .key = {"my-key.txt"}, + }, + "basic_s3_bucket", + }, + ParsedS3URLTestCase{ + "s3://prod-cache/nix/store/abc123.nar.xz?region=eu-west-1", + { + .bucket = "prod-cache", + .key = {"nix", "store", "abc123.nar.xz"}, + .region = "eu-west-1", + }, + "with_region", + }, + ParsedS3URLTestCase{ + "s3://bucket/key?region=us-west-2&profile=prod&endpoint=custom.s3.com&scheme=https®ion=us-east-1", + { + .bucket = "bucket", + .key = {"key"}, + .profile = "prod", + .region = "us-west-2", //< using the first parameter (decodeQuery ignores dupicates) + .scheme = "https", + .endpoint = ParsedURL::Authority{.host = "custom.s3.com"}, + }, + "complex", + }, + ParsedS3URLTestCase{ + "s3://cache/file.txt?profile=production®ion=ap-southeast-2", + { + .bucket = "cache", + .key = {"file.txt"}, + .profile = "production", + .region = "ap-southeast-2", + }, + "with_profile_and_region", + }, + ParsedS3URLTestCase{ + "s3://bucket/key?endpoint=https://minio.local&scheme=http", + { + .bucket = "bucket", + .key = {"key"}, + /* TODO: Figure out what AWS SDK is doing when both endpointOverride and scheme are set. */ + .scheme = "http", + .endpoint = + ParsedURL{ + .scheme = "https", + .authority = ParsedURL::Authority{.host = "minio.local"}, + .path = {""}, + }, + }, + "with_absolute_endpoint_uri", + }), + [](const ::testing::TestParamInfo & info) { return info.param.description; }); + +// Parameterized test for invalid S3 URLs +struct InvalidS3URLTestCase +{ + std::string url; + std::string expectedErrorSubstring; + std::string description; +}; + +class InvalidParsedS3URLTest : public ::testing::WithParamInterface, public ::testing::Test +{}; + +TEST_P(InvalidParsedS3URLTest, parseS3URLErrors) +{ + const auto & testCase = GetParam(); + + ASSERT_THAT( + [&testCase]() { ParsedS3URL::parse(parseURL(testCase.url)); }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher(testCase.expectedErrorSubstring))); +} + +INSTANTIATE_TEST_SUITE_P( + InvalidUrls, + InvalidParsedS3URLTest, + ::testing::Values( + InvalidS3URLTestCase{"s3:///key", "error: URI has a missing or invalid bucket name", "empty_bucket"}, + InvalidS3URLTestCase{"s3://127.0.0.1", "error: URI has a missing or invalid bucket name", "ip_address_bucket"}, + InvalidS3URLTestCase{"s3://bucket with spaces/key", "is not a valid URL", "bucket_with_spaces"}, + InvalidS3URLTestCase{"s3://", "error: URI has a missing or invalid bucket name", "completely_empty"}, + InvalidS3URLTestCase{"s3://bucket", "error: URI has a missing or invalid key", "missing_key"}), + [](const ::testing::TestParamInfo & info) { return info.param.description; }); + +// ============================================================================= +// S3 URL to HTTPS Conversion Tests +// ============================================================================= + +struct S3ToHttpsConversionTestCase +{ + ParsedS3URL input; + ParsedURL expected; + std::string expectedRendered; + std::string description; +}; + +class S3ToHttpsConversionTest : public ::testing::WithParamInterface, + public ::testing::Test +{}; + +TEST_P(S3ToHttpsConversionTest, ConvertsCorrectly) +{ + const auto & testCase = GetParam(); + auto result = testCase.input.toHttpsUrl(); + EXPECT_EQ(result, testCase.expected) << "Failed for: " << testCase.description; + EXPECT_EQ(result.to_string(), testCase.expectedRendered); +} + +INSTANTIATE_TEST_SUITE_P( + S3ToHttpsConversion, + S3ToHttpsConversionTest, + ::testing::Values( + S3ToHttpsConversionTestCase{ + ParsedS3URL{ + .bucket = "my-bucket", + .key = {"my-key.txt"}, + }, + ParsedURL{ + .scheme = "https", + .authority = ParsedURL::Authority{.host = "s3.us-east-1.amazonaws.com"}, + .path = {"", "my-bucket", "my-key.txt"}, + }, + "https://s3.us-east-1.amazonaws.com/my-bucket/my-key.txt", + "basic_s3_default_region", + }, + S3ToHttpsConversionTestCase{ + ParsedS3URL{ + .bucket = "prod-cache", + .key = {"nix", "store", "abc123.nar.xz"}, + .region = "eu-west-1", + }, + ParsedURL{ + .scheme = "https", + .authority = ParsedURL::Authority{.host = "s3.eu-west-1.amazonaws.com"}, + .path = {"", "prod-cache", "nix", "store", "abc123.nar.xz"}, + }, + "https://s3.eu-west-1.amazonaws.com/prod-cache/nix/store/abc123.nar.xz", + "with_eu_west_1_region", + }, + S3ToHttpsConversionTestCase{ + ParsedS3URL{ + .bucket = "bucket", + .key = {"key"}, + .scheme = "http", + .endpoint = ParsedURL::Authority{.host = "custom.s3.com"}, + }, + ParsedURL{ + .scheme = "http", + .authority = ParsedURL::Authority{.host = "custom.s3.com"}, + .path = {"", "bucket", "key"}, + }, + "http://custom.s3.com/bucket/key", + "custom_endpoint_authority", + }, + S3ToHttpsConversionTestCase{ + ParsedS3URL{ + .bucket = "bucket", + .key = {"key"}, + .endpoint = + ParsedURL{ + .scheme = "http", + .authority = ParsedURL::Authority{.host = "server", .port = 9000}, + .path = {""}, + }, + }, + ParsedURL{ + .scheme = "http", + .authority = ParsedURL::Authority{.host = "server", .port = 9000}, + .path = {"", "bucket", "key"}, + }, + "http://server:9000/bucket/key", + "custom_endpoint_with_port", + }, + S3ToHttpsConversionTestCase{ + ParsedS3URL{ + .bucket = "bucket", + .key = {"path", "to", "file.txt"}, + .region = "ap-southeast-2", + .scheme = "https", + }, + ParsedURL{ + .scheme = "https", + .authority = ParsedURL::Authority{.host = "s3.ap-southeast-2.amazonaws.com"}, + .path = {"", "bucket", "path", "to", "file.txt"}, + }, + "https://s3.ap-southeast-2.amazonaws.com/bucket/path/to/file.txt", + "complex_path_and_region", + }), + [](const ::testing::TestParamInfo & info) { return info.param.description; }); + +} // namespace nix + +#endif diff --git a/src/libstore-tests/s3.cc b/src/libstore-tests/s3.cc deleted file mode 100644 index b66005cb925..00000000000 --- a/src/libstore-tests/s3.cc +++ /dev/null @@ -1,96 +0,0 @@ -#include "nix/store/s3.hh" -#include "nix/util/tests/gmock-matchers.hh" - -#if NIX_WITH_S3_SUPPORT - -# include -# include - -namespace nix { - -struct ParsedS3URLTestCase -{ - std::string url; - ParsedS3URL expected; - std::string description; -}; - -class ParsedS3URLTest : public ::testing::WithParamInterface, public ::testing::Test -{}; - -TEST_P(ParsedS3URLTest, parseS3URLSuccessfully) -{ - const auto & testCase = GetParam(); - auto parsed = ParsedS3URL::parse(testCase.url); - ASSERT_EQ(parsed, testCase.expected); -} - -INSTANTIATE_TEST_SUITE_P( - QueryParams, - ParsedS3URLTest, - ::testing::Values( - ParsedS3URLTestCase{ - "s3://my-bucket/my-key.txt", - { - .bucket = "my-bucket", - .key = "my-key.txt", - }, - "basic_s3_bucket"}, - ParsedS3URLTestCase{ - "s3://prod-cache/nix/store/abc123.nar.xz?region=eu-west-1", - { - .bucket = "prod-cache", - .key = "nix/store/abc123.nar.xz", - .region = "eu-west-1", - }, - "with_region"}, - ParsedS3URLTestCase{ - "s3://bucket/key?region=us-west-2&profile=prod&endpoint=custom.s3.com&scheme=https®ion=us-east-1", - { - .bucket = "bucket", - .key = "key", - .profile = "prod", - .region = "us-west-2", //< using the first parameter (decodeQuery ignores dupicates) - .scheme = "https", - .endpoint = ParsedURL::Authority{.host = "custom.s3.com"}, - }, - "complex"}, - ParsedS3URLTestCase{ - "s3://cache/file.txt?profile=production®ion=ap-southeast-2", - { - .bucket = "cache", - .key = "file.txt", - .profile = "production", - .region = "ap-southeast-2", - }, - "with_profile_and_region"}, - ParsedS3URLTestCase{ - "s3://bucket/key?endpoint=https://minio.local&scheme=http", - { - .bucket = "bucket", - .key = "key", - /* TODO: Figure out what AWS SDK is doing when both endpointOverride and scheme are set. */ - .scheme = "http", - .endpoint = - ParsedURL{ - .scheme = "https", - .authority = ParsedURL::Authority{.host = "minio.local"}, - }, - }, - "with_absolute_endpoint_uri"}), - [](const ::testing::TestParamInfo & info) { return info.param.description; }); - -TEST(InvalidParsedS3URLTest, parseS3URLErrors) -{ - auto invalidBucketMatcher = ::testing::ThrowsMessage( - testing::HasSubstrIgnoreANSIMatcher("error: URI has a missing or invalid bucket name")); - - /* Empty bucket (authority) */ - ASSERT_THAT([]() { ParsedS3URL::parse("s3:///key"); }, invalidBucketMatcher); - /* Invalid bucket name */ - ASSERT_THAT([]() { ParsedS3URL::parse("s3://127.0.0.1"); }, invalidBucketMatcher); -} - -} // namespace nix - -#endif diff --git a/src/libstore-tests/serve-protocol.cc b/src/libstore-tests/serve-protocol.cc index 01d6058cbde..a63201164b7 100644 --- a/src/libstore-tests/serve-protocol.cc +++ b/src/libstore-tests/serve-protocol.cc @@ -20,9 +20,9 @@ struct ServeProtoTest : VersionedProtoTest { /** * For serializers that don't care about the minimum version, we - * used the oldest one: 1.0. + * used the oldest one: 2.5. */ - ServeProto::Version defaultVersion = 2 << 8 | 0; + ServeProto::Version defaultVersion = 2 << 8 | 5; }; VERSIONED_CHARACTERIZATION_TEST( @@ -127,17 +127,17 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_2, "build-result-2.2", 2 << 8 | 2, ({ using namespace std::literals::chrono_literals; std::tuple t{ - BuildResult{ - .status = BuildResult::OutputRejected, + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, .errorMsg = "no idea why", - }, - BuildResult{ - .status = BuildResult::NotDeterministic, + }}}, + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, .errorMsg = "no idea why", - }, - BuildResult{ - .status = BuildResult::Built, - }, + }}}, + BuildResult{.inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + }}}, }; t; })) @@ -145,20 +145,24 @@ VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_2, "build-result-2 VERSIONED_CHARACTERIZATION_TEST(ServeProtoTest, buildResult_2_3, "build-result-2.3", 2 << 8 | 3, ({ using namespace std::literals::chrono_literals; std::tuple t{ - BuildResult{ - .status = BuildResult::OutputRejected, + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, .errorMsg = "no idea why", - }, + }}}, BuildResult{ - .status = BuildResult::NotDeterministic, - .errorMsg = "no idea why", + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = true, + }}, .timesBuilt = 3, - .isNonDeterministic = true, .startTime = 30, .stopTime = 50, }, BuildResult{ - .status = BuildResult::Built, + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + }}, .startTime = 30, .stopTime = 50, }, @@ -170,48 +174,52 @@ VERSIONED_CHARACTERIZATION_TEST( ServeProtoTest, buildResult_2_6, "build-result-2.6", 2 << 8 | 6, ({ using namespace std::literals::chrono_literals; std::tuple t{ - BuildResult{ - .status = BuildResult::OutputRejected, + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, .errorMsg = "no idea why", - }, + }}}, BuildResult{ - .status = BuildResult::NotDeterministic, - .errorMsg = "no idea why", + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = true, + }}, .timesBuilt = 3, - .isNonDeterministic = true, .startTime = 30, .stopTime = 50, }, BuildResult{ - .status = BuildResult::Built, - .timesBuilt = 1, - .builtOutputs = - { + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = { - "foo", { - .id = - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + "foo", + { + .id = + DrvOutput{ + .drvHash = + Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, }, - }, - { - "bar", { - .id = - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + "bar", + { + .id = + DrvOutput{ + .drvHash = + Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, }, }, - }, + }}, + .timesBuilt = 1, .startTime = 30, .stopTime = 50, #if 0 @@ -274,7 +282,7 @@ VERSIONED_CHARACTERIZATION_TEST( info; }), ({ - ValidPathInfo info{ + auto info = ValidPathInfo::makeFromCA( store, "foo", FixedOutputInfo{ @@ -291,8 +299,7 @@ VERSIONED_CHARACTERIZATION_TEST( .self = true, }, }, - Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - }; + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=")); info.deriver = StorePath{ "g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar.drv", }; diff --git a/src/libstore-tests/store-reference.cc b/src/libstore-tests/store-reference.cc index 01b75f3d264..7b42b45a220 100644 --- a/src/libstore-tests/store-reference.cc +++ b/src/libstore-tests/store-reference.cc @@ -107,6 +107,13 @@ URI_TEST_READ(local_shorthand_1, localExample_1) URI_TEST_READ(local_shorthand_2, localExample_2) +URI_TEST( + local_shorthand_3, + (StoreReference{ + .variant = StoreReference::Local{}, + .params = {}, + })) + static StoreReference unixExample{ .variant = StoreReference::Specified{ @@ -134,4 +141,46 @@ URI_TEST( .params = {}, })) +URI_TEST( + daemon_shorthand, + (StoreReference{ + .variant = StoreReference::Daemon{}, + .params = {}, + })) + +static StoreReference sshLoopbackIPv6{ + .variant = + StoreReference::Specified{ + .scheme = "ssh", + .authority = "[::1]", + }, +}; + +URI_TEST_READ(ssh_unbracketed_ipv6_1, sshLoopbackIPv6) + +static StoreReference sshIPv6AuthorityWithUserinfo{ + .variant = + StoreReference::Specified{ + .scheme = "ssh", + .authority = "userinfo@[fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e]", + }, +}; + +URI_TEST_READ(ssh_unbracketed_ipv6_2, sshIPv6AuthorityWithUserinfo) + +static StoreReference sshIPv6AuthorityWithUserinfoAndParams{ + .variant = + StoreReference::Specified{ + .scheme = "ssh", + .authority = "userinfo@[fea5:23e1:3916:fc24:cb52:2837:2ecb:ea8e]", + }, + .params = + { + {"a", "b"}, + {"c", "d"}, + }, +}; + +URI_TEST_READ(ssh_unbracketed_ipv6_3, sshIPv6AuthorityWithUserinfoAndParams) + } // namespace nix diff --git a/src/libstore-tests/uds-remote-store.cc b/src/libstore-tests/uds-remote-store.cc index c215d6e18ff..415dfc4ac94 100644 --- a/src/libstore-tests/uds-remote-store.cc +++ b/src/libstore-tests/uds-remote-store.cc @@ -16,4 +16,30 @@ TEST(UDSRemoteStore, constructConfigWrongScheme) EXPECT_THROW(UDSRemoteStoreConfig("http", "/tmp/socket", {}), UsageError); } +TEST(UDSRemoteStore, constructConfig_to_string) +{ + UDSRemoteStoreConfig config{"unix", "", {}}; + EXPECT_EQ(config.getReference().to_string(), "daemon"); +} + +TEST(UDSRemoteStore, constructConfigWithParams) +{ + StoreConfig::Params params{{"max-connections", "1"}}; + UDSRemoteStoreConfig config{"unix", "/tmp/socket", params}; + auto storeReference = config.getReference(); + EXPECT_EQ(storeReference.to_string(), "unix:///tmp/socket?max-connections=1"); + EXPECT_EQ(storeReference.render(/*withParams=*/false), "unix:///tmp/socket"); + EXPECT_EQ(storeReference.params, params); +} + +TEST(UDSRemoteStore, constructConfigWithParamsNoPath) +{ + StoreConfig::Params params{{"max-connections", "1"}}; + UDSRemoteStoreConfig config{"unix", "", params}; + auto storeReference = config.getReference(); + EXPECT_EQ(storeReference.to_string(), "daemon?max-connections=1"); + EXPECT_EQ(storeReference.render(/*withParams=*/false), "daemon"); + EXPECT_EQ(storeReference.params, params); +} + } // namespace nix diff --git a/src/libstore-tests/worker-protocol.cc b/src/libstore-tests/worker-protocol.cc index a761c96dd81..489151c8c28 100644 --- a/src/libstore-tests/worker-protocol.cc +++ b/src/libstore-tests/worker-protocol.cc @@ -180,17 +180,17 @@ VERSIONED_CHARACTERIZATION_TEST( VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, buildResult_1_27, "build-result-1.27", 1 << 8 | 27, ({ using namespace std::literals::chrono_literals; std::tuple t{ - BuildResult{ - .status = BuildResult::OutputRejected, + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, .errorMsg = "no idea why", - }, - BuildResult{ - .status = BuildResult::NotDeterministic, + }}}, + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, .errorMsg = "no idea why", - }, - BuildResult{ - .status = BuildResult::Built, - }, + }}}, + BuildResult{.inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + }}}, }; t; })) @@ -199,16 +199,16 @@ VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_28, "build-result-1.28", 1 << 8 | 28, ({ using namespace std::literals::chrono_literals; std::tuple t{ - BuildResult{ - .status = BuildResult::OutputRejected, + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, .errorMsg = "no idea why", - }, - BuildResult{ - .status = BuildResult::NotDeterministic, + }}}, + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, .errorMsg = "no idea why", - }, - BuildResult{ - .status = BuildResult::Built, + }}}, + BuildResult{.inner{BuildResult::Success{ + .status = BuildResult::Success::Built, .builtOutputs = { { @@ -236,7 +236,7 @@ VERSIONED_CHARACTERIZATION_TEST( }, }, }, - }, + }}}, }; t; })) @@ -245,48 +245,52 @@ VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_29, "build-result-1.29", 1 << 8 | 29, ({ using namespace std::literals::chrono_literals; std::tuple t{ - BuildResult{ - .status = BuildResult::OutputRejected, + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, .errorMsg = "no idea why", - }, + }}}, BuildResult{ - .status = BuildResult::NotDeterministic, - .errorMsg = "no idea why", + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = true, + }}, .timesBuilt = 3, - .isNonDeterministic = true, .startTime = 30, .stopTime = 50, }, BuildResult{ - .status = BuildResult::Built, - .timesBuilt = 1, - .builtOutputs = - { + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = { - "foo", { - .id = - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + "foo", + { + .id = + DrvOutput{ + .drvHash = + Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, }, - }, - { - "bar", { - .id = - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + "bar", + { + .id = + DrvOutput{ + .drvHash = + Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, }, }, - }, + }}, + .timesBuilt = 1, .startTime = 30, .stopTime = 50, }, @@ -298,48 +302,52 @@ VERSIONED_CHARACTERIZATION_TEST( WorkerProtoTest, buildResult_1_37, "build-result-1.37", 1 << 8 | 37, ({ using namespace std::literals::chrono_literals; std::tuple t{ - BuildResult{ - .status = BuildResult::OutputRejected, + BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::OutputRejected, .errorMsg = "no idea why", - }, + }}}, BuildResult{ - .status = BuildResult::NotDeterministic, - .errorMsg = "no idea why", + .inner{BuildResult::Failure{ + .status = BuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = true, + }}, .timesBuilt = 3, - .isNonDeterministic = true, .startTime = 30, .stopTime = 50, }, BuildResult{ - .status = BuildResult::Built, - .timesBuilt = 1, - .builtOutputs = - { + .inner{BuildResult::Success{ + .status = BuildResult::Success::Built, + .builtOutputs = { - "foo", { - .id = - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "foo", - }, - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + "foo", + { + .id = + DrvOutput{ + .drvHash = + Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "foo", + }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-foo"}, + }, }, - }, - { - "bar", { - .id = - DrvOutput{ - .drvHash = - Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), - .outputName = "bar", - }, - .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + "bar", + { + .id = + DrvOutput{ + .drvHash = + Hash::parseSRI("sha256-b4afnqKCO9oWXgYHb9DeQ2berSwOjS27rSd9TxXDc/U="), + .outputName = "bar", + }, + .outPath = StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-bar"}, + }, }, }, - }, + }}, + .timesBuilt = 1, .startTime = 30, .stopTime = 50, .cpuUser = std::chrono::microseconds(500s), @@ -353,10 +361,10 @@ VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, keyedBuildResult_1_29, "keyed-b using namespace std::literals::chrono_literals; std::tuple t{ KeyedBuildResult{ - { - .status = KeyedBuildResult::OutputRejected, + {.inner{BuildResult::Failure{ + .status = KeyedBuildResult::Failure::OutputRejected, .errorMsg = "no idea why", - }, + }}}, /* .path = */ DerivedPath::Opaque{ StorePath{"g1w7hy3qg1w7hy3qg1w7hy3qg1w7hy3q-xxx"}, @@ -364,10 +372,12 @@ VERSIONED_CHARACTERIZATION_TEST(WorkerProtoTest, keyedBuildResult_1_29, "keyed-b }, KeyedBuildResult{ { - .status = KeyedBuildResult::NotDeterministic, - .errorMsg = "no idea why", + .inner{BuildResult::Failure{ + .status = KeyedBuildResult::Failure::NotDeterministic, + .errorMsg = "no idea why", + .isNonDeterministic = true, + }}, .timesBuilt = 3, - .isNonDeterministic = true, .startTime = 30, .stopTime = 50, }, @@ -515,7 +525,7 @@ VERSIONED_CHARACTERIZATION_TEST( info; }), ({ - ValidPathInfo info{ + auto info = ValidPathInfo::makeFromCA( store, "foo", FixedOutputInfo{ @@ -532,8 +542,7 @@ VERSIONED_CHARACTERIZATION_TEST( .self = true, }, }, - Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc="), - }; + Hash::parseSRI("sha256-FePFYIlMuycIXPZbWi7LGEiMmZSX9FMbaQenWBzm1Sc=")); info.registrationTime = 23423; info.narSize = 34878; info; diff --git a/src/libstore/async-path-writer.cc b/src/libstore/async-path-writer.cc new file mode 100644 index 00000000000..3271e7926a8 --- /dev/null +++ b/src/libstore/async-path-writer.cc @@ -0,0 +1,173 @@ +#include "nix/store/async-path-writer.hh" +#include "nix/util/archive.hh" + +#include +#include + +namespace nix { + +struct AsyncPathWriterImpl : AsyncPathWriter +{ + ref store; + + struct Item + { + StorePath storePath; + std::string contents; + std::string name; + Hash hash; + StorePathSet references; + RepairFlag repair; + std::promise promise; + }; + + struct State + { + std::vector items; + std::unordered_map> futures; + bool quit = false; + }; + + Sync state_; + + std::thread workerThread; + + std::condition_variable wakeupCV; + + AsyncPathWriterImpl(ref store) + : store(store) + { + workerThread = std::thread([&]() { + while (true) { + std::vector items; + + { + auto state(state_.lock()); + while (!state->quit && state->items.empty()) + state.wait(wakeupCV); + if (state->items.empty() && state->quit) + return; + std::swap(items, state->items); + } + + try { + writePaths(items); + for (auto & item : items) + item.promise.set_value(); + } catch (...) { + for (auto & item : items) + item.promise.set_exception(std::current_exception()); + } + } + }); + } + + virtual ~AsyncPathWriterImpl() + { + state_.lock()->quit = true; + wakeupCV.notify_all(); + workerThread.join(); + } + + StorePath + addPath(std::string contents, std::string name, StorePathSet references, RepairFlag repair, bool readOnly) override + { + auto hash = hashString(HashAlgorithm::SHA256, contents); + + auto storePath = store->makeFixedOutputPathFromCA( + name, + TextInfo{ + .hash = hash, + .references = references, + }); + + if (!readOnly) { + auto state(state_.lock()); + std::promise promise; + state->futures.insert_or_assign(storePath, promise.get_future()); + state->items.push_back( + Item{ + .storePath = storePath, + .contents = std::move(contents), + .name = std::move(name), + .hash = hash, + .references = std::move(references), + .repair = repair, + .promise = std::move(promise), + }); + wakeupCV.notify_all(); + } + + return storePath; + } + + void waitForPath(const StorePath & path) override + { + auto future = ({ + auto state = state_.lock(); + auto i = state->futures.find(path); + if (i == state->futures.end()) + return; + i->second; + }); + future.get(); + } + + void waitForAllPaths() override + { + auto futures = ({ + auto state(state_.lock()); + std::move(state->futures); + }); + for (auto & future : futures) + future.second.get(); + } + + void writePaths(const std::vector & items) + { +// FIXME: addMultipeToStore() shouldn't require a NAR hash. +#if 0 + Store::PathsSource sources; + RepairFlag repair = NoRepair; + + for (auto & item : items) { + ValidPathInfo info{item.storePath, Hash(HashAlgorithm::SHA256)}; + info.references = item.references; + info.ca = ContentAddress { + .method = ContentAddressMethod::Raw::Text, + .hash = item.hash, + }; + if (item.repair) repair = item.repair; + auto source = sinkToSource([&](Sink & sink) + { + dumpString(item.contents, sink); + }); + sources.push_back({std::move(info), std::move(source)}); + } + + Activity act(*logger, lvlDebug, actUnknown, fmt("adding %d paths to the store", items.size())); + + store->addMultipleToStore(std::move(sources), act, repair); +#endif + + for (auto & item : items) { + StringSource source(item.contents); + auto storePath = store->addToStoreFromDump( + source, + item.storePath.name(), + FileSerialisationMethod::Flat, + ContentAddressMethod::Raw::Text, + HashAlgorithm::SHA256, + item.references, + item.repair); + assert(storePath == item.storePath); + } + } +}; + +ref AsyncPathWriter::make(ref store) +{ + return make_ref(store); +} + +} // namespace nix diff --git a/src/libstore/aws-creds.cc b/src/libstore/aws-creds.cc new file mode 100644 index 00000000000..dc8584e1ba6 --- /dev/null +++ b/src/libstore/aws-creds.cc @@ -0,0 +1,178 @@ +#include "nix/store/aws-creds.hh" + +#if NIX_WITH_CURL_S3 + +# include +# include "nix/store/s3-url.hh" +# include "nix/util/finally.hh" +# include "nix/util/logging.hh" +# include "nix/util/url.hh" +# include "nix/util/util.hh" + +# include +# include +# include + +# include + +# include +# include +# include +# include + +namespace nix { + +namespace { + +static void initAwsCrt() +{ + struct CrtWrapper + { + Aws::Crt::ApiHandle apiHandle; + + CrtWrapper() + { + apiHandle.InitializeLogging(Aws::Crt::LogLevel::Warn, static_cast(nullptr)); + } + + ~CrtWrapper() + { + try { + // CRITICAL: Clear credential provider cache BEFORE AWS CRT shuts down + // This ensures all providers (which hold references to ClientBootstrap) + // are destroyed while AWS CRT is still valid + clearAwsCredentialsCache(); + // Now it's safe for ApiHandle destructor to run + } catch (...) { + ignoreExceptionInDestructor(); + } + } + }; + + static CrtWrapper crt; +} + +static AwsCredentials getCredentialsFromProvider(std::shared_ptr provider) +{ + if (!provider || !provider->IsValid()) { + throw AwsAuthError("AWS credential provider is invalid"); + } + + auto prom = std::make_shared>(); + auto fut = prom->get_future(); + + provider->GetCredentials([prom](std::shared_ptr credentials, int errorCode) { + if (errorCode != 0 || !credentials) { + prom->set_exception( + std::make_exception_ptr(AwsAuthError("Failed to resolve AWS credentials: error code %d", errorCode))); + } else { + auto accessKeyId = Aws::Crt::ByteCursorToStringView(credentials->GetAccessKeyId()); + auto secretAccessKey = Aws::Crt::ByteCursorToStringView(credentials->GetSecretAccessKey()); + auto sessionToken = Aws::Crt::ByteCursorToStringView(credentials->GetSessionToken()); + + std::optional sessionTokenStr; + if (!sessionToken.empty()) { + sessionTokenStr = std::string(sessionToken.data(), sessionToken.size()); + } + + prom->set_value(AwsCredentials( + std::string(accessKeyId.data(), accessKeyId.size()), + std::string(secretAccessKey.data(), secretAccessKey.size()), + sessionTokenStr)); + } + }); + + // AWS CRT GetCredentials is asynchronous and only guarantees the callback will be + // invoked if the initial call returns success. There's no documented timeout mechanism, + // so we add a timeout to prevent indefinite hanging if the callback is never called. + auto timeout = std::chrono::seconds(30); + if (fut.wait_for(timeout) == std::future_status::timeout) { + throw AwsAuthError( + "Timeout waiting for AWS credentials (%d seconds)", + std::chrono::duration_cast(timeout).count()); + } + + return fut.get(); // This will throw if set_exception was called +} + +// Global credential provider cache using boost's concurrent map +// Key: profile name (empty string for default profile) +using CredentialProviderCache = + boost::concurrent_flat_map>; + +static CredentialProviderCache credentialProviderCache; + +} // anonymous namespace + +AwsCredentials getAwsCredentials(const std::string & profile) +{ + // Get or create credential provider with caching + std::shared_ptr provider; + + // Try to find existing provider + credentialProviderCache.visit(profile, [&](const auto & pair) { provider = pair.second; }); + + if (!provider) { + // Create new provider if not found + debug( + "[pid=%d] creating new AWS credential provider for profile '%s'", + getpid(), + profile.empty() ? "(default)" : profile.c_str()); + + try { + initAwsCrt(); + + if (profile.empty()) { + Aws::Crt::Auth::CredentialsProviderChainDefaultConfig config; + config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap(); + provider = Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderChainDefault(config); + } else { + Aws::Crt::Auth::CredentialsProviderProfileConfig config; + config.Bootstrap = Aws::Crt::ApiHandle::GetOrCreateStaticDefaultClientBootstrap(); + // This is safe because the underlying C library will copy this string + // c.f. https://github.com/awslabs/aws-c-auth/blob/main/source/credentials_provider_profile.c#L220 + config.ProfileNameOverride = Aws::Crt::ByteCursorFromCString(profile.c_str()); + provider = Aws::Crt::Auth::CredentialsProvider::CreateCredentialsProviderProfile(config); + } + } catch (Error & e) { + e.addTrace( + {}, + "while creating AWS credentials provider for %s", + profile.empty() ? "default profile" : fmt("profile '%s'", profile)); + throw; + } + + if (!provider) { + throw AwsAuthError( + "Failed to create AWS credentials provider for %s", + profile.empty() ? "default profile" : fmt("profile '%s'", profile)); + } + + // Insert into cache (try_emplace is thread-safe and won't overwrite if another thread added it) + credentialProviderCache.try_emplace(profile, provider); + } + + return getCredentialsFromProvider(provider); +} + +void invalidateAwsCredentials(const std::string & profile) +{ + credentialProviderCache.erase(profile); +} + +void clearAwsCredentialsCache() +{ + credentialProviderCache.clear(); +} + +AwsCredentials preResolveAwsCredentials(const ParsedS3URL & s3Url) +{ + std::string profile = s3Url.profile.value_or(""); + + // Get credentials (automatically cached) + return getAwsCredentials(profile); +} + +} // namespace nix + +#endif diff --git a/src/libstore/binary-cache-store.cc b/src/libstore/binary-cache-store.cc index 0a44b0cf04f..badfb4b1484 100644 --- a/src/libstore/binary-cache-store.cc +++ b/src/libstore/binary-cache-store.cc @@ -125,11 +125,8 @@ void BinaryCacheStore::writeNarInfo(ref narInfo) upsertFile(narInfoFile, narInfo->to_string(*this), "text/x-nix-narinfo"); - { - auto state_(state.lock()); - state_->pathInfoCache.upsert( - std::string(narInfo->path.to_string()), PathInfoCacheValue{.value = std::shared_ptr(narInfo)}); - } + pathInfoCache->lock()->upsert( + std::string(narInfo->path.to_string()), PathInfoCacheValue{.value = std::shared_ptr(narInfo)}); if (diskCache) diskCache->upsertNarInfo( @@ -369,7 +366,7 @@ StorePath BinaryCacheStore::addToStoreFromDump( repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info{ + auto info = ValidPathInfo::makeFromCA( *this, name, ContentAddressWithReferences::fromParts( @@ -381,8 +378,7 @@ StorePath BinaryCacheStore::addToStoreFromDump( // without modulus .self = false, }), - nar.hash, - }; + nar.hash); info.narSize = nar.numBytesDigested; return info; }) @@ -487,7 +483,7 @@ StorePath BinaryCacheStore::addToStore( repair, CheckSigs, [&](HashResult nar) { - ValidPathInfo info{ + auto info = ValidPathInfo::makeFromCA( *this, name, ContentAddressWithReferences::fromParts( @@ -499,8 +495,7 @@ StorePath BinaryCacheStore::addToStore( // without modulus .self = false, }), - nar.hash, - }; + nar.hash); info.narSize = nar.numBytesDigested; return info; }) @@ -520,8 +515,14 @@ void BinaryCacheStore::queryRealisationUncached( if (!data) return (*callbackPtr)({}); - auto realisation = Realisation::fromJSON(nlohmann::json::parse(*data), outputInfoFilePath); - return (*callbackPtr)(std::make_shared(realisation)); + std::shared_ptr realisation; + try { + realisation = std::make_shared(nlohmann::json::parse(*data)); + } catch (Error & e) { + e.addTrace({}, "while parsing file '%s' as a realisation", outputInfoFilePath); + throw; + } + return (*callbackPtr)(std::move(realisation)); } catch (...) { callbackPtr->rethrow(); } @@ -535,14 +536,24 @@ void BinaryCacheStore::registerDrvOutput(const Realisation & info) if (diskCache) diskCache->upsertRealisation(config.getReference().render(/*FIXME withParams=*/false), info); auto filePath = realisationsPrefix + "/" + info.id.to_string() + ".doi"; - upsertFile(filePath, info.toJSON().dump(), "application/json"); + upsertFile(filePath, static_cast(info).dump(), "application/json"); } -ref BinaryCacheStore::getFSAccessor(bool requireValidPath) +ref BinaryCacheStore::getRemoteFSAccessor(bool requireValidPath) { return make_ref(ref(shared_from_this()), requireValidPath, config.localNarCache); } +ref BinaryCacheStore::getFSAccessor(bool requireValidPath) +{ + return getRemoteFSAccessor(requireValidPath); +} + +std::shared_ptr BinaryCacheStore::getFSAccessor(const StorePath & storePath, bool requireValidPath) +{ + return getRemoteFSAccessor(requireValidPath)->accessObject(storePath); +} + void BinaryCacheStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { /* Note: this is inherently racy since there is no locking on diff --git a/src/libstore/build-result.cc b/src/libstore/build-result.cc index 43c7adb11d6..e0eb1899722 100644 --- a/src/libstore/build-result.cc +++ b/src/libstore/build-result.cc @@ -1,8 +1,110 @@ #include "nix/store/build-result.hh" +#include + namespace nix { bool BuildResult::operator==(const BuildResult &) const noexcept = default; std::strong_ordering BuildResult::operator<=>(const BuildResult &) const noexcept = default; +bool BuildResult::Success::operator==(const BuildResult::Success &) const noexcept = default; +std::strong_ordering BuildResult::Success::operator<=>(const BuildResult::Success &) const noexcept = default; + +bool BuildResult::Failure::operator==(const BuildResult::Failure &) const noexcept = default; +std::strong_ordering BuildResult::Failure::operator<=>(const BuildResult::Failure &) const noexcept = default; + +static std::string_view statusToString(BuildResult::Success::Status status) +{ + switch (status) { + case BuildResult::Success::Built: + return "Built"; + case BuildResult::Success::Substituted: + return "Substituted"; + case BuildResult::Success::AlreadyValid: + return "AlreadyValid"; + case BuildResult::Success::ResolvesToAlreadyValid: + return "ResolvesToAlreadyValid"; + default: + unreachable(); + } +} + +static std::string_view statusToString(BuildResult::Failure::Status status) +{ + switch (status) { + case BuildResult::Failure::PermanentFailure: + return "PermanentFailure"; + case BuildResult::Failure::InputRejected: + return "InputRejected"; + case BuildResult::Failure::OutputRejected: + return "OutputRejected"; + case BuildResult::Failure::TransientFailure: + return "TransientFailure"; + case BuildResult::Failure::CachedFailure: + return "CachedFailure"; + case BuildResult::Failure::TimedOut: + return "TimedOut"; + case BuildResult::Failure::MiscFailure: + return "MiscFailure"; + case BuildResult::Failure::DependencyFailed: + return "DependencyFailed"; + case BuildResult::Failure::LogLimitExceeded: + return "LogLimitExceeded"; + case BuildResult::Failure::NotDeterministic: + return "NotDeterministic"; + case BuildResult::Failure::NoSubstituters: + return "NoSubstituters"; + case BuildResult::Failure::HashMismatch: + return "HashMismatch"; + default: + unreachable(); + } +} + +void to_json(nlohmann::json & json, const BuildResult & buildResult) +{ + json = nlohmann::json::object(); + // FIXME: change this to have `success` and `failure` objects. + if (auto success = buildResult.tryGetSuccess()) { + json["status"] = statusToString(success->status); + } else if (auto failure = buildResult.tryGetFailure()) { + json["status"] = statusToString(failure->status); + if (failure->errorMsg != "") + json["errorMsg"] = failure->errorMsg; + if (failure->isNonDeterministic) + json["isNonDeterministic"] = failure->isNonDeterministic; + } + if (buildResult.timesBuilt) + json["timesBuilt"] = buildResult.timesBuilt; + if (buildResult.startTime) + json["startTime"] = buildResult.startTime; + if (buildResult.stopTime) + json["stopTime"] = buildResult.stopTime; +} + +void to_json(nlohmann::json & json, const KeyedBuildResult & buildResult) +{ + to_json(json, (const BuildResult &) buildResult); + auto path = nlohmann::json::object(); + std::visit( + overloaded{ + [&](const DerivedPathOpaque & opaque) { path["opaque"] = opaque.path.to_string(); }, + [&](const DerivedPathBuilt & drv) { + path["drvPath"] = drv.drvPath->getBaseStorePath().to_string(); + path["outputs"] = drv.outputs; + auto outputs = nlohmann::json::object(); + if (auto success = buildResult.tryGetSuccess()) { + for (auto & [name, output] : success->builtOutputs) + outputs[name] = { + {"path", output.outPath.to_string()}, + {"signatures", output.signatures}, + }; + json["builtOutputs"] = std::move(outputs); + } + }, + }, + buildResult.path.raw()); + json["path"] = std::move(path); +} + } // namespace nix diff --git a/src/libstore/build/derivation-building-goal.cc b/src/libstore/build/derivation-building-goal.cc index a82f7f9281d..8712708eb07 100644 --- a/src/libstore/build/derivation-building-goal.cc +++ b/src/libstore/build/derivation-building-goal.cc @@ -1,4 +1,5 @@ #include "nix/store/build/derivation-building-goal.hh" +#include "nix/store/build/derivation-env-desugar.hh" #include "nix/store/build/derivation-trampoline-goal.hh" #ifndef _WIN32 // TODO enable build hook on Windows # include "nix/store/build/hook-instance.hh" @@ -53,24 +54,9 @@ DerivationBuildingGoal::~DerivationBuildingGoal() { /* Careful: we should never ever throw an exception from a destructor. */ - try { - killChild(); - } catch (...) { - ignoreExceptionInDestructor(); - } #ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows - if (builder) { - try { - builder->stopDaemon(); - } catch (...) { - ignoreExceptionInDestructor(); - } - try { - builder->deleteTmpDir(false); - } catch (...) { - ignoreExceptionInDestructor(); - } - } + if (builder) + builder.reset(); #endif try { closeLogFile(); @@ -94,22 +80,8 @@ void DerivationBuildingGoal::killChild() hook.reset(); #endif #ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows - if (builder && builder->pid != -1) { + if (builder && builder->killChild()) worker.childTerminated(this); - - // FIXME: move this into DerivationBuilder. - - /* If we're using a build user, then there is a tricky race - condition: if we kill the build user before the child has - done its setuid() to the build user uid, then it won't be - killed, and we'll potentially lock up in pid.wait(). So - also send a conventional kill to the child. */ - ::kill(-builder->pid, SIGKILL); /* ignore the result */ - - builder->killSandbox(true); - - builder->pid.wait(); - } #endif } @@ -118,7 +90,7 @@ void DerivationBuildingGoal::timedOut(Error && ex) killChild(); // We're not inside a coroutine, hence we can't use co_return here. // Thus we ignore the return value. - [[maybe_unused]] Done _ = done(BuildResult::TimedOut, {}, std::move(ex)); + [[maybe_unused]] Done _ = doneFailure({BuildResult::Failure::TimedOut, std::move(ex)}); } /** @@ -148,35 +120,13 @@ std::string showKnownOutputs(const StoreDirConfig & store, const Derivation & dr return msg; } +static void runPostBuildHook( + const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths); + /* At least one of the output paths could not be produced using a substitute. So we have to build instead. */ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution() { - /* Recheck at goal start. In particular, whereas before we were - given this information by the downstream goal, that cannot happen - anymore if the downstream goal only cares about one output, but - we care about all outputs. */ - auto outputHashes = staticOutputHashes(worker.evalStore, *drv); - for (auto & [outputName, outputHash] : outputHashes) { - InitialOutput v{.outputHash = outputHash}; - - /* TODO we might want to also allow randomizing the paths - for regular CA derivations, e.g. for sake of checking - determinism. */ - if (drv->type().isImpure()) { - v.known = InitialOutputStatus{ - .path = StorePath::random(outputPathName(drv->name, outputName)), - .status = PathStatus::Absent, - }; - } - - initialOutputs.insert({ - outputName, - std::move(v), - }); - } - checkPathValidity(); - Goals waitees; std::map, GoalPtr, value_comparison> inputGoals; @@ -255,7 +205,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution() nrFailed, nrFailed == 1 ? "dependency" : "dependencies"); msg += showKnownOutputs(worker.store, *drv); - co_return done(BuildResult::DependencyFailed, {}, Error(msg)); + co_return doneFailure(BuildError(BuildResult::Failure::DependencyFailed, msg)); } /* Gather information necessary for computing the closure and/or @@ -306,14 +256,18 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution() return std::nullopt; auto & buildResult = (*mEntry)->buildResult; - if (!buildResult.success()) - return std::nullopt; - - auto i = get(buildResult.builtOutputs, outputName); - if (!i) - return std::nullopt; - - return i->outPath; + return std::visit( + overloaded{ + [](const BuildResult::Failure &) -> std::optional { return std::nullopt; }, + [&](const BuildResult::Success & success) -> std::optional { + auto i = get(success.builtOutputs, outputName); + if (!i) + return std::nullopt; + + return i->outPath; + }, + }, + buildResult.inner); }); if (!attempt) { /* TODO (impure derivations-induced tech debt) (see below): @@ -356,24 +310,27 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution() auto resolvedResult = resolvedDrvGoal->buildResult; - SingleDrvOutputs builtOutputs; + // No `std::visit` for coroutines yet + if (auto * successP = resolvedResult.tryGetSuccess()) { + auto & success = *successP; + SingleDrvOutputs builtOutputs; - if (resolvedResult.success()) { + auto outputHashes = staticOutputHashes(worker.evalStore, *drv); auto resolvedHashes = staticOutputHashes(worker.store, drvResolved); StorePathSet outputPaths; for (auto & outputName : drvResolved.outputNames()) { - auto initialOutput = get(initialOutputs, outputName); + auto outputHash = get(outputHashes, outputName); auto resolvedHash = get(resolvedHashes, outputName); - if ((!initialOutput) || (!resolvedHash)) + if ((!outputHash) || (!resolvedHash)) throw Error( "derivation '%s' doesn't have expected output '%s' (derivation-goal.cc/resolve)", worker.store.printStorePath(drvPath), outputName); auto realisation = [&] { - auto take1 = get(resolvedResult.builtOutputs, outputName); + auto take1 = get(success.builtOutputs, outputName); if (take1) return *take1; @@ -393,7 +350,7 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution() if (!drv->type().isImpure()) { auto newRealisation = realisation; - newRealisation.id = DrvOutput{initialOutput->outputHash, outputName}; + newRealisation.id = DrvOutput{*outputHash, outputName}; newRealisation.signatures.clear(); if (!drv->type().isFixed()) { auto & drvStore = worker.evalStore.isValidPath(drvPath) ? worker.evalStore : worker.store; @@ -408,13 +365,20 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution() } runPostBuildHook(worker.store, *logger, drvPath, outputPaths); - } - auto status = resolvedResult.status; - if (status == BuildResult::AlreadyValid) - status = BuildResult::ResolvesToAlreadyValid; + auto status = success.status; + if (status == BuildResult::Success::AlreadyValid) + status = BuildResult::Success::ResolvesToAlreadyValid; - co_return done(status, std::move(builtOutputs)); + co_return doneSuccess(success.status, std::move(builtOutputs)); + } else if (resolvedResult.tryGetFailure()) { + co_return doneFailure({ + BuildResult::Failure::DependencyFailed, + "build of resolved derivation '%s' failed", + worker.store.printStorePath(pathResolved), + }); + } else + assert(false); } /* If we get this far, we know no dynamic drvs inputs */ @@ -458,136 +422,263 @@ Goal::Co DerivationBuildingGoal::gaveUpOnSubstitution() co_return tryToBuild(); } -void DerivationBuildingGoal::started() +Goal::Co DerivationBuildingGoal::tryToBuild() { - auto msg = - fmt(buildMode == bmRepair ? "repairing outputs of '%s'" - : buildMode == bmCheck ? "checking outputs of '%s'" - : "building '%s'", - worker.store.printStorePath(drvPath)); - fmt("building '%s'", worker.store.printStorePath(drvPath)); + std::map initialOutputs; + + /* Recheck at this point. In particular, whereas before we were + given this information by the downstream goal, that cannot happen + anymore if the downstream goal only cares about one output, but + we care about all outputs. */ + auto outputHashes = staticOutputHashes(worker.evalStore, *drv); + for (auto & [outputName, outputHash] : outputHashes) { + InitialOutput v{.outputHash = outputHash}; + + /* TODO we might want to also allow randomizing the paths + for regular CA derivations, e.g. for sake of checking + determinism. */ + if (drv->type().isImpure()) { + v.known = InitialOutputStatus{ + .path = StorePath::random(outputPathName(drv->name, outputName)), + .status = PathStatus::Absent, + }; + } + + initialOutputs.insert({ + outputName, + std::move(v), + }); + } + checkPathValidity(initialOutputs); + + auto started = [&]() { + auto msg = + fmt(buildMode == bmRepair ? "repairing outputs of '%s'" + : buildMode == bmCheck ? "checking outputs of '%s'" + : "building '%s'", + worker.store.printStorePath(drvPath)); + fmt("building '%s'", worker.store.printStorePath(drvPath)); #ifndef _WIN32 // TODO enable build hook on Windows - if (hook) - msg += fmt(" on '%s'", machineName); + if (hook) + msg += fmt(" on '%s'", hook->machineName); #endif - act = std::make_unique( - *logger, - lvlInfo, - actBuild, - msg, - Logger::Fields{ - worker.store.printStorePath(drvPath), + act = std::make_unique( + *logger, + lvlInfo, + actBuild, + msg, + Logger::Fields{ + worker.store.printStorePath(drvPath), #ifndef _WIN32 // TODO enable build hook on Windows - hook ? machineName : + hook ? hook->machineName : #endif - "", - 1, - 1}); - mcRunningBuilds = std::make_unique>(worker.runningBuilds); - worker.updateProgress(); -} + "", + 1, + 1}); + mcRunningBuilds = std::make_unique>(worker.runningBuilds); + worker.updateProgress(); + }; -Goal::Co DerivationBuildingGoal::tryToBuild() -{ - trace("trying to build"); - - /* Obtain locks on all output paths, if the paths are known a priori. - - The locks are automatically released when we exit this function or Nix - crashes. If we can't acquire the lock, then continue; hopefully some - other goal can start a build, and if not, the main loop will sleep a few - seconds and then retry this goal. */ - PathSet lockFiles; - /* FIXME: Should lock something like the drv itself so we don't build same - CA drv concurrently */ - if (dynamic_cast(&worker.store)) { - /* If we aren't a local store, we might need to use the local store as - a build remote, but that would cause a deadlock. */ - /* FIXME: Make it so we can use ourselves as a build remote even if we - are the local store (separate locking for building vs scheduling? */ - /* FIXME: find some way to lock for scheduling for the other stores so - a forking daemon with --store still won't farm out redundant builds. - */ - for (auto & i : drv->outputsAndOptPaths(worker.store)) { - if (i.second.second) - lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); - else - lockFiles.insert(worker.store.Store::toRealPath(drvPath) + "." + i.first); - } - } + /** + * Activity that denotes waiting for a lock. + */ + std::unique_ptr actLock; - if (!outputLocks.lockPaths(lockFiles, "", false)) { - Activity act(*logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); + /** + * Locks on (fixed) output paths. + */ + PathLocks outputLocks; - /* Wait then try locking again, repeat until success (returned - boolean is true). */ - do { - co_await waitForAWhile(); - } while (!outputLocks.lockPaths(lockFiles, "", false)); - } + bool useHook; - /* Now check again whether the outputs are valid. This is because - another process may have started building in parallel. After - it has finished and released the locks, we can (and should) - reuse its results. (Strictly speaking the first check can be - omitted, but that would be less efficient.) Note that since we - now hold the locks on the output paths, no other process can - build this derivation, so no further checks are necessary. */ - auto [allValid, validOutputs] = checkPathValidity(); - - if (buildMode != bmCheck && allValid) { - debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); - outputLocks.setDeletion(true); - outputLocks.unlock(); - co_return done(BuildResult::AlreadyValid, std::move(validOutputs)); - } + while (true) { + trace("trying to build"); + + /* Obtain locks on all output paths, if the paths are known a priori. + + The locks are automatically released when we exit this function or Nix + crashes. If we can't acquire the lock, then continue; hopefully some + other goal can start a build, and if not, the main loop will sleep a few + seconds and then retry this goal. */ + PathSet lockFiles; + /* FIXME: Should lock something like the drv itself so we don't build same + CA drv concurrently */ + if (dynamic_cast(&worker.store)) { + /* If we aren't a local store, we might need to use the local store as + a build remote, but that would cause a deadlock. */ + /* FIXME: Make it so we can use ourselves as a build remote even if we + are the local store (separate locking for building vs scheduling? */ + /* FIXME: find some way to lock for scheduling for the other stores so + a forking daemon with --store still won't farm out redundant builds. + */ + for (auto & i : drv->outputsAndOptPaths(worker.store)) { + if (i.second.second) + lockFiles.insert(worker.store.Store::toRealPath(*i.second.second)); + else + lockFiles.insert(worker.store.Store::toRealPath(drvPath) + "." + i.first); + } + } - /* If any of the outputs already exist but are not valid, delete - them. */ - for (auto & [_, status] : initialOutputs) { - if (!status.known || status.known->isValid()) - continue; - auto storePath = status.known->path; - debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); - deletePath(worker.store.Store::toRealPath(storePath)); - } + if (!outputLocks.lockPaths(lockFiles, "", false)) { + Activity act( + *logger, lvlWarn, actBuildWaiting, fmt("waiting for lock on %s", Magenta(showPaths(lockFiles)))); - /* Don't do a remote build if the derivation has the attribute - `preferLocalBuild' set. Also, check and repair modes are only - supported for local builds. */ - bool buildLocally = - (buildMode != bmNormal || drvOptions->willBuildLocally(worker.store, *drv)) && settings.maxBuildJobs.get() != 0; - - if (!buildLocally) { - switch (tryBuildHook()) { - case rpAccept: - /* Yes, it has started doing so. Wait until we get - EOF from the hook. */ - actLock.reset(); - buildResult.startTime = time(0); // inexact - started(); - co_await Suspend{}; - co_return hookDone(); - case rpPostpone: - /* Not now; wait until at least one child finishes or - the wake-up timeout expires. */ - if (!actLock) - actLock = std::make_unique( - *logger, - lvlWarn, - actBuildWaiting, - fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); + /* Wait then try locking again, repeat until success (returned + boolean is true). */ + do { + co_await waitForAWhile(); + } while (!outputLocks.lockPaths(lockFiles, "", false)); + } + + /* Now check again whether the outputs are valid. This is because + another process may have started building in parallel. After + it has finished and released the locks, we can (and should) + reuse its results. (Strictly speaking the first check can be + omitted, but that would be less efficient.) Note that since we + now hold the locks on the output paths, no other process can + build this derivation, so no further checks are necessary. */ + auto [allValid, validOutputs] = checkPathValidity(initialOutputs); + + if (buildMode != bmCheck && allValid) { + debug("skipping build of derivation '%s', someone beat us to it", worker.store.printStorePath(drvPath)); + outputLocks.setDeletion(true); outputLocks.unlock(); - co_await waitForAWhile(); - co_return tryToBuild(); - case rpDecline: - /* We should do it ourselves. */ - break; + co_return doneSuccess(BuildResult::Success::AlreadyValid, std::move(validOutputs)); + } + + /* If any of the outputs already exist but are not valid, delete + them. */ + for (auto & [_, status] : initialOutputs) { + if (!status.known || status.known->isValid()) + continue; + auto storePath = status.known->path; + debug("removing invalid path '%s'", worker.store.printStorePath(status.known->path)); + deletePath(worker.store.Store::toRealPath(storePath)); } + + /* Don't do a remote build if the derivation has the attribute + `preferLocalBuild' set. Also, check and repair modes are only + supported for local builds. */ + bool buildLocally = (buildMode != bmNormal || drvOptions->willBuildLocally(worker.store, *drv)) + && settings.maxBuildJobs.get() != 0; + + if (buildLocally) { + useHook = false; + } else { + switch (tryBuildHook(initialOutputs)) { + case rpAccept: + /* Yes, it has started doing so. Wait until we get + EOF from the hook. */ + useHook = true; + break; + case rpPostpone: + /* Not now; wait until at least one child finishes or + the wake-up timeout expires. */ + if (!actLock) + actLock = std::make_unique( + *logger, + lvlWarn, + actBuildWaiting, + fmt("waiting for a machine to build '%s'", Magenta(worker.store.printStorePath(drvPath)))); + outputLocks.unlock(); + co_await waitForAWhile(); + continue; + case rpDecline: + /* We should do it ourselves. */ + useHook = false; + break; + } + } + break; } actLock.reset(); + if (useHook) { + buildResult.startTime = time(0); // inexact + started(); + co_await Suspend{}; + +#ifndef _WIN32 + assert(hook); +#endif + + trace("hook build done"); + + /* Since we got an EOF on the logger pipe, the builder is presumed + to have terminated. In fact, the builder could also have + simply have closed its end of the pipe, so just to be sure, + kill it. */ + int status = +#ifndef _WIN32 // TODO enable build hook on Windows + hook->pid.kill(); +#else + 0; +#endif + + debug("build hook for '%s' finished", worker.store.printStorePath(drvPath)); + + buildResult.timesBuilt++; + buildResult.stopTime = time(0); + + /* So the child is gone now. */ + worker.childTerminated(this); + + /* Close the read side of the logger pipe. */ +#ifndef _WIN32 // TODO enable build hook on Windows + hook->builderOut.readSide.close(); + hook->fromHook.readSide.close(); +#endif + + /* Close the log file. */ + closeLogFile(); + + /* Check the exit status. */ + if (!statusOk(status)) { + auto e = fixupBuilderFailureErrorMessage({BuildResult::Failure::MiscFailure, status, ""}); + + outputLocks.unlock(); + + /* TODO (once again) support fine-grained error codes, see issue #12641. */ + + co_return doneFailure(std::move(e)); + } + + /* Compute the FS closure of the outputs and register them as + being valid. */ + auto builtOutputs = + /* When using a build hook, the build hook can register the output + as valid (by doing `nix-store --import'). If so we don't have + to do anything here. + + We can only early return when the outputs are known a priori. For + floating content-addressing derivations this isn't the case. + + Aborts if any output is not valid or corrupt, and otherwise + returns a 'SingleDrvOutputs' structure containing all outputs. + */ + [&] { + auto [allValid, validOutputs] = checkPathValidity(initialOutputs); + if (!allValid) + throw Error("some outputs are unexpectedly invalid"); + return validOutputs; + }(); + + StorePathSet outputPaths; + for (auto & [_, output] : builtOutputs) + outputPaths.insert(output.outPath); + runPostBuildHook(worker.store, *logger, drvPath, outputPaths); + + /* It is now safe to delete the lock files, since all future + lockers will see that the output paths are valid; they will + not create new lock files with the same names as the old + (unlinked) lock files. */ + outputLocks.setDeletion(true); + outputLocks.unlock(); + + co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs)); + } + co_await yield(); if (!dynamic_cast(&worker.store)) { @@ -603,12 +694,13 @@ Goal::Co DerivationBuildingGoal::tryToBuild() #ifdef _WIN32 // TODO enable `DerivationBuilder` on Windows throw UnimplementedError("building derivations is not yet implemented on Windows"); #else + assert(!hook); + + Descriptor builderOut; // Will continue here while waiting for a build user below while (true) { - assert(!hook); - unsigned int curBuilds = worker.getNrLocalBuilds(); if (curBuilds >= settings.maxBuildJobs) { outputLocks.unlock(); @@ -633,31 +725,11 @@ Goal::Co DerivationBuildingGoal::tryToBuild() ~DerivationBuildingGoalCallbacks() override = default; - void childStarted(Descriptor builderOut) override - { - goal.worker.childStarted(goal.shared_from_this(), {builderOut}, true, true); - } - void childTerminated() override { goal.worker.childTerminated(&goal); } - void noteHashMismatch() override - { - goal.worker.hashMismatch = true; - } - - void noteCheckMismatch() override - { - goal.worker.checkMismatch = true; - } - - void markContentsGood(const StorePath & path) override - { - goal.worker.markContentsGood(path); - } - Path openLogFile() override { return goal.openLogFile(); @@ -667,19 +739,13 @@ Goal::Co DerivationBuildingGoal::tryToBuild() { goal.closeLogFile(); } - - void appendLogTailErrorMsg(std::string & msg) override - { - goal.appendLogTailErrorMsg(msg); - } }; auto * localStoreP = dynamic_cast(&worker.store); assert(localStoreP); decltype(DerivationBuilderParams::defaultPathsInChroot) defaultPathsInChroot = settings.sandboxPaths.get(); - decltype(DerivationBuilderParams::finalEnv) finalEnv; - decltype(DerivationBuilderParams::extraFiles) extraFiles; + DesugaredEnv desugaredEnv; /* Add the closure of store paths to the chroot. */ StorePathSet closure; @@ -698,58 +764,11 @@ Goal::Co DerivationBuildingGoal::tryToBuild() } try { - if (drv->structuredAttrs) { - auto json = drv->structuredAttrs->prepareStructuredAttrs( - worker.store, *drvOptions, inputPaths, drv->outputs); - - finalEnv.insert_or_assign( - "NIX_ATTRS_SH_FILE", - DerivationBuilderParams::EnvEntry{ - .nameOfPassAsFile = ".attrs.sh", - .value = StructuredAttrs::writeShell(json), - }); - finalEnv.insert_or_assign( - "NIX_ATTRS_JSON_FILE", - DerivationBuilderParams::EnvEntry{ - .nameOfPassAsFile = ".attrs.json", - .value = json.dump(), - }); - } else { - /* In non-structured mode, set all bindings either directory in the - environment or via a file, as specified by - `DerivationOptions::passAsFile`. */ - for (auto & [envName, envValue] : drv->env) { - if (drvOptions->passAsFile.find(envName) == drvOptions->passAsFile.end()) { - finalEnv.insert_or_assign( - envName, - DerivationBuilderParams::EnvEntry{ - .nameOfPassAsFile = std::nullopt, - .value = envValue, - }); - } else { - auto hash = hashString(HashAlgorithm::SHA256, envName); - finalEnv.insert_or_assign( - envName + "Path", - DerivationBuilderParams::EnvEntry{ - .nameOfPassAsFile = ".attr-" + hash.to_string(HashFormat::Nix32, false), - .value = envValue, - }); - } - } - - /* Handle exportReferencesGraph(), if set. */ - for (auto & [fileName, storePaths] : drvOptions->getParsedExportReferencesGraph(worker.store)) { - /* Write closure info to . */ - extraFiles.insert_or_assign( - fileName, - worker.store.makeValidityRegistration( - worker.store.exportReferences(storePaths, inputPaths), false, false)); - } - } + desugaredEnv = DesugaredEnv::create(worker.store, *drv, *drvOptions, inputPaths); } catch (BuildError & e) { outputLocks.unlock(); worker.permanentFailure = true; - co_return done(BuildResult::InputRejected, {}, std::move(e)); + co_return doneFailure(std::move(e)); } /* If we have to wait and retry (see below), then `builder` will @@ -758,20 +777,31 @@ Goal::Co DerivationBuildingGoal::tryToBuild() *localStoreP, std::make_unique(*this, builder), DerivationBuilderParams{ - drvPath, - buildMode, - buildResult, - *drv, - *drvOptions, - inputPaths, - initialOutputs, - std::move(defaultPathsInChroot), - std::move(finalEnv), - std::move(extraFiles), + .drvPath = drvPath, + .buildResult = buildResult, + .drv = *drv, + .drvOptions = *drvOptions, + .inputPaths = inputPaths, + .initialOutputs = initialOutputs, + .buildMode = buildMode, + .defaultPathsInChroot = std::move(defaultPathsInChroot), + .systemFeatures = worker.store.config.systemFeatures.get(), + .desugaredEnv = std::move(desugaredEnv), + .act = act, }); } - if (!builder->prepareBuild()) { + std::optional builderOutOpt; + try { + /* Okay, we have to build. */ + builderOutOpt = builder->startBuild(); + } catch (BuildError & e) { + builder.reset(); + outputLocks.unlock(); + worker.permanentFailure = true; + co_return doneFailure(std::move(e)); // InputRejected + } + if (!builderOutOpt) { if (!actLock) actLock = std::make_unique( *logger, @@ -780,50 +810,76 @@ Goal::Co DerivationBuildingGoal::tryToBuild() fmt("waiting for a free build user ID for '%s'", Magenta(worker.store.printStorePath(drvPath)))); co_await waitForAWhile(); continue; - } + } else { + builderOut = *std::move(builderOutOpt); + }; break; } actLock.reset(); - try { - - /* Okay, we have to build. */ - builder->startBuilder(); - - } catch (BuildError & e) { - builder.reset(); - outputLocks.unlock(); - worker.permanentFailure = true; - co_return done(BuildResult::InputRejected, {}, std::move(e)); - } + worker.childStarted(shared_from_this(), {builderOut}, true, true); started(); co_await Suspend{}; trace("build done"); - auto res = builder->unprepareBuild(); - // N.B. cannot use `std::visit` with co-routine return - if (auto * ste = std::get_if<0>(&res)) { + SingleDrvOutputs builtOutputs; + try { + builtOutputs = builder->unprepareBuild(); + } catch (BuilderFailureError & e) { + builder.reset(); + outputLocks.unlock(); + co_return doneFailure(fixupBuilderFailureErrorMessage(std::move(e))); + } catch (BuildError & e) { + builder.reset(); outputLocks.unlock(); - co_return done(std::move(ste->first), {}, std::move(ste->second)); - } else if (auto * builtOutputs = std::get_if<1>(&res)) { +// Allow selecting a subset of enum values +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wswitch-enum" + switch (e.status) { + case BuildResult::Failure::HashMismatch: + worker.hashMismatch = true; + /* See header, the protocols don't know about `HashMismatch` + yet, so change it to `OutputRejected`, which they expect + for this case (hash mismatch is a type of output + rejection). */ + e.status = BuildResult::Failure::OutputRejected; + break; + case BuildResult::Failure::NotDeterministic: + worker.checkMismatch = true; + break; + default: + /* Other statuses need no adjusting */ + break; + } +# pragma GCC diagnostic pop + co_return doneFailure(std::move(e)); + } + { + builder.reset(); + StorePathSet outputPaths; + for (auto & [_, output] : builtOutputs) { + // for sake of `bmRepair` + worker.markContentsGood(output.outPath); + outputPaths.insert(output.outPath); + } + runPostBuildHook(worker.store, *logger, drvPath, outputPaths); + /* It is now safe to delete the lock files, since all future lockers will see that the output paths are valid; they will not create new lock files with the same names as the old (unlinked) lock files. */ outputLocks.setDeletion(true); outputLocks.unlock(); - co_return done(BuildResult::Built, std::move(*builtOutputs)); - } else { - unreachable(); + co_return doneSuccess(BuildResult::Success::Built, std::move(builtOutputs)); } #endif } -void runPostBuildHook( +static void runPostBuildHook( const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths) { auto hook = settings.postBuildHook; @@ -889,8 +945,16 @@ void runPostBuildHook( }); } -void DerivationBuildingGoal::appendLogTailErrorMsg(std::string & msg) +BuildError DerivationBuildingGoal::fixupBuilderFailureErrorMessage(BuilderFailureError e) { + auto msg = + fmt("Cannot build '%s'.\n" + "Reason: " ANSI_RED "builder %s" ANSI_NORMAL ".", + Magenta(worker.store.printStorePath(drvPath)), + statusToString(e.builderStatus)); + + msg += showKnownOutputs(worker.store, *drv); + if (!logger->isVerbose() && !logTail.empty()) { msg += fmt("\nLast %d log lines:\n", logTail.size()); for (auto & line : logTail) { @@ -898,7 +962,7 @@ void DerivationBuildingGoal::appendLogTailErrorMsg(std::string & msg) msg += line; msg += "\n"; } - auto nixLogCommand = experimentalFeatureSettings.isEnabled(Xp::NixCommand) ? "nix log" : "nix-store -l"; + auto nixLogCommand = "nix log"; // The command is on a separate line for easy copying, such as with triple click. // This message will be indented elsewhere, so removing the indentation before the // command will not put it at the start of the line unfortunately. @@ -907,91 +971,13 @@ void DerivationBuildingGoal::appendLogTailErrorMsg(std::string & msg) nixLogCommand, worker.store.printStorePath(drvPath)); } -} - -Goal::Co DerivationBuildingGoal::hookDone() -{ -#ifndef _WIN32 - assert(hook); -#endif - - trace("hook build done"); - - /* Since we got an EOF on the logger pipe, the builder is presumed - to have terminated. In fact, the builder could also have - simply have closed its end of the pipe, so just to be sure, - kill it. */ - int status = -#ifndef _WIN32 // TODO enable build hook on Windows - hook->pid.kill(); -#else - 0; -#endif - - debug("build hook for '%s' finished", worker.store.printStorePath(drvPath)); - - buildResult.timesBuilt++; - buildResult.stopTime = time(0); - - /* So the child is gone now. */ - worker.childTerminated(this); - /* Close the read side of the logger pipe. */ -#ifndef _WIN32 // TODO enable build hook on Windows - hook->builderOut.readSide.close(); - hook->fromHook.readSide.close(); -#endif - - /* Close the log file. */ - closeLogFile(); - - /* Check the exit status. */ - if (!statusOk(status)) { - auto msg = - fmt("Cannot build '%s'.\n" - "Reason: " ANSI_RED "builder %s" ANSI_NORMAL ".", - Magenta(worker.store.printStorePath(drvPath)), - statusToString(status)); - - msg += showKnownOutputs(worker.store, *drv); - - appendLogTailErrorMsg(msg); - - outputLocks.unlock(); + msg += e.extraMsgAfter; - /* TODO (once again) support fine-grained error codes, see issue #12641. */ - - co_return done(BuildResult::MiscFailure, {}, BuildError(msg)); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - auto builtOutputs = - /* When using a build hook, the build hook can register the output - as valid (by doing `nix-store --import'). If so we don't have - to do anything here. - - We can only early return when the outputs are known a priori. For - floating content-addressing derivations this isn't the case. - */ - assertPathValidity(); - - StorePathSet outputPaths; - for (auto & [_, output] : builtOutputs) - outputPaths.insert(output.outPath); - runPostBuildHook(worker.store, *logger, drvPath, outputPaths); - - /* It is now safe to delete the lock files, since all future - lockers will see that the output paths are valid; they will - not create new lock files with the same names as the old - (unlinked) lock files. */ - outputLocks.setDeletion(true); - outputLocks.unlock(); - - co_return done(BuildResult::Built, std::move(builtOutputs)); + return BuildError{e.status, msg}; } -HookReply DerivationBuildingGoal::tryBuildHook() +HookReply DerivationBuildingGoal::tryBuildHook(const std::map & initialOutputs) { #ifdef _WIN32 // TODO enable build hook on Windows return rpDecline; @@ -1059,7 +1045,7 @@ HookReply DerivationBuildingGoal::tryBuildHook() hook = std::move(worker.hook); try { - machineName = readLine(hook->fromHook.readSide.get()); + hook->machineName = readLine(hook->fromHook.readSide.get()); } catch (Error & e) { e.addTrace({}, "while reading the machine name from the build hook"); throw; @@ -1170,10 +1156,11 @@ void DerivationBuildingGoal::handleChildOutput(Descriptor fd, std::string_view d killChild(); // We're not inside a coroutine, hence we can't use co_return here. // Thus we ignore the return value. - [[maybe_unused]] Done _ = done( - BuildResult::LogLimitExceeded, - {}, - Error("%s killed after writing more than %d bytes of log output", getName(), settings.maxLogSize)); + [[maybe_unused]] Done _ = doneFailure(BuildError( + BuildResult::Failure::LogLimitExceeded, + "%s killed after writing more than %d bytes of log output", + getName(), + settings.maxLogSize)); return; } @@ -1269,7 +1256,8 @@ std::map> DerivationBuildingGoal::queryPar return res; } -std::pair DerivationBuildingGoal::checkPathValidity() +std::pair +DerivationBuildingGoal::checkPathValidity(std::map & initialOutputs) { if (drv->type().isImpure()) return {false, {}}; @@ -1326,47 +1314,49 @@ std::pair DerivationBuildingGoal::checkPathValidity() return {allValid, validOutputs}; } -SingleDrvOutputs DerivationBuildingGoal::assertPathValidity() +Goal::Done DerivationBuildingGoal::doneSuccess(BuildResult::Success::Status status, SingleDrvOutputs builtOutputs) { - auto [allValid, validOutputs] = checkPathValidity(); - if (!allValid) - throw Error("some outputs are unexpectedly invalid"); - return validOutputs; + buildResult.inner = BuildResult::Success{ + .status = status, + .builtOutputs = std::move(builtOutputs), + }; + + mcRunningBuilds.reset(); + + if (status == BuildResult::Success::Built) + worker.doneBuilds++; + + worker.updateProgress(); + + return amDone(ecSuccess, std::nullopt); } -Goal::Done -DerivationBuildingGoal::done(BuildResult::Status status, SingleDrvOutputs builtOutputs, std::optional ex) +Goal::Done DerivationBuildingGoal::doneFailure(BuildError ex) { - outputLocks.unlock(); - buildResult.status = status; - if (ex) - buildResult.errorMsg = fmt("%s", Uncolored(ex->info().msg)); - if (buildResult.status == BuildResult::TimedOut) - worker.timedOut = true; - if (buildResult.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; + buildResult.inner = BuildResult::Failure{ + .status = ex.status, + .errorMsg = fmt("%s", Uncolored(ex.info().msg)), + }; + + logger->result( + act ? act->id : getCurActivity(), + resBuildResult, + nlohmann::json(KeyedBuildResult( + buildResult, + DerivedPath::Built{.drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}}))); mcRunningBuilds.reset(); - if (buildResult.success()) { - buildResult.builtOutputs = std::move(builtOutputs); - if (status == BuildResult::Built) - worker.doneBuilds++; - } else { - if (status != BuildResult::DependencyFailed) - worker.failedBuilds++; - } + if (ex.status == BuildResult::Failure::TimedOut) + worker.timedOut = true; + if (ex.status == BuildResult::Failure::PermanentFailure) + worker.permanentFailure = true; + if (ex.status != BuildResult::Failure::DependencyFailed) + worker.failedBuilds++; worker.updateProgress(); - auto traceBuiltOutputsFile = getEnv("_NIX_TRACE_BUILT_OUTPUTS").value_or(""); - if (traceBuiltOutputsFile != "") { - std::fstream fs; - fs.open(traceBuiltOutputsFile, std::fstream::out); - fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; - } - - return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); + return amDone(ecFailed, {std::move(ex)}); } } // namespace nix diff --git a/src/libstore/build/derivation-check.cc b/src/libstore/build/derivation-check.cc new file mode 100644 index 00000000000..64e824368bd --- /dev/null +++ b/src/libstore/build/derivation-check.cc @@ -0,0 +1,202 @@ +#include + +#include "nix/store/store-api.hh" +#include "nix/store/build-result.hh" + +#include "derivation-check.hh" + +namespace nix { + +void checkOutputs( + Store & store, + const StorePath & drvPath, + const decltype(Derivation::outputs) & drvOutputs, + const decltype(DerivationOptions::outputChecks) & outputChecks, + const std::map & outputs, + Activity & act) +{ + std::map outputsByPath; + for (auto & output : outputs) + outputsByPath.emplace(store.printStorePath(output.second.path), output.second); + + for (auto & pair : outputs) { + // We can't use auto destructuring here because + // clang-tidy seems to complain about it. + const std::string & outputName = pair.first; + const auto & info = pair.second; + + auto * outputSpec = get(drvOutputs, outputName); + assert(outputSpec); + + if (const auto * dof = std::get_if(&outputSpec->raw)) { + auto & wanted = dof->ca.hash; + + /* Check wanted hash */ + assert(info.ca); + auto & got = info.ca->hash; + if (wanted != got) { + /* Throw an error after registering the path as + valid. */ + act.result( + resHashMismatch, + { + {"storePath", store.printStorePath(drvPath)}, + {"wanted", wanted}, + {"got", got}, + }); + throw BuildError( + BuildResult::Failure::HashMismatch, + "hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", + store.printStorePath(drvPath), + wanted.to_string(HashFormat::SRI, true), + got.to_string(HashFormat::SRI, true)); + } + if (!info.references.empty()) { + auto numViolations = info.references.size(); + throw BuildError( + BuildResult::Failure::HashMismatch, + "fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", + store.printStorePath(drvPath), + numViolations, + store.printStorePath(*info.references.begin())); + } + } + + /* Compute the closure and closure size of some output. This + is slightly tricky because some of its references (namely + other outputs) may not be valid yet. */ + auto getClosure = [&](const StorePath & path) { + uint64_t closureSize = 0; + StorePathSet pathsDone; + std::queue pathsLeft; + pathsLeft.push(path); + + while (!pathsLeft.empty()) { + auto path = pathsLeft.front(); + pathsLeft.pop(); + if (!pathsDone.insert(path).second) + continue; + + auto i = outputsByPath.find(store.printStorePath(path)); + if (i != outputsByPath.end()) { + closureSize += i->second.narSize; + for (auto & ref : i->second.references) + pathsLeft.push(ref); + } else { + auto info = store.queryPathInfo(path); + closureSize += info->narSize; + for (auto & ref : info->references) + pathsLeft.push(ref); + } + } + + return std::make_pair(std::move(pathsDone), closureSize); + }; + + auto applyChecks = [&](const DerivationOptions::OutputChecks & checks) { + if (checks.maxSize && info.narSize > *checks.maxSize) + throw BuildError( + BuildResult::Failure::OutputRejected, + "path '%s' is too large at %d bytes; limit is %d bytes", + store.printStorePath(info.path), + info.narSize, + *checks.maxSize); + + if (checks.maxClosureSize) { + uint64_t closureSize = getClosure(info.path).second; + if (closureSize > *checks.maxClosureSize) + throw BuildError( + BuildResult::Failure::OutputRejected, + "closure of path '%s' is too large at %d bytes; limit is %d bytes", + store.printStorePath(info.path), + closureSize, + *checks.maxClosureSize); + } + + auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive) { + /* Parse a list of reference specifiers. Each element must + either be a store path, or the symbolic name of the output + of the derivation (such as `out'). */ + StorePathSet spec; + for (auto & i : value) { + if (store.isStorePath(i)) + spec.insert(store.parseStorePath(i)); + else if (auto output = get(outputs, i)) + spec.insert(output->path); + else { + std::string outputsListing = + concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; }); + throw BuildError( + BuildResult::Failure::OutputRejected, + "derivation '%s' output check for '%s' contains an illegal reference specifier '%s'," + " expected store path or output name (one of [%s])", + store.printStorePath(drvPath), + outputName, + i, + outputsListing); + } + } + + auto used = recursive ? getClosure(info.path).first : info.references; + + if (recursive && checks.ignoreSelfRefs) + used.erase(info.path); + + StorePathSet badPaths; + + for (auto & i : used) + if (allowed) { + if (!spec.count(i)) + badPaths.insert(i); + } else { + if (spec.count(i)) + badPaths.insert(i); + } + + if (!badPaths.empty()) { + std::string badPathsStr; + for (auto & i : badPaths) { + badPathsStr += "\n "; + badPathsStr += store.printStorePath(i); + } + throw BuildError( + BuildResult::Failure::OutputRejected, + "output '%s' is not allowed to refer to the following paths:%s", + store.printStorePath(info.path), + badPathsStr); + } + }; + + /* Mandatory check: absent whitelist, and present but empty + whitelist mean very different things. */ + if (auto & refs = checks.allowedReferences) { + checkRefs(*refs, true, false); + } + if (auto & refs = checks.allowedRequisites) { + checkRefs(*refs, true, true); + } + + /* Optimization: don't need to do anything when + disallowed and empty set. */ + if (!checks.disallowedReferences.empty()) { + checkRefs(checks.disallowedReferences, false, false); + } + if (!checks.disallowedRequisites.empty()) { + checkRefs(checks.disallowedRequisites, false, true); + } + }; + + std::visit( + overloaded{ + [&](const DerivationOptions::OutputChecks & checks) { applyChecks(checks); }, + [&](const std::map & checksPerOutput) { + if (auto outputChecks = get(checksPerOutput, outputName)) + + applyChecks(*outputChecks); + }, + }, + outputChecks); + } +} + +} // namespace nix diff --git a/src/libstore/build/derivation-check.hh b/src/libstore/build/derivation-check.hh new file mode 100644 index 00000000000..d4808f9a2fc --- /dev/null +++ b/src/libstore/build/derivation-check.hh @@ -0,0 +1,28 @@ +#pragma once +///@file + +#include "nix/store/derivations.hh" +#include "nix/store/derivation-options.hh" +#include "nix/store/path-info.hh" + +namespace nix { + +/** + * Check that outputs meets the requirements specified by the + * 'outputChecks' attribute (or the legacy + * '{allowed,disallowed}{References,Requisites}' attributes). + * + * The outputs may not be valid yet, hence outputs needs to contain all + * needed info like the NAR size. However, the external (not other + * output) references of the output must be valid, so we can compute the + * closure size. + */ +void checkOutputs( + Store & store, + const StorePath & drvPath, + const decltype(Derivation::outputs) & drvOutputs, + const decltype(DerivationOptions::outputChecks) & drvOptions, + const std::map & outputs, + Activity & act); + +} // namespace nix diff --git a/src/libstore/build/derivation-env-desugar.cc b/src/libstore/build/derivation-env-desugar.cc new file mode 100644 index 00000000000..d6e002d911e --- /dev/null +++ b/src/libstore/build/derivation-env-desugar.cc @@ -0,0 +1,59 @@ +#include "nix/store/build/derivation-env-desugar.hh" +#include "nix/store/store-api.hh" +#include "nix/store/derivations.hh" +#include "nix/store/derivation-options.hh" + +namespace nix { + +std::string & DesugaredEnv::atFileEnvPair(std::string_view name, std::string fileName) +{ + auto & ret = extraFiles[fileName]; + variables.insert_or_assign( + std::string{name}, + EnvEntry{ + .prependBuildDirectory = true, + .value = std::move(fileName), + }); + return ret; +} + +DesugaredEnv DesugaredEnv::create( + Store & store, const Derivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths) +{ + DesugaredEnv res; + + if (drv.structuredAttrs) { + auto json = drv.structuredAttrs->prepareStructuredAttrs(store, drvOptions, inputPaths, drv.outputs); + res.atFileEnvPair("NIX_ATTRS_SH_FILE", ".attrs.sh") = StructuredAttrs::writeShell(json); + res.atFileEnvPair("NIX_ATTRS_JSON_FILE", ".attrs.json") = json.dump(); + } else { + /* In non-structured mode, set all bindings either directory in the + environment or via a file, as specified by + `DerivationOptions::passAsFile`. */ + for (auto & [envName, envValue] : drv.env) { + if (!drvOptions.passAsFile.contains(envName)) { + res.variables.insert_or_assign( + envName, + EnvEntry{ + .value = envValue, + }); + } else { + res.atFileEnvPair( + envName + "Path", + ".attr-" + hashString(HashAlgorithm::SHA256, envName).to_string(HashFormat::Nix32, false)) = + envValue; + } + } + + /* Handle exportReferencesGraph(), if set. */ + for (auto & [fileName, storePaths] : drvOptions.getParsedExportReferencesGraph(store)) { + /* Write closure info to . */ + res.extraFiles.insert_or_assign( + fileName, store.makeValidityRegistration(store.exportReferences(storePaths, inputPaths), false, false)); + } + } + + return res; +} + +} // namespace nix diff --git a/src/libstore/build/derivation-goal.cc b/src/libstore/build/derivation-goal.cc index 883121d9476..7c672e746c9 100644 --- a/src/libstore/build/derivation-goal.cc +++ b/src/libstore/build/derivation-goal.cc @@ -94,7 +94,7 @@ Goal::Co DerivationGoal::haveDerivation() /* If they are all valid, then we're done. */ if (checkResult && checkResult->second == PathStatus::Valid && buildMode == bmNormal) { - co_return done(BuildResult::AlreadyValid, checkResult->first); + co_return doneSuccess(BuildResult::Success::AlreadyValid, checkResult->first); } Goals waitees; @@ -122,12 +122,10 @@ Goal::Co DerivationGoal::haveDerivation() assert(!drv->type().isImpure()); if (nrFailed > 0 && nrFailed > nrNoSubstituters && !settings.tryFallback) { - co_return done( - BuildResult::TransientFailure, - {}, - Error( - "some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", - worker.store.printStorePath(drvPath))); + co_return doneFailure(BuildError( + BuildResult::Failure::TransientFailure, + "some substitutes for the outputs of derivation '%s' failed (usually happens due to networking issues); try '--fallback' to build derivation from source ", + worker.store.printStorePath(drvPath))); } nrFailed = nrNoSubstituters = 0; @@ -137,7 +135,7 @@ Goal::Co DerivationGoal::haveDerivation() bool allValid = checkResult && checkResult->second == PathStatus::Valid; if (buildMode == bmNormal && allValid) { - co_return done(BuildResult::Substituted, checkResult->first); + co_return doneSuccess(BuildResult::Success::Substituted, checkResult->first); } if (buildMode == bmRepair && allValid) { co_return repairClosure(); @@ -165,25 +163,27 @@ Goal::Co DerivationGoal::haveDerivation() buildResult = g->buildResult; - if (buildMode == bmCheck) { - /* In checking mode, the builder will not register any outputs. - So we want to make sure the ones that we wanted to check are - properly there. */ - buildResult.builtOutputs = {{wantedOutput, assertPathValidity()}}; - } else { - /* Otherwise the builder will give us info for out output, but - also for other outputs. Filter down to just our output so as - not to leak info on unrelated things. */ - for (auto it = buildResult.builtOutputs.begin(); it != buildResult.builtOutputs.end();) { - if (it->first != wantedOutput) { - it = buildResult.builtOutputs.erase(it); - } else { - ++it; + if (auto * successP = buildResult.tryGetSuccess()) { + auto & success = *successP; + if (buildMode == bmCheck) { + /* In checking mode, the builder will not register any outputs. + So we want to make sure the ones that we wanted to check are + properly there. */ + success.builtOutputs = {{wantedOutput, assertPathValidity()}}; + } else { + /* Otherwise the builder will give us info for out output, but + also for other outputs. Filter down to just our output so as + not to leak info on unrelated things. */ + for (auto it = success.builtOutputs.begin(); it != success.builtOutputs.end();) { + if (it->first != wantedOutput) { + it = success.builtOutputs.erase(it); + } else { + ++it; + } } - } - if (buildResult.success()) - assert(buildResult.builtOutputs.count(wantedOutput) > 0); + assert(success.builtOutputs.count(wantedOutput) > 0); + } } co_return amDone(g->exitCode, g->ex); @@ -272,16 +272,17 @@ Goal::Co DerivationGoal::repairClosure() bmRepair)); } + bool haveWaitees = !waitees.empty(); co_await await(std::move(waitees)); - if (!waitees.empty()) { + if (haveWaitees) { trace("closure repaired"); if (nrFailed > 0) throw Error( "some paths in the output closure of derivation '%s' could not be repaired", worker.store.printStorePath(drvPath)); } - co_return done(BuildResult::AlreadyValid, assertPathValidity()); + co_return doneSuccess(BuildResult::Success::AlreadyValid, assertPathValidity()); } std::optional> DerivationGoal::checkPathValidity() @@ -339,39 +340,56 @@ Realisation DerivationGoal::assertPathValidity() return checkResult->first; } -Goal::Done -DerivationGoal::done(BuildResult::Status status, std::optional builtOutput, std::optional ex) +Goal::Done DerivationGoal::doneSuccess(BuildResult::Success::Status status, Realisation builtOutput) { - buildResult.status = status; - if (ex) - buildResult.errorMsg = fmt("%s", Uncolored(ex->info().msg)); - if (buildResult.status == BuildResult::TimedOut) - worker.timedOut = true; - if (buildResult.status == BuildResult::PermanentFailure) - worker.permanentFailure = true; + buildResult.inner = BuildResult::Success{ + .status = status, + .builtOutputs = {{wantedOutput, std::move(builtOutput)}}, + }; + + logger->result( + getCurActivity(), + resBuildResult, + nlohmann::json(KeyedBuildResult( + buildResult, + DerivedPath::Built{.drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}}))); mcExpectedBuilds.reset(); - if (buildResult.success()) { - assert(builtOutput); - buildResult.builtOutputs = {{wantedOutput, std::move(*builtOutput)}}; - if (status == BuildResult::Built) - worker.doneBuilds++; - } else { - if (status != BuildResult::DependencyFailed) - worker.failedBuilds++; - } + if (status == BuildResult::Success::Built) + worker.doneBuilds++; worker.updateProgress(); - auto traceBuiltOutputsFile = getEnv("_NIX_TRACE_BUILT_OUTPUTS").value_or(""); - if (traceBuiltOutputsFile != "") { - std::fstream fs; - fs.open(traceBuiltOutputsFile, std::fstream::out); - fs << worker.store.printStorePath(drvPath) << "\t" << buildResult.toString() << std::endl; - } + return amDone(ecSuccess, std::nullopt); +} + +Goal::Done DerivationGoal::doneFailure(BuildError ex) +{ + buildResult.inner = BuildResult::Failure{ + .status = ex.status, + .errorMsg = fmt("%s", Uncolored(ex.info().msg)), + }; + + logger->result( + getCurActivity(), + resBuildResult, + nlohmann::json(KeyedBuildResult( + buildResult, + DerivedPath::Built{.drvPath = makeConstantStorePathRef(drvPath), .outputs = OutputsSpec::All{}}))); + + mcExpectedBuilds.reset(); + + if (ex.status == BuildResult::Failure::TimedOut) + worker.timedOut = true; + if (ex.status == BuildResult::Failure::PermanentFailure) + worker.permanentFailure = true; + if (ex.status != BuildResult::Failure::DependencyFailed) + worker.failedBuilds++; + + worker.updateProgress(); - return amDone(buildResult.success() ? ecSuccess : ecFailed, std::move(ex)); + return amDone(ecFailed, {std::move(ex)}); } } // namespace nix diff --git a/src/libstore/build/derivation-trampoline-goal.cc b/src/libstore/build/derivation-trampoline-goal.cc index 5038a4ea0a4..205f5c427ee 100644 --- a/src/libstore/build/derivation-trampoline-goal.cc +++ b/src/libstore/build/derivation-trampoline-goal.cc @@ -164,10 +164,11 @@ Goal::Co DerivationTrampolineGoal::haveDerivation(StorePath drvPath, Derivation auto & g = *concreteDrvGoals.begin(); buildResult = g->buildResult; - for (auto & g2 : concreteDrvGoals) { - for (auto && [x, y] : g2->buildResult.builtOutputs) - buildResult.builtOutputs.insert_or_assign(x, y); - } + if (auto * successP = buildResult.tryGetSuccess()) + for (auto & g2 : concreteDrvGoals) + if (auto * successP2 = g2->buildResult.tryGetSuccess()) + for (auto && [x, y] : successP2->builtOutputs) + successP->builtOutputs.insert_or_assign(x, y); co_return amDone(g->exitCode, g->ex); } diff --git a/src/libstore/build/entry-points.cc b/src/libstore/build/entry-points.cc index 1dd5402650f..4bbd4c8f059 100644 --- a/src/libstore/build/entry-points.cc +++ b/src/libstore/build/entry-points.cc @@ -82,10 +82,10 @@ BuildResult Store::buildDerivation(const StorePath & drvPath, const BasicDerivat worker.run(Goals{goal}); return goal->buildResult; } catch (Error & e) { - return BuildResult{ - .status = BuildResult::MiscFailure, + return BuildResult{.inner{BuildResult::Failure{ + .status = BuildResult::Failure::MiscFailure, .errorMsg = e.msg(), - }; + }}}; }; } diff --git a/src/libstore/build/substitution-goal.cc b/src/libstore/build/substitution-goal.cc index ab95ea4a2b9..a4d3cbb555e 100644 --- a/src/libstore/build/substitution-goal.cc +++ b/src/libstore/build/substitution-goal.cc @@ -8,6 +8,8 @@ #include +#include + namespace nix { PathSubstitutionGoal::PathSubstitutionGoal( @@ -27,13 +29,33 @@ PathSubstitutionGoal::~PathSubstitutionGoal() cleanup(); } -Goal::Done PathSubstitutionGoal::done(ExitCode result, BuildResult::Status status, std::optional errorMsg) +Goal::Done PathSubstitutionGoal::doneSuccess(BuildResult::Success::Status status) { - buildResult.status = status; - if (errorMsg) { - debug(*errorMsg); - buildResult.errorMsg = *errorMsg; - } + buildResult.inner = BuildResult::Success{ + .status = status, + }; + + logger->result( + getCurActivity(), + resBuildResult, + nlohmann::json(KeyedBuildResult(buildResult, DerivedPath::Opaque{storePath}))); + + return amDone(ecSuccess); +} + +Goal::Done PathSubstitutionGoal::doneFailure(ExitCode result, BuildResult::Failure::Status status, std::string errorMsg) +{ + debug(errorMsg); + buildResult.inner = BuildResult::Failure{ + .status = status, + .errorMsg = std::move(errorMsg), + }; + + logger->result( + getCurActivity(), + resBuildResult, + nlohmann::json(KeyedBuildResult(buildResult, DerivedPath::Opaque{storePath}))); + return amDone(result); } @@ -45,7 +67,7 @@ Goal::Co PathSubstitutionGoal::init() /* If the path already exists we're done. */ if (!repair && worker.store.isValidPath(storePath)) { - co_return done(ecSuccess, BuildResult::AlreadyValid); + co_return doneSuccess(BuildResult::Success::AlreadyValid); } if (settings.readOnlyMode) @@ -55,9 +77,14 @@ Goal::Co PathSubstitutionGoal::init() auto subs = settings.useSubstitutes ? getDefaultSubstituters() : std::list>(); bool substituterFailed = false; + std::optional lastStoresException = std::nullopt; for (const auto & sub : subs) { trace("trying next substituter"); + if (lastStoresException.has_value()) { + logError(lastStoresException->info()); + lastStoresException.reset(); + } cleanup(); @@ -80,19 +107,13 @@ Goal::Co PathSubstitutionGoal::init() try { // FIXME: make async info = sub->queryPathInfo(subPath ? *subPath : storePath); - } catch (InvalidPath &) { + } catch (InvalidPath & e) { continue; } catch (SubstituterDisabled & e) { - if (settings.tryFallback) - continue; - else - throw e; + continue; } catch (Error & e) { - if (settings.tryFallback) { - logError(e.info()); - continue; - } else - throw e; + lastStoresException = std::make_optional(std::move(e)); + continue; } if (info->path != storePath) { @@ -156,13 +177,19 @@ Goal::Co PathSubstitutionGoal::init() worker.failedSubstitutions++; worker.updateProgress(); } + if (lastStoresException.has_value()) { + if (!settings.tryFallback) { + throw *lastStoresException; + } else + logError(lastStoresException->info()); + } /* Hack: don't indicate failure if there were no substituters. In that case the calling derivation should just do a build. */ - co_return done( + co_return doneFailure( substituterFailed ? ecFailed : ecNoSubstituters, - BuildResult::NoSubstituters, + BuildResult::Failure::NoSubstituters, fmt("path '%s' is required, but there is no substituter that can build it", worker.store.printStorePath(storePath))); } @@ -173,9 +200,9 @@ Goal::Co PathSubstitutionGoal::tryToRun( trace("all references realised"); if (nrFailed > 0) { - co_return done( + co_return doneFailure( nrNoSubstituters > 0 ? ecNoSubstituters : ecFailed, - BuildResult::DependencyFailed, + BuildResult::Failure::DependencyFailed, fmt("some references of path '%s' could not be realised", worker.store.printStorePath(storePath))); } @@ -292,7 +319,7 @@ Goal::Co PathSubstitutionGoal::tryToRun( worker.updateProgress(); - co_return done(ecSuccess, BuildResult::Substituted); + co_return doneSuccess(BuildResult::Success::Substituted); } void PathSubstitutionGoal::handleEOF(Descriptor fd) diff --git a/src/libstore/builtins/fetchurl.cc b/src/libstore/builtins/fetchurl.cc index 519ad94287f..df056954e27 100644 --- a/src/libstore/builtins/fetchurl.cc +++ b/src/libstore/builtins/fetchurl.cc @@ -37,7 +37,7 @@ static void builtinFetchurl(const BuiltinBuilderContext & ctx) auto fetch = [&](const std::string & url) { auto source = sinkToSource([&](Sink & sink) { - FileTransferRequest request(url); + FileTransferRequest request(VerbatimURL{url}); request.decompress = false; auto decompressor = makeDecompressionSink(unpack && hasSuffix(mainUrl, ".xz") ? "xz" : "none", sink); diff --git a/src/libstore/common-protocol.cc b/src/libstore/common-protocol.cc index d4f3efc9b5c..3db3c419fb2 100644 --- a/src/libstore/common-protocol.cc +++ b/src/libstore/common-protocol.cc @@ -26,13 +26,13 @@ void CommonProto::Serialise::write( StorePath CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { - return store.parseStorePath(readString(conn.from)); + return conn.shortStorePaths ? StorePath(readString(conn.from)) : store.parseStorePath(readString(conn.from)); } void CommonProto::Serialise::write( const StoreDirConfig & store, CommonProto::WriteConn conn, const StorePath & storePath) { - conn.to << store.printStorePath(storePath); + conn.to << (conn.shortStorePaths ? storePath.to_string() : store.printStorePath(storePath)); } ContentAddress CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) @@ -49,13 +49,18 @@ void CommonProto::Serialise::write( Realisation CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { std::string rawInput = readString(conn.from); - return Realisation::fromJSON(nlohmann::json::parse(rawInput), "remote-protocol"); + try { + return nlohmann::json::parse(rawInput); + } catch (Error & e) { + e.addTrace({}, "while parsing a realisation object in the remote protocol"); + throw; + } } void CommonProto::Serialise::write( const StoreDirConfig & store, CommonProto::WriteConn conn, const Realisation & realisation) { - conn.to << realisation.toJSON().dump(); + conn.to << static_cast(realisation).dump(); } DrvOutput CommonProto::Serialise::read(const StoreDirConfig & store, CommonProto::ReadConn conn) @@ -73,13 +78,15 @@ std::optional CommonProto::Serialise>::read(const StoreDirConfig & store, CommonProto::ReadConn conn) { auto s = readString(conn.from); - return s == "" ? std::optional{} : store.parseStorePath(s); + return s == "" ? std::optional{} : conn.shortStorePaths ? StorePath(s) : store.parseStorePath(s); } void CommonProto::Serialise>::write( const StoreDirConfig & store, CommonProto::WriteConn conn, const std::optional & storePathOpt) { - conn.to << (storePathOpt ? store.printStorePath(*storePathOpt) : ""); + conn.to + << (storePathOpt ? (conn.shortStorePaths ? storePathOpt->to_string() : store.printStorePath(*storePathOpt)) + : ""); } std::optional diff --git a/src/libstore/daemon.cc b/src/libstore/daemon.cc index 4f28a1e0d98..7df44158132 100644 --- a/src/libstore/daemon.cc +++ b/src/libstore/daemon.cc @@ -102,7 +102,7 @@ struct TunnelLogger : public Logger showErrorInfo(oss, ei, false); StringSink buf; - buf << STDERR_NEXT << toView(oss); + buf << STDERR_NEXT << oss.view(); enqueueMsg(buf.s); } @@ -546,47 +546,22 @@ static void performOp( break; } - case WorkerProto::Op::ExportPath: { - auto path = store->parseStorePath(readString(conn.from)); - readInt(conn.from); // obsolete - logger->startWork(); - TunnelSink sink(conn.to); - store->exportPath(path, sink); - logger->stopWork(); - conn.to << 1; - break; - } - - case WorkerProto::Op::ImportPaths: { - logger->startWork(); - TunnelSource source(conn.from, conn.to); - auto paths = store->importPaths(source, trusted ? NoCheckSigs : CheckSigs); - logger->stopWork(); - Strings paths2; - for (auto & i : paths) - paths2.push_back(store->printStorePath(i)); - conn.to << paths2; - break; - } - case WorkerProto::Op::BuildPaths: { auto drvs = WorkerProto::Serialise::read(*store, rconn); BuildMode mode = bmNormal; - if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 15) { - mode = WorkerProto::Serialise::read(*store, rconn); - - /* Repairing is not atomic, so disallowed for "untrusted" - clients. - - FIXME: layer violation in this message: the daemon code (i.e. - this file) knows whether a client/connection is trusted, but it - does not how how the client was authenticated. The mechanism - need not be getting the UID of the other end of a Unix Domain - Socket. - */ - if (mode == bmRepair && !trusted) - throw Error("repairing is not allowed because you are not in 'trusted-users'"); - } + mode = WorkerProto::Serialise::read(*store, rconn); + + /* Repairing is not atomic, so disallowed for "untrusted" + clients. + + FIXME: layer violation in this message: the daemon code (i.e. + this file) knows whether a client/connection is trusted, but it + does not how how the client was authenticated. The mechanism + need not be getting the UID of the other end of a Unix Domain + Socket. + */ + if (mode == bmRepair && !trusted) + throw Error("repairing is not allowed because you are not in 'trusted-users'"); logger->startWork(); store->buildPaths(drvs, mode); logger->stopWork(); @@ -769,6 +744,7 @@ static void performOp( options.action = (GCOptions::GCAction) readInt(conn.from); options.pathsToDelete = WorkerProto::Serialise::read(*store, rconn); conn.from >> options.ignoreLiveness >> options.maxFreed; + options.censor = !trusted; // obsolete fields readInt(conn.from); readInt(conn.from); @@ -777,7 +753,7 @@ static void performOp( GCResults results; logger->startWork(); - if (options.ignoreLiveness) + if (options.ignoreLiveness && !getEnv("_NIX_IN_TEST").has_value()) throw Error("you are not allowed to ignore liveness"); auto & gcStore = require(*store); gcStore.collectGarbage(options, results); @@ -805,13 +781,11 @@ static void performOp( clientSettings.buildCores = readInt(conn.from); clientSettings.useSubstitutes = readInt(conn.from); - if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { - unsigned int n = readInt(conn.from); - for (unsigned int i = 0; i < n; i++) { - auto name = readString(conn.from); - auto value = readString(conn.from); - clientSettings.overrides.emplace(name, value); - } + unsigned int n = readInt(conn.from); + for (unsigned int i = 0; i < n; i++) { + auto name = readString(conn.from); + auto value = readString(conn.from); + clientSettings.overrides.emplace(name, value); } logger->startWork(); @@ -876,19 +850,12 @@ static void performOp( auto path = store->parseStorePath(readString(conn.from)); std::shared_ptr info; logger->startWork(); - try { - info = store->queryPathInfo(path); - } catch (InvalidPath &) { - if (GET_PROTOCOL_MINOR(conn.protoVersion) < 17) - throw; - } + info = store->queryPathInfo(path); logger->stopWork(); if (info) { - if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 17) - conn.to << 1; + conn.to << 1; WorkerProto::write(*store, wconn, static_cast(*info)); } else { - assert(GET_PROTOCOL_MINOR(conn.protoVersion) >= 17); conn.to << 0; } break; @@ -1063,7 +1030,7 @@ void processConnection(ref store, FdSource && from, FdSink && to, Trusted auto [protoVersion, features] = WorkerProto::BasicServerConnection::handshake(to, from, PROTOCOL_VERSION, WorkerProto::allFeatures); - if (protoVersion < 0x10a) + if (protoVersion < MINIMUM_PROTOCOL_VERSION) throw Error("the Nix client version is too old"); WorkerProto::BasicServerConnection conn; diff --git a/src/libstore/derivation-options.cc b/src/libstore/derivation-options.cc index 1acb9dc0310..698485c0df4 100644 --- a/src/libstore/derivation-options.cc +++ b/src/libstore/derivation-options.cc @@ -99,6 +99,17 @@ DerivationOptions DerivationOptions::fromStructuredAttrs( return fromStructuredAttrs(env, parsed ? &*parsed : nullptr); } +static void flatten(const nlohmann::json & value, StringSet & res) +{ + if (value.is_array()) + for (auto & v : value) + flatten(v, res); + else if (value.is_string()) + res.insert(value); + else + throw Error("'exportReferencesGraph' value is not an array or a string"); +} + DerivationOptions DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAttrs * parsed, bool shouldWarn) { @@ -219,12 +230,9 @@ DerivationOptions::fromStructuredAttrs(const StringMap & env, const StructuredAt if (!e || !e->is_object()) return ret; for (auto & [key, value] : getObject(*e)) { - if (value.is_array()) - ret.insert_or_assign(key, value); - else if (value.is_string()) - ret.insert_or_assign(key, StringSet{value}); - else - throw Error("'exportReferencesGraph' value is not an array or a string"); + StringSet ss; + flatten(value, ss); + ret.insert_or_assign(key, std::move(ss)); } } else { auto s = getOr(env, "exportReferencesGraph", ""); @@ -265,7 +273,10 @@ DerivationOptions::getParsedExportReferencesGraph(const StoreDirConfig & store) StorePathSet storePaths; for (auto & storePathS : ss) { if (!store.isInStore(storePathS)) - throw BuildError("'exportReferencesGraph' contains a non-store path '%1%'", storePathS); + throw BuildError( + BuildResult::Failure::InputRejected, + "'exportReferencesGraph' contains a non-store path '%1%'", + storePathS); storePaths.insert(store.toStorePath(storePathS).first); } res.insert_or_assign(fileName, storePaths); @@ -355,7 +366,7 @@ DerivationOptions adl_serializer::from_json(const json & json }; } -void adl_serializer::to_json(json & json, DerivationOptions o) +void adl_serializer::to_json(json & json, const DerivationOptions & o) { json["outputChecks"] = std::visit( overloaded{ @@ -397,7 +408,7 @@ DerivationOptions::OutputChecks adl_serializer: }; } -void adl_serializer::to_json(json & json, DerivationOptions::OutputChecks c) +void adl_serializer::to_json(json & json, const DerivationOptions::OutputChecks & c) { json["ignoreSelfRefs"] = c.ignoreSelfRefs; json["allowedReferences"] = c.allowedReferences; diff --git a/src/libstore/derivations.cc b/src/libstore/derivations.cc index 1afc343d7b6..85581534c3f 100644 --- a/src/libstore/derivations.cc +++ b/src/libstore/derivations.cc @@ -9,8 +9,10 @@ #include "nix/store/common-protocol-impl.hh" #include "nix/util/strings-inline.hh" #include "nix/util/json-utils.hh" +#include "nix/store/async-path-writer.hh" #include +#include #include namespace nix { @@ -133,6 +135,20 @@ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repa }); } +StorePath writeDerivation( + Store & store, AsyncPathWriter & asyncPathWriter, const Derivation & drv, RepairFlag repair, bool readOnly) +{ + auto references = drv.inputSrcs; + for (auto & i : drv.inputDrvs.map) + references.insert(i.first); + return asyncPathWriter.addPath( + drv.unparse(store, false), + std::string(drv.name) + drvExtension, + references, + repair, + readOnly || settings.readOnlyMode); +} + namespace { /** * This mimics std::istream to some extent. We use this much smaller implementation @@ -498,28 +514,33 @@ Derivation parseDerivation( */ static void printString(std::string & res, std::string_view s) { - boost::container::small_vector buffer; - buffer.reserve(s.size() * 2 + 2); - char * buf = buffer.data(); - char * p = buf; - *p++ = '"'; - for (auto c : s) - if (c == '\"' || c == '\\') { - *p++ = '\\'; - *p++ = c; - } else if (c == '\n') { - *p++ = '\\'; - *p++ = 'n'; - } else if (c == '\r') { - *p++ = '\\'; - *p++ = 'r'; - } else if (c == '\t') { - *p++ = '\\'; - *p++ = 't'; - } else - *p++ = c; - *p++ = '"'; - res.append(buf, p - buf); + res.reserve(res.size() + s.size() * 2 + 2); + res += '"'; + static constexpr auto chunkSize = 1024; + std::array buffer; + while (!s.empty()) { + auto chunk = s.substr(0, /*n=*/chunkSize); + s.remove_prefix(chunk.size()); + char * buf = buffer.data(); + char * p = buf; + for (auto c : chunk) + if (c == '\"' || c == '\\') { + *p++ = '\\'; + *p++ = c; + } else if (c == '\n') { + *p++ = '\\'; + *p++ = 'n'; + } else if (c == '\r') { + *p++ = '\\'; + *p++ = 'r'; + } else if (c == '\t') { + *p++ = '\\'; + *p++ = 't'; + } else + *p++ = c; + res.append(buf, p - buf); + } + res += '"'; } static void printUnquotedString(std::string & res, std::string_view s) @@ -829,7 +850,7 @@ DerivationType BasicDerivation::type() const throw Error("can't mix derivation output types"); } -Sync drvHashes; +DrvHashes drvHashes; /* pathDerivationModulo and hashDerivationModulo are mutually recursive */ @@ -839,16 +860,13 @@ Sync drvHashes; */ static const DrvHash pathDerivationModulo(Store & store, const StorePath & drvPath) { - { - auto hashes = drvHashes.lock(); - auto h = hashes->find(drvPath); - if (h != hashes->end()) { - return h->second; - } + std::optional hash; + if (drvHashes.cvisit(drvPath, [&hash](const auto & kv) { hash.emplace(kv.second); })) { + return *hash; } auto h = hashDerivationModulo(store, store.readInvalidDerivation(drvPath), false); // Cache it - drvHashes.lock()->insert_or_assign(drvPath, h); + drvHashes.insert_or_assign(drvPath, h); return h; } @@ -1254,15 +1272,18 @@ void Derivation::checkInvariants(Store & store, const StorePath & drvPath) const const Hash impureOutputHash = hashString(HashAlgorithm::SHA256, "impure"); -nlohmann::json -DerivationOutput::toJSON(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const +nlohmann::json DerivationOutput::toJSON() const { nlohmann::json res = nlohmann::json::object(); std::visit( overloaded{ - [&](const DerivationOutput::InputAddressed & doi) { res["path"] = store.printStorePath(doi.path); }, + [&](const DerivationOutput::InputAddressed & doi) { res["path"] = doi.path; }, [&](const DerivationOutput::CAFixed & dof) { - res["path"] = store.printStorePath(dof.path(store, drvName, outputName)); + /* it would be nice to output the path for user convenience, but + this would require us to know the store dir. */ +#if 0 + res["path"] = dof.path(store, drvName, outputName); +#endif res["method"] = std::string{dof.ca.method.render()}; res["hashAlgo"] = printHashAlgo(dof.ca.hash.algo); res["hash"] = dof.ca.hash.to_string(HashFormat::Base16, false); @@ -1283,12 +1304,8 @@ DerivationOutput::toJSON(const StoreDirConfig & store, std::string_view drvName, return res; } -DerivationOutput DerivationOutput::fromJSON( - const StoreDirConfig & store, - std::string_view drvName, - OutputNameView outputName, - const nlohmann::json & _json, - const ExperimentalFeatureSettings & xpSettings) +DerivationOutput +DerivationOutput::fromJSON(const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) { std::set keys; auto & json = getObject(_json); @@ -1307,11 +1324,11 @@ DerivationOutput DerivationOutput::fromJSON( if (keys == (std::set{"path"})) { return DerivationOutput::InputAddressed{ - .path = store.parseStorePath(getString(valueAt(json, "path"))), + .path = valueAt(json, "path"), }; } - else if (keys == (std::set{"path", "method", "hashAlgo", "hash"})) { + else if (keys == (std::set{"method", "hashAlgo", "hash"})) { auto [method, hashAlgo] = methodAlgo(); auto dof = DerivationOutput::CAFixed{ .ca = @@ -1320,8 +1337,12 @@ DerivationOutput DerivationOutput::fromJSON( .hash = Hash::parseNonSRIUnprefixed(getString(valueAt(json, "hash")), hashAlgo), }, }; - if (dof.path(store, drvName, outputName) != store.parseStorePath(getString(valueAt(json, "path")))) + /* We no longer produce this (denormalized) field (for the + reasons described above), so we don't need to check it. */ +#if 0 + if (dof.path(store, drvName, outputName) != static_cast(valueAt(json, "path"))) throw Error("Path doesn't match derivation output"); +#endif return dof; } @@ -1352,17 +1373,19 @@ DerivationOutput DerivationOutput::fromJSON( } } -nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const +nlohmann::json Derivation::toJSON() const { nlohmann::json res = nlohmann::json::object(); res["name"] = name; + res["version"] = 3; + { nlohmann::json & outputsObj = res["outputs"]; outputsObj = nlohmann::json::object(); for (auto & [outputName, output] : outputs) { - outputsObj[outputName] = output.toJSON(store, name, outputName); + outputsObj[outputName] = output; } } @@ -1370,7 +1393,7 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const auto & inputsList = res["inputSrcs"]; inputsList = nlohmann::json ::array(); for (auto & input : inputSrcs) - inputsList.emplace_back(store.printStorePath(input)); + inputsList.emplace_back(input); } { @@ -1390,7 +1413,7 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const auto & inputDrvsObj = res["inputDrvs"]; inputDrvsObj = nlohmann::json::object(); for (auto & [inputDrv, inputNode] : inputDrvs.map) { - inputDrvsObj[store.printStorePath(inputDrv)] = doInput(inputNode); + inputDrvsObj[inputDrv.to_string()] = doInput(inputNode); } } } @@ -1406,8 +1429,7 @@ nlohmann::json Derivation::toJSON(const StoreDirConfig & store) const return res; } -Derivation Derivation::fromJSON( - const StoreDirConfig & store, const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) +Derivation Derivation::fromJSON(const nlohmann::json & _json, const ExperimentalFeatureSettings & xpSettings) { using nlohmann::detail::value_t; @@ -1417,11 +1439,13 @@ Derivation Derivation::fromJSON( res.name = getString(valueAt(json, "name")); + if (valueAt(json, "version") != 3) + throw Error("Only derivation format version 3 is currently supported."); + try { auto outputs = getObject(valueAt(json, "outputs")); for (auto & [outputName, output] : outputs) { - res.outputs.insert_or_assign( - outputName, DerivationOutput::fromJSON(store, res.name, outputName, output, xpSettings)); + res.outputs.insert_or_assign(outputName, DerivationOutput::fromJSON(output, xpSettings)); } } catch (Error & e) { e.addTrace({}, "while reading key 'outputs'"); @@ -1431,7 +1455,7 @@ Derivation Derivation::fromJSON( try { auto inputSrcs = getArray(valueAt(json, "inputSrcs")); for (auto & input : inputSrcs) - res.inputSrcs.insert(store.parseStorePath(static_cast(input))); + res.inputSrcs.insert(input); } catch (Error & e) { e.addTrace({}, "while reading key 'inputSrcs'"); throw; @@ -1452,7 +1476,7 @@ Derivation Derivation::fromJSON( }; auto drvs = getObject(valueAt(json, "inputDrvs")); for (auto & [inputDrvPath, inputOutputs] : drvs) - res.inputDrvs.map[store.parseStorePath(inputDrvPath)] = doInput(inputOutputs); + res.inputDrvs.map[StorePath{inputDrvPath}] = doInput(inputOutputs); } catch (Error & e) { e.addTrace({}, "while reading key 'inputDrvs'"); throw; @@ -1477,3 +1501,29 @@ Derivation Derivation::fromJSON( } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +DerivationOutput adl_serializer::from_json(const json & json) +{ + return DerivationOutput::fromJSON(json); +} + +void adl_serializer::to_json(json & json, const DerivationOutput & c) +{ + json = c.toJSON(); +} + +Derivation adl_serializer::from_json(const json & json) +{ + return Derivation::fromJSON(json); +} + +void adl_serializer::to_json(json & json, const Derivation & c) +{ + json = c.toJSON(); +} + +} // namespace nlohmann diff --git a/src/libstore/derived-path.cc b/src/libstore/derived-path.cc index 1fee1ae75ba..2cf720b8221 100644 --- a/src/libstore/derived-path.cc +++ b/src/libstore/derived-path.cc @@ -2,8 +2,7 @@ #include "nix/store/derivations.hh" #include "nix/store/store-api.hh" #include "nix/util/comparator.hh" - -#include +#include "nix/util/json-utils.hh" #include @@ -19,59 +18,6 @@ GENERATE_CMP_EXT(, std::strong_ordering, SingleDerivedPathBuilt, *me->drvPath, m GENERATE_EQUAL(, DerivedPathBuilt ::, DerivedPathBuilt, *me->drvPath, me->outputs); GENERATE_ONE_CMP(, bool, DerivedPathBuilt ::, <, DerivedPathBuilt, *me->drvPath, me->outputs); -nlohmann::json DerivedPath::Opaque::toJSON(const StoreDirConfig & store) const -{ - return store.printStorePath(path); -} - -nlohmann::json SingleDerivedPath::Built::toJSON(Store & store) const -{ - nlohmann::json res; - res["drvPath"] = drvPath->toJSON(store); - // Fallback for the input-addressed derivation case: We expect to always be - // able to print the output paths, so let’s do it - // FIXME try-resolve on drvPath - const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); - res["output"] = output; - auto outputPathIter = outputMap.find(output); - if (outputPathIter == outputMap.end()) - res["outputPath"] = nullptr; - else if (std::optional p = outputPathIter->second) - res["outputPath"] = store.printStorePath(*p); - else - res["outputPath"] = nullptr; - return res; -} - -nlohmann::json DerivedPath::Built::toJSON(Store & store) const -{ - nlohmann::json res; - res["drvPath"] = drvPath->toJSON(store); - // Fallback for the input-addressed derivation case: We expect to always be - // able to print the output paths, so let’s do it - // FIXME try-resolve on drvPath - const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *drvPath)); - for (const auto & [output, outputPathOpt] : outputMap) { - if (!outputs.contains(output)) - continue; - if (outputPathOpt) - res["outputs"][output] = store.printStorePath(*outputPathOpt); - else - res["outputs"][output] = nullptr; - } - return res; -} - -nlohmann::json SingleDerivedPath::toJSON(Store & store) const -{ - return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw()); -} - -nlohmann::json DerivedPath::toJSON(Store & store) const -{ - return std::visit([&](const auto & buildable) { return buildable.toJSON(store); }, raw()); -} - std::string DerivedPath::Opaque::to_string(const StoreDirConfig & store) const { return store.printStorePath(path); @@ -273,3 +219,77 @@ const StorePath & DerivedPath::getBaseStorePath() const } } // namespace nix + +namespace nlohmann { + +void adl_serializer::to_json(json & json, const SingleDerivedPath::Opaque & o) +{ + json = o.path; +} + +SingleDerivedPath::Opaque adl_serializer::from_json(const json & json) +{ + return SingleDerivedPath::Opaque{json}; +} + +void adl_serializer::to_json(json & json, const SingleDerivedPath::Built & sdpb) +{ + json = { + {"drvPath", *sdpb.drvPath}, + {"output", sdpb.output}, + }; +} + +void adl_serializer::to_json(json & json, const DerivedPath::Built & dbp) +{ + json = { + {"drvPath", *dbp.drvPath}, + {"outputs", dbp.outputs}, + }; +} + +SingleDerivedPath::Built adl_serializer::from_json(const json & json0) +{ + auto & json = getObject(json0); + return { + .drvPath = make_ref(static_cast(valueAt(json, "drvPath"))), + .output = getString(valueAt(json, "output")), + }; +} + +DerivedPath::Built adl_serializer::from_json(const json & json0) +{ + auto & json = getObject(json0); + return { + .drvPath = make_ref(static_cast(valueAt(json, "drvPath"))), + .outputs = adl_serializer::from_json(valueAt(json, "outputs")), + }; +} + +void adl_serializer::to_json(json & json, const SingleDerivedPath & sdp) +{ + std::visit([&](const auto & buildable) { json = buildable; }, sdp.raw()); +} + +void adl_serializer::to_json(json & json, const DerivedPath & sdp) +{ + std::visit([&](const auto & buildable) { json = buildable; }, sdp.raw()); +} + +SingleDerivedPath adl_serializer::from_json(const json & json) +{ + if (json.is_string()) + return static_cast(json); + else + return static_cast(json); +} + +DerivedPath adl_serializer::from_json(const json & json) +{ + if (json.is_string()) + return static_cast(json); + else + return static_cast(json); +} + +} // namespace nlohmann diff --git a/src/libstore/dummy-store.cc b/src/libstore/dummy-store.cc index d0e2989681a..c739db6ba54 100644 --- a/src/libstore/dummy-store.cc +++ b/src/libstore/dummy-store.cc @@ -1,71 +1,153 @@ #include "nix/store/store-registration.hh" +#include "nix/util/archive.hh" #include "nix/util/callback.hh" +#include "nix/util/memory-source-accessor.hh" +#include "nix/store/dummy-store-impl.hh" + +#include namespace nix { -struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig +std::string DummyStoreConfig::doc() +{ + return +#include "dummy-store.md" + ; +} + +namespace { + +class WholeStoreViewAccessor : public SourceAccessor { - using StoreConfig::StoreConfig; + using BaseName = std::string; + + /** + * Map from store path basenames to corresponding accessors. + */ + boost::concurrent_flat_map> subdirs; + + /** + * Helper accessor for accessing just the CanonPath::root. + */ + MemorySourceAccessor rootPathAccessor; + + /** + * Helper empty accessor. + */ + MemorySourceAccessor emptyAccessor; - DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) - : StoreConfig(params) + auto + callWithAccessorForPath(CanonPath path, std::invocable auto callback) { - if (!authority.empty()) - throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); + if (path.isRoot()) + return callback(rootPathAccessor, path); + + BaseName baseName(*path.begin()); + MemorySourceAccessor * res = nullptr; + + subdirs.cvisit(baseName, [&](const auto & kv) { + path = path.removePrefix(CanonPath{baseName}); + res = &*kv.second; + }); + + if (!res) + res = &emptyAccessor; + + return callback(*res, path); } - static const std::string name() +public: + WholeStoreViewAccessor() { - return "Dummy Store"; + MemorySink sink{rootPathAccessor}; + sink.createDirectory(CanonPath::root); } - static std::string doc() + void addObject(std::string_view baseName, ref accessor) { - return -#include "dummy-store.md" - ; + subdirs.emplace(baseName, std::move(accessor)); } - static StringSet uriSchemes() + std::string readFile(const CanonPath & path) override { - return {"dummy"}; + return callWithAccessorForPath( + path, [](SourceAccessor & accessor, const CanonPath & path) { return accessor.readFile(path); }); } - ref openStore() const override; + void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override + { + return callWithAccessorForPath(path, [&](SourceAccessor & accessor, const CanonPath & path) { + return accessor.readFile(path, sink, sizeCallback); + }); + } - StoreReference getReference() const override + bool pathExists(const CanonPath & path) override { - return { - .variant = - StoreReference::Specified{ - .scheme = *uriSchemes().begin(), - }, - }; + return callWithAccessorForPath( + path, [](SourceAccessor & accessor, const CanonPath & path) { return accessor.pathExists(path); }); + } + + std::optional maybeLstat(const CanonPath & path) override + { + return callWithAccessorForPath( + path, [](SourceAccessor & accessor, const CanonPath & path) { return accessor.maybeLstat(path); }); + } + + DirEntries readDirectory(const CanonPath & path) override + { + return callWithAccessorForPath( + path, [](SourceAccessor & accessor, const CanonPath & path) { return accessor.readDirectory(path); }); + } + + std::string readLink(const CanonPath & path) override + { + return callWithAccessorForPath( + path, [](SourceAccessor & accessor, const CanonPath & path) { return accessor.readLink(path); }); } }; -struct DummyStore : virtual Store +} // namespace + +ref DummyStoreConfig::openStore() const +{ + return openDummyStore(); +} + +struct DummyStoreImpl : DummyStore { using Config = DummyStoreConfig; - ref config; + /** + * This view conceptually just borrows the file systems objects of + * each store object from `contents`, and combines them together + * into one store-wide source accessor. + * + * This is needed just in order to implement `Store::getFSAccessor`. + */ + ref wholeStoreView = make_ref(); - DummyStore(ref config) + DummyStoreImpl(ref config) : Store{*config} - , config(config) + , DummyStore{config} { + wholeStoreView->setPathDisplay(config->storeDir); } void queryPathInfoUncached( const StorePath & path, Callback> callback) noexcept override { - callback(nullptr); + bool visited = contents.cvisit(path, [&](const auto & kv) { + callback(std::make_shared(StorePath{kv.first}, kv.second.info)); + }); + + if (!visited) + callback(nullptr); } /** * The dummy store is incapable of *not* trusting! :) */ - virtual std::optional isTrustedClient() override + std::optional isTrustedClient() override { return Trusted; } @@ -77,11 +159,32 @@ struct DummyStore : virtual Store void addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override { - unsupported("addToStore"); + if (config->readOnly) + unsupported("addToStore"); + + if (repair) + throw Error("repairing is not supported for '%s' store", config->getHumanReadableURI()); + + if (checkSigs) + throw Error("checking signatures is not supported for '%s' store", config->getHumanReadableURI()); + + auto temp = make_ref(); + MemorySink tempSink{*temp}; + parseDump(tempSink, source); + auto path = info.path; + + auto accessor = make_ref(std::move(*temp)); + contents.insert( + {path, + PathInfoAndContents{ + std::move(info), + accessor, + }}); + wholeStoreView->addObject(path.to_string(), accessor); } - virtual StorePath addToStoreFromDump( - Source & dump, + StorePath addToStoreFromDump( + Source & source, std::string_view name, FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, ContentAddressMethod hashMethod = FileIngestionMethod::NixArchive, @@ -89,12 +192,78 @@ struct DummyStore : virtual Store const StorePathSet & references = StorePathSet(), RepairFlag repair = NoRepair) override { - unsupported("addToStore"); + if (config->readOnly) + unsupported("addToStoreFromDump"); + + if (repair) + throw Error("repairing is not supported for '%s' store", config->getHumanReadableURI()); + + auto temp = make_ref(); + + { + MemorySink tempSink{*temp}; + + // TODO factor this out into `restorePath`, same todo on it. + switch (dumpMethod) { + case FileSerialisationMethod::NixArchive: + parseDump(tempSink, source); + break; + case FileSerialisationMethod::Flat: { + // Replace root dir with file so next part succeeds. + temp->root = MemorySourceAccessor::File::Regular{}; + tempSink.createRegularFile(CanonPath::root, [&](auto & sink) { source.drainInto(sink); }); + break; + } + } + } + + auto hash = hashPath({temp, CanonPath::root}, hashMethod.getFileIngestionMethod(), hashAlgo).first; + auto narHash = hashPath({temp, CanonPath::root}, FileIngestionMethod::NixArchive, HashAlgorithm::SHA256); + + auto info = ValidPathInfo::makeFromCA( + *this, + name, + ContentAddressWithReferences::fromParts( + hashMethod, + std::move(hash), + { + .others = references, + // caller is not capable of creating a self-reference, because + // this is content-addressed without modulus + .self = false, + }), + std::move(narHash.first)); + + info.narSize = narHash.second.value(); + + auto path = info.path; + auto accessor = make_ref(std::move(*temp)); + contents.insert( + {path, + PathInfoAndContents{ + std::move(info), + accessor, + }}); + wholeStoreView->addObject(path.to_string(), accessor); + + return path; + } + + void registerDrvOutput(const Realisation & output) override + { + unsupported("registerDrvOutput"); } void narFromPath(const StorePath & path, Sink & sink) override { - unsupported("narFromPath"); + bool visited = contents.cvisit(path, [&](const auto & kv) { + const auto & [info, accessor] = kv.second; + SourcePath sourcePath(accessor); + dumpPath(sourcePath, sink, FileSerialisationMethod::NixArchive); + }); + + if (!visited) + throw Error("path '%s' is not valid", printStorePath(path)); } void @@ -103,15 +272,22 @@ struct DummyStore : virtual Store callback(nullptr); } - virtual ref getFSAccessor(bool requireValidPath) override + std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath) override + { + std::shared_ptr res; + contents.cvisit(path, [&](const auto & kv) { res = kv.second.contents.get_ptr(); }); + return res; + } + + ref getFSAccessor(bool requireValidPath) override { - return makeEmptySourceAccessor(); + return wholeStoreView; } }; -ref DummyStore::Config::openStore() const +ref DummyStore::Config::openDummyStore() const { - return make_ref(ref{shared_from_this()}); + return make_ref(ref{shared_from_this()}); } static RegisterStoreImplementation regDummyStore; diff --git a/src/libstore/dummy-store.md b/src/libstore/dummy-store.md index eb7b4ba0dd0..3ba96fecbf2 100644 --- a/src/libstore/dummy-store.md +++ b/src/libstore/dummy-store.md @@ -2,9 +2,11 @@ R"( **Store URL format**: `dummy://` -This store type represents a store that contains no store paths and -cannot be written to. It's useful when you want to use the Nix -evaluator when no actual Nix store exists, e.g. +This store type represents a store in memory. +Store objects can be read and written, but only so long as the store is open. +Once the store is closed, all data will be discarded. + +It's useful when you want to use the Nix evaluator when no actual Nix store exists, e.g. ```console # nix eval --store dummy:// --expr '1 + 2' diff --git a/src/libstore/export-import.cc b/src/libstore/export-import.cc index 13444deb256..17491055f2d 100644 --- a/src/libstore/export-import.cc +++ b/src/libstore/export-import.cc @@ -1,94 +1,166 @@ +#include "nix/store/export-import.hh" #include "nix/util/serialise.hh" #include "nix/store/store-api.hh" #include "nix/util/archive.hh" #include "nix/store/common-protocol.hh" #include "nix/store/common-protocol-impl.hh" - -#include +#include "nix/store/worker-protocol.hh" namespace nix { -void Store::exportPaths(const StorePathSet & paths, Sink & sink) +static const uint32_t exportMagicV1 = 0x4558494e; +static const uint64_t exportMagicV2 = 0x324f4952414e; // = 'NARIO2' + +void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version) { - auto sorted = topoSortPaths(paths); + auto sorted = store.topoSortPaths(paths); std::reverse(sorted.begin(), sorted.end()); - for (auto & path : sorted) { - sink << 1; - exportPath(path, sink); + auto dumpNar = [&](const ValidPathInfo & info) { + HashSink hashSink(HashAlgorithm::SHA256); + TeeSink teeSink(sink, hashSink); + + store.narFromPath(info.path, teeSink); + + /* Refuse to export paths that have changed. This prevents + filesystem corruption from spreading to other machines. + Don't complain if the stored hash is zero (unknown). */ + Hash hash = hashSink.currentHash().hash; + if (hash != info.narHash && info.narHash != Hash(info.narHash.algo)) + throw Error( + "hash of path '%s' has changed from '%s' to '%s'!", + store.printStorePath(info.path), + info.narHash.to_string(HashFormat::Nix32, true), + hash.to_string(HashFormat::Nix32, true)); + }; + + switch (version) { + + case 1: + for (auto & path : sorted) { + sink << 1; + auto info = store.queryPathInfo(path); + dumpNar(*info); + sink << exportMagicV1 << store.printStorePath(path); + CommonProto::write(store, CommonProto::WriteConn{.to = sink}, info->references); + sink << (info->deriver ? store.printStorePath(*info->deriver) : "") << 0; + } + sink << 0; + break; + + case 2: + sink << exportMagicV2; + + for (auto & path : sorted) { + Activity act(*logger, lvlTalkative, actUnknown, fmt("exporting path '%s'", store.printStorePath(path))); + sink << 1; + auto info = store.queryPathInfo(path); + // FIXME: move to CommonProto? + WorkerProto::Serialise::write( + store, WorkerProto::WriteConn{.to = sink, .version = 16, .shortStorePaths = true}, *info); + dumpNar(*info); + } + + sink << 0; + break; + + default: + throw Error("unsupported nario version %d", version); } - - sink << 0; -} - -void Store::exportPath(const StorePath & path, Sink & sink) -{ - auto info = queryPathInfo(path); - - HashSink hashSink(HashAlgorithm::SHA256); - TeeSink teeSink(sink, hashSink); - - narFromPath(path, teeSink); - - /* Refuse to export paths that have changed. This prevents - filesystem corruption from spreading to other machines. - Don't complain if the stored hash is zero (unknown). */ - Hash hash = hashSink.currentHash().hash; - if (hash != info->narHash && info->narHash != Hash(info->narHash.algo)) - throw Error( - "hash of path '%s' has changed from '%s' to '%s'!", - printStorePath(path), - info->narHash.to_string(HashFormat::Nix32, true), - hash.to_string(HashFormat::Nix32, true)); - - teeSink << exportMagic << printStorePath(path); - CommonProto::write(*this, CommonProto::WriteConn{.to = teeSink}, info->references); - teeSink << (info->deriver ? printStorePath(*info->deriver) : "") << 0; } -StorePaths Store::importPaths(Source & source, CheckSigsFlag checkSigs) +StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs) { StorePaths res; - while (true) { - auto n = readNum(source); - if (n == 0) - break; - if (n != 1) - throw Error("input doesn't look like something created by 'nix-store --export'"); - - /* Extract the NAR from the source. */ + + auto version = readNum(source); + + /* Note: nario version 1 lacks an explicit header. The first + integer denotes whether a store path follows or not. So look + for 0 or 1. */ + switch (version) { + + case 0: + /* Empty version 1 nario, nothing to do. */ + break; + + case 1: { + /* Reuse a string buffer to avoid kernel overhead allocating + memory for large strings. */ StringSink saved; - TeeSource tee{source, saved}; - NullFileSystemObjectSink ether; - parseDump(ether, tee); - uint32_t magic = readInt(source); - if (magic != exportMagic) - throw Error("Nix archive cannot be imported; wrong format"); + /* Non-empty version 1 nario. */ + while (true) { + /* Extract the NAR from the source. */ + saved.s.clear(); + TeeSource tee{source, saved}; + NullFileSystemObjectSink ether; + parseDump(ether, tee); + + uint32_t magic = readInt(source); + if (magic != exportMagicV1) + throw Error("nario cannot be imported; wrong format"); + + auto path = store.parseStorePath(readString(source)); + + auto references = CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = source}); + auto deriver = readString(source); + + // Ignore optional legacy signature. + if (readInt(source) == 1) + readString(source); + + if (!store.isValidPath(path)) { + auto narHash = hashString(HashAlgorithm::SHA256, saved.s); + + ValidPathInfo info{path, narHash}; + if (deriver != "") + info.deriver = store.parseStorePath(deriver); + info.references = references; + info.narSize = saved.s.size(); + + // Can't use underlying source, which would have been exhausted. + auto source2 = StringSource(saved.s); + store.addToStore(info, source2, NoRepair, checkSigs); + } + + res.push_back(path); + + auto n = readNum(source); + if (n == 0) + break; + if (n != 1) + throw Error("input doesn't look like a nario"); + } + break; + } - auto path = parseStorePath(readString(source)); + case exportMagicV2: + while (true) { + auto n = readNum(source); + if (n == 0) + break; + if (n != 1) + throw Error("input doesn't look like a nario"); - // Activity act(*logger, lvlInfo, "importing path '%s'", info.path); + auto info = WorkerProto::Serialise::read( + store, WorkerProto::ReadConn{.from = source, .version = 16, .shortStorePaths = true}); - auto references = CommonProto::Serialise::read(*this, CommonProto::ReadConn{.from = source}); - auto deriver = readString(source); - auto narHash = hashString(HashAlgorithm::SHA256, saved.s); + if (!store.isValidPath(info.path)) { + Activity act( + *logger, lvlTalkative, actUnknown, fmt("importing path '%s'", store.printStorePath(info.path))); - ValidPathInfo info{path, narHash}; - if (deriver != "") - info.deriver = parseStorePath(deriver); - info.references = references; - info.narSize = saved.s.size(); + store.addToStore(info, source, NoRepair, checkSigs); + } else + source.skip(info.narSize); - // Ignore optional legacy signature. - if (readInt(source) == 1) - readString(source); + res.push_back(info.path); + } - // Can't use underlying source, which would have been exhausted - auto source = StringSource(saved.s); - addToStore(info, source, NoRepair, checkSigs); + break; - res.push_back(info.path); + default: + throw Error("input doesn't look like a nario"); } return res; diff --git a/src/libstore/filetransfer.cc b/src/libstore/filetransfer.cc index 974797e1272..805213f0abe 100644 --- a/src/libstore/filetransfer.cc +++ b/src/libstore/filetransfer.cc @@ -100,7 +100,7 @@ struct curlFileTransfer : public FileTransfer lvlTalkative, actFileTransfer, fmt("%sing '%s'", request.verb(), request.uri), - {request.uri}, + {request.uri.to_string()}, request.parentAct) , callback(std::move(callback)) , finalSink([this](std::string_view data) { @@ -121,7 +121,7 @@ struct curlFileTransfer : public FileTransfer this->result.data.append(data); }) { - result.urls.push_back(request.uri); + result.urls.push_back(request.uri.to_string()); requestHeaders = curl_slist_append(requestHeaders, "Accept-Encoding: zstd, br, gzip, deflate, bzip2, xz"); if (!request.expectedETag.empty()) @@ -350,14 +350,14 @@ struct curlFileTransfer : public FileTransfer curl_easy_setopt(req, CURLOPT_DEBUGFUNCTION, TransferItem::debugCallback); } - curl_easy_setopt(req, CURLOPT_URL, request.uri.c_str()); + curl_easy_setopt(req, CURLOPT_URL, request.uri.to_string().c_str()); curl_easy_setopt(req, CURLOPT_FOLLOWLOCATION, 1L); curl_easy_setopt(req, CURLOPT_MAXREDIRS, 10); curl_easy_setopt(req, CURLOPT_NOSIGNAL, 1); curl_easy_setopt( req, CURLOPT_USERAGENT, - ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + ("curl/" LIBCURL_VERSION " Nix/" + nixVersion + " DeterminateNix/" + determinateNixVersion + (fileTransferSettings.userAgentSuffix != "" ? " " + fileTransferSettings.userAgentSuffix.get() : "")) .c_str()); #if LIBCURL_VERSION_NUM >= 0x072b00 @@ -594,10 +594,24 @@ struct curlFileTransfer : public FileTransfer } }; - bool quit = false; std:: priority_queue, std::vector>, EmbargoComparator> incoming; + private: + bool quitting = false; + public: + void quit() + { + quitting = true; + /* We wil not be processing any more incoming requests */ + while (!incoming.empty()) + incoming.pop(); + } + + bool isQuitting() + { + return quitting; + } }; Sync state_; @@ -649,7 +663,7 @@ struct curlFileTransfer : public FileTransfer /* Signal the worker thread to exit. */ { auto state(state_.lock()); - state->quit = true; + state->quit(); } #ifndef _WIN32 // TODO need graceful async exit support on Windows? writeFull(wakeupPipe.writeSide.get(), " ", false); @@ -750,7 +764,7 @@ struct curlFileTransfer : public FileTransfer break; } } - quit = state->quit; + quit = state->isQuitting(); } for (auto & item : incoming) { @@ -767,29 +781,31 @@ struct curlFileTransfer : public FileTransfer void workerThreadEntry() { + // Unwinding or because someone called `quit`. + bool normalExit = true; try { workerThreadMain(); } catch (nix::Interrupted & e) { + normalExit = false; } catch (std::exception & e) { printError("unexpected error in download thread: %s", e.what()); + normalExit = false; } - { + if (!normalExit) { auto state(state_.lock()); - while (!state->incoming.empty()) - state->incoming.pop(); - state->quit = true; + state->quit(); } } void enqueueItem(std::shared_ptr item) { - if (item->request.data && !hasPrefix(item->request.uri, "http://") && !hasPrefix(item->request.uri, "https://")) - throw nix::Error("uploading to '%s' is not supported", item->request.uri); + if (item->request.data && item->request.uri.scheme() != "http" && item->request.uri.scheme() != "https") + throw nix::Error("uploading to '%s' is not supported", item->request.uri.to_string()); { auto state(state_.lock()); - if (state->quit) + if (state->isQuitting()) throw nix::Error("cannot enqueue download request because the download thread is shutting down"); state->incoming.push(item); } @@ -801,11 +817,11 @@ struct curlFileTransfer : public FileTransfer void enqueueFileTransfer(const FileTransferRequest & request, Callback callback) override { /* Ugly hack to support s3:// URIs. */ - if (hasPrefix(request.uri, "s3://")) { + if (request.uri.scheme() == "s3") { // FIXME: do this on a worker thread try { #if NIX_WITH_S3_SUPPORT - auto parsed = ParsedS3URL::parse(request.uri); + auto parsed = ParsedS3URL::parse(request.uri.parsed()); std::string profile = parsed.profile.value_or(""); std::string region = parsed.region.value_or(Aws::Region::US_EAST_1); @@ -815,15 +831,16 @@ struct curlFileTransfer : public FileTransfer S3Helper s3Helper(profile, region, scheme, endpoint); // FIXME: implement ETag - auto s3Res = s3Helper.getObject(parsed.bucket, parsed.key); + auto s3Res = s3Helper.getObject(parsed.bucket, encodeUrlPath(parsed.key)); FileTransferResult res; if (!s3Res.data) throw FileTransferError(NotFound, {}, "S3 object '%s' does not exist", request.uri); res.data = std::move(*s3Res.data); - res.urls.push_back(request.uri); + res.urls.push_back(request.uri.to_string()); callback(std::move(res)); #else - throw nix::Error("cannot download '%s' because Nix is not built with S3 support", request.uri); + throw nix::Error( + "cannot download '%s' because Nix is not built with S3 support", request.uri.to_string()); #endif } catch (...) { callback.rethrow(); @@ -835,24 +852,29 @@ struct curlFileTransfer : public FileTransfer } }; -ref makeCurlFileTransfer() -{ - return make_ref(); -} +static Sync> _fileTransfer; ref getFileTransfer() { - static ref fileTransfer = makeCurlFileTransfer(); + auto fileTransfer(_fileTransfer.lock()); - if (fileTransfer->state_.lock()->quit) - fileTransfer = makeCurlFileTransfer(); + if (!*fileTransfer || (*fileTransfer)->state_.lock()->isQuitting()) + *fileTransfer = std::make_shared(); - return fileTransfer; + return ref(*fileTransfer); } ref makeFileTransfer() { - return makeCurlFileTransfer(); + return make_ref(); +} + +std::shared_ptr resetFileTransfer() +{ + auto fileTransfer(_fileTransfer.lock()); + std::shared_ptr prev; + fileTransfer->swap(prev); + return prev; } std::future FileTransfer::enqueueFileTransfer(const FileTransferRequest & request) diff --git a/src/libstore/gc.cc b/src/libstore/gc.cc index 0366fe0b029..f200926e842 100644 --- a/src/libstore/gc.cc +++ b/src/libstore/gc.cc @@ -1,6 +1,7 @@ #include "nix/store/derivations.hh" #include "nix/store/globals.hh" #include "nix/store/local-store.hh" +#include "nix/store/path.hh" #include "nix/util/finally.hh" #include "nix/util/unix-domain-socket.hh" #include "nix/util/signals.hh" @@ -13,14 +14,11 @@ # include "nix/util/processes.hh" #endif +#include +#include #include - -#include #include -#include -#include - -#include +#include #include #include #include @@ -209,7 +207,7 @@ void LocalStore::findTempRoots(Roots & tempRoots, bool censor) while ((end = contents.find((char) 0, pos)) != std::string::npos) { Path root(contents, pos, end - pos); debug("got temporary root '%s'", root); - tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{temp:%d}", pid)); + tempRoots[parseStorePath(root)].emplace(censor ? censored : fmt("{nix-process:%d}", pid)); pos = end + 1; } } @@ -314,7 +312,12 @@ Roots LocalStore::findRoots(bool censor) /** * Key is a mere string because cannot has path with macOS's libc++ */ -typedef std::unordered_map> UncheckedRoots; +typedef boost::unordered_flat_map< + std::string, + boost::unordered_flat_set>, + StringViewHash, + std::equal_to<>> + UncheckedRoots; static void readProcLink(const std::filesystem::path & file, UncheckedRoots & roots) { @@ -341,7 +344,7 @@ static std::string quoteRegexChars(const std::string & raw) static void readFileRoots(const std::filesystem::path & path, UncheckedRoots & roots) { try { - roots[readFile(path)].emplace(path); + roots[readFile(path)].emplace(path.string()); } catch (SysError & e) { if (e.errNo != ENOENT && e.errNo != EACCES) throw; @@ -463,13 +466,14 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) bool gcKeepOutputs = settings.gcKeepOutputs; bool gcKeepDerivations = settings.gcKeepDerivations; - std::unordered_set roots, dead, alive; + Roots roots; + boost::unordered_flat_set> dead, alive; struct Shared { // The temp roots only store the hash part to make it easier to // ignore suffixes like '.lock', '.chroot' and '.check'. - std::unordered_set tempRoots; + boost::unordered_flat_map tempRoots; // Hash part of the store path currently being deleted, if // any. @@ -578,9 +582,10 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) auto storePath = maybeParseStorePath(path); if (storePath) { debug("got new GC root '%s'", path); - auto hashPart = std::string(storePath->hashPart()); + auto hashPart = storePath->hashPart(); auto shared(_shared.lock()); - shared->tempRoots.insert(hashPart); + // FIXME: could get the PID from the socket. + shared->tempRoots.insert_or_assign(std::string(hashPart), "{nix-process:unknown}"); /* If this path is currently being deleted, then we have to wait until deletion is finished to ensure that @@ -620,20 +625,16 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) /* Find the roots. Since we've grabbed the GC lock, the set of permanent roots cannot increase now. */ printInfo("finding garbage collector roots..."); - Roots rootMap; if (!options.ignoreLiveness) - findRootsNoTemp(rootMap, true); - - for (auto & i : rootMap) - roots.insert(i.first); + findRootsNoTemp(roots, options.censor); /* Read the temporary roots created before we acquired the global GC root. Any new roots will be sent to our socket. */ - Roots tempRoots; - findTempRoots(tempRoots, true); - for (auto & root : tempRoots) { - _shared.lock()->tempRoots.insert(std::string(root.first.hashPart())); - roots.insert(root.first); + { + Roots tempRoots; + findTempRoots(tempRoots, options.censor); + for (auto & root : tempRoots) + _shared.lock()->tempRoots.insert_or_assign(std::string(root.first.hashPart()), *root.second.begin()); } /* Synchronisation point for testing, see tests/functional/gc-non-blocking.sh. */ @@ -672,7 +673,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; - std::unordered_map referrersCache; + boost::unordered_flat_map> referrersCache; /* Helper function that visits all paths reachable from `start` via the referrers edges and optionally derivers and derivation @@ -729,20 +730,32 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) } }; + if (options.action == GCOptions::gcDeleteSpecific && !options.pathsToDelete.count(*path)) { + throw Error( + "Cannot delete path '%s' because it's referenced by path '%s'.", + printStorePath(start), + printStorePath(*path)); + } + /* If this is a root, bail out. */ - if (roots.count(*path)) { + if (auto i = roots.find(*path); i != roots.end()) { + if (options.action == GCOptions::gcDeleteSpecific) + throw Error( + "Cannot delete path '%s' because it's referenced by the GC root '%s'.", + printStorePath(start), + *i->second.begin()); debug("cannot delete '%s' because it's a root", printStorePath(*path)); return markAlive(); } - if (options.action == GCOptions::gcDeleteSpecific && !options.pathsToDelete.count(*path)) - return; - - { - auto hashPart = std::string(path->hashPart()); + static bool inTest = getEnv("_NIX_IN_TEST").has_value(); + if (!(inTest && options.ignoreLiveness)) { + auto hashPart = path->hashPart(); auto shared(_shared.lock()); - if (shared->tempRoots.count(hashPart)) { - debug("cannot delete '%s' because it's a temporary root", printStorePath(*path)); + if (auto i = shared->tempRoots.find(std::string(hashPart)); i != shared->tempRoots.end()) { + if (options.action == GCOptions::gcDeleteSpecific) + throw Error( + "Cannot delete path '%s' because it's in use by '%s'.", printStorePath(start), i->second); return markAlive(); } shared->pending = hashPart; @@ -801,12 +814,7 @@ void LocalStore::collectGarbage(const GCOptions & options, GCResults & results) for (auto & i : options.pathsToDelete) { deleteReferrersClosure(i); - if (!dead.count(i)) - throw Error( - "Cannot delete path '%1%' since it is still alive. " - "To find out why, use: " - "nix-store --query --roots and nix-store --query --referrers", - printStorePath(i)); + assert(dead.count(i)); } } else if (options.maxFreed > 0) { @@ -931,7 +939,7 @@ void LocalStore::autoGC(bool sync) std::shared_future future; { - auto state(_state.lock()); + auto state(_state->lock()); if (state->gcRunning) { future = state->gcFuture; @@ -964,7 +972,7 @@ void LocalStore::autoGC(bool sync) /* Wake up any threads waiting for the auto-GC to finish. */ Finally wakeup([&]() { - auto state(_state.lock()); + auto state(_state->lock()); state->gcRunning = false; state->lastGCCheck = std::chrono::steady_clock::now(); promise.set_value(); @@ -979,7 +987,7 @@ void LocalStore::autoGC(bool sync) collectGarbage(options, results); - _state.lock()->availAfterGC = getAvail(); + _state->lock()->availAfterGC = getAvail(); } catch (...) { // FIXME: we could propagate the exception to the diff --git a/src/libstore/globals.cc b/src/libstore/globals.cc index 612e79ab00c..e2873a87bec 100644 --- a/src/libstore/globals.cc +++ b/src/libstore/globals.cc @@ -150,7 +150,7 @@ std::vector getUserConfigFiles() return files; } -unsigned int Settings::getDefaultCores() const +unsigned int Settings::getDefaultCores() { const unsigned int concurrency = std::max(1U, std::thread::hardware_concurrency()); const unsigned int maxCPU = getMaxCPU(); @@ -260,6 +260,8 @@ Path Settings::getDefaultSSLCertFile() std::string nixVersion = PACKAGE_VERSION; +const std::string determinateNixVersion = DETERMINATE_NIX_VERSION; + NLOHMANN_JSON_SERIALIZE_ENUM( SandboxMode, { @@ -341,10 +343,15 @@ PathsInChroot BaseSetting::parse(const std::string & str) const i.pop_back(); } size_t p = i.find('='); - if (p == std::string::npos) - pathsInChroot[i] = {.source = i, .optional = optional}; - else - pathsInChroot[i.substr(0, p)] = {.source = i.substr(p + 1), .optional = optional}; + std::string inside, outside; + if (p == std::string::npos) { + inside = i; + outside = i; + } else { + inside = i.substr(0, p); + outside = i.substr(p + 1); + } + pathsInChroot[inside] = {.source = outside, .optional = optional}; } return pathsInChroot; } @@ -374,6 +381,24 @@ unsigned int MaxBuildJobsSetting::parse(const std::string & str) const } } +NLOHMANN_DEFINE_TYPE_NON_INTRUSIVE(Settings::ExternalBuilder, systems, program, args); + +template<> +Settings::ExternalBuilders BaseSetting::parse(const std::string & str) const +{ + try { + return nlohmann::json::parse(str).template get(); + } catch (std::exception & e) { + throw UsageError("parsing setting '%s': %s", name, e.what()); + } +} + +template<> +std::string BaseSetting::to_string() const +{ + return nlohmann::json(value).dump(); +} + template<> void BaseSetting::appendOrSet(PathsInChroot newValue, bool append) { diff --git a/src/libstore/http-binary-cache-store.cc b/src/libstore/http-binary-cache-store.cc index 2777b88276c..5d4fba16331 100644 --- a/src/libstore/http-binary-cache-store.cc +++ b/src/libstore/http-binary-cache-store.cc @@ -4,6 +4,7 @@ #include "nix/store/nar-info-disk-cache.hh" #include "nix/util/callback.hh" #include "nix/store/store-registration.hh" +#include "nix/util/compression.hh" namespace nix { @@ -27,7 +28,7 @@ HttpBinaryCacheStoreConfig::HttpBinaryCacheStoreConfig( + (!_cacheUri.empty() ? _cacheUri : throw UsageError("`%s` Store requires a non-empty authority in Store URL", scheme)))) { - while (!cacheUri.path.empty() && cacheUri.path.back() == '/') + while (!cacheUri.path.empty() && cacheUri.path.back() == "") cacheUri.path.pop_back(); } @@ -37,9 +38,9 @@ StoreReference HttpBinaryCacheStoreConfig::getReference() const .variant = StoreReference::Specified{ .scheme = cacheUri.scheme, - .authority = (cacheUri.authority ? cacheUri.authority->to_string() : "") + cacheUri.path, + .authority = cacheUri.renderAuthorityAndPath(), }, - .params = cacheUri.query, + .params = getQueryParams(), }; } @@ -142,8 +143,27 @@ class HttpBinaryCacheStore : public virtual BinaryCacheStore const std::string & mimeType) override { auto req = makeRequest(path); - req.data = StreamToSourceAdapter(istream).drain(); + + auto data = StreamToSourceAdapter(istream).drain(); + + // Determine compression method based on file type + std::string compressionMethod; + if (hasSuffix(path, ".narinfo")) + compressionMethod = config->narinfoCompression; + else if (hasSuffix(path, ".ls")) + compressionMethod = config->lsCompression; + else if (hasPrefix(path, "log/")) + compressionMethod = config->logCompression; + + // Apply compression if configured + if (!compressionMethod.empty()) { + data = compress(compressionMethod, data); + req.headers.emplace_back("Content-Encoding", compressionMethod); + } + + req.data = std::move(data); req.mimeType = mimeType; + try { getFileTransfer()->upload(req); } catch (FileTransferError & e) { @@ -154,22 +174,17 @@ class HttpBinaryCacheStore : public virtual BinaryCacheStore FileTransferRequest makeRequest(const std::string & path) { - /* FIXME path is not a path, but a full relative or absolute + /* Otherwise the last path fragment will get discarded. */ + auto cacheUriWithTrailingSlash = config->cacheUri; + if (!cacheUriWithTrailingSlash.path.empty()) + cacheUriWithTrailingSlash.path.push_back(""); + + /* path is not a path, but a full relative or absolute URL, e.g. we've seen in the wild NARINFO files have a URL field which is `nar/15f99rdaf26k39knmzry4xd0d97wp6yfpnfk1z9avakis7ipb9yg.nar?hash=zphkqn2wg8mnvbkixnl2aadkbn0rcnfj` - (note the query param) and that gets passed here. - - What should actually happen is that we have two parsed URLs - (if we support relative URLs), and then we combined them with - a URL `operator/` which would be like - `std::filesystem::path`'s equivalent operator, which properly - combines the the URLs, whether the right is relative or - absolute. */ - return FileTransferRequest( - hasPrefix(path, "https://") || hasPrefix(path, "http://") || hasPrefix(path, "file://") - ? path - : config->cacheUri.to_string() + "/" + path); + (note the query param) and that gets passed here. */ + return FileTransferRequest(parseURLRelative(path, cacheUriWithTrailingSlash)); } void getFile(const std::string & path, Sink & sink) override diff --git a/src/libstore/include/nix/store/async-path-writer.hh b/src/libstore/include/nix/store/async-path-writer.hh new file mode 100644 index 00000000000..80997dc6ac2 --- /dev/null +++ b/src/libstore/include/nix/store/async-path-writer.hh @@ -0,0 +1,19 @@ +#pragma once + +#include "nix/store/store-api.hh" + +namespace nix { + +struct AsyncPathWriter +{ + virtual StorePath addPath( + std::string contents, std::string name, StorePathSet references, RepairFlag repair, bool readOnly = false) = 0; + + virtual void waitForPath(const StorePath & path) = 0; + + virtual void waitForAllPaths() = 0; + + static ref make(ref store); +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/aws-creds.hh b/src/libstore/include/nix/store/aws-creds.hh new file mode 100644 index 00000000000..16643c55552 --- /dev/null +++ b/src/libstore/include/nix/store/aws-creds.hh @@ -0,0 +1,73 @@ +#pragma once +///@file +#include "nix/store/config.hh" + +#if NIX_WITH_CURL_S3 + +# include "nix/store/s3-url.hh" +# include "nix/util/error.hh" + +# include +# include +# include + +namespace nix { + +/** + * AWS credentials obtained from credential providers + */ +struct AwsCredentials +{ + std::string accessKeyId; + std::string secretAccessKey; + std::optional sessionToken; + + AwsCredentials( + const std::string & accessKeyId, + const std::string & secretAccessKey, + const std::optional & sessionToken = std::nullopt) + : accessKeyId(accessKeyId) + , secretAccessKey(secretAccessKey) + , sessionToken(sessionToken) + { + } +}; + +/** + * Exception thrown when AWS authentication fails + */ +MakeError(AwsAuthError, Error); + +/** + * Get AWS credentials for the given profile. + * This function automatically caches credential providers to avoid + * creating multiple providers for the same profile. + * + * @param profile The AWS profile name (empty string for default profile) + * @return AWS credentials + * @throws AwsAuthError if credentials cannot be resolved + */ +AwsCredentials getAwsCredentials(const std::string & profile = ""); + +/** + * Invalidate cached credentials for a profile (e.g., on authentication failure). + * The next request for this profile will create a new provider. + * + * @param profile The AWS profile name to invalidate + */ +void invalidateAwsCredentials(const std::string & profile); + +/** + * Clear all cached credential providers. + * Typically called during application cleanup. + */ +void clearAwsCredentialsCache(); + +/** + * Pre-resolve AWS credentials for S3 URLs. + * Used to cache credentials in parent process before forking. + */ +AwsCredentials preResolveAwsCredentials(const ParsedS3URL & s3Url); + +} // namespace nix +#endif diff --git a/src/libstore/include/nix/store/binary-cache-store.hh b/src/libstore/include/nix/store/binary-cache-store.hh index 908500b4280..c316b1199b4 100644 --- a/src/libstore/include/nix/store/binary-cache-store.hh +++ b/src/libstore/include/nix/store/binary-cache-store.hh @@ -12,6 +12,7 @@ namespace nix { struct NarInfo; +class RemoteFSAccessor; struct BinaryCacheStoreConfig : virtual StoreConfig { @@ -136,6 +137,11 @@ private: CheckSigsFlag checkSigs, std::function mkInfo); + /** + * Same as `getFSAccessor`, but with a more preceise return type. + */ + ref getRemoteFSAccessor(bool requireValidPath = true); + public: bool isValidPathUncached(const StorePath & path) override; @@ -175,6 +181,8 @@ public: ref getFSAccessor(bool requireValidPath = true) override; + std::shared_ptr getFSAccessor(const StorePath &, bool requireValidPath = true) override; + void addSignatures(const StorePath & storePath, const StringSet & sigs) override; std::optional getBuildLogExact(const StorePath & path) override; diff --git a/src/libstore/include/nix/store/build-result.hh b/src/libstore/include/nix/store/build-result.hh index 3b70b781f54..52bd179cb4a 100644 --- a/src/libstore/include/nix/store/build-result.hh +++ b/src/libstore/include/nix/store/build-result.hh @@ -1,90 +1,127 @@ #pragma once ///@file -#include "nix/store/realisation.hh" -#include "nix/store/derived-path.hh" - #include #include #include +#include "nix/store/derived-path.hh" +#include "nix/store/realisation.hh" + +#include + namespace nix { struct BuildResult { + struct Success + { + /** + * @note This is directly used in the nix-store --serve protocol. + * That means we need to worry about compatibility across versions. + * Therefore, don't remove status codes, and only add new status + * codes at the end of the list. + * + * Must be disjoint with `Failure::Status`. + */ + enum Status : uint8_t { + Built = 0, + Substituted = 1, + AlreadyValid = 2, + ResolvesToAlreadyValid = 13, + } status; + + /** + * For derivations, a mapping from the names of the wanted outputs + * to actual paths. + */ + SingleDrvOutputs builtOutputs; + + bool operator==(const BuildResult::Success &) const noexcept; + std::strong_ordering operator<=>(const BuildResult::Success &) const noexcept; + + static bool statusIs(uint8_t status) + { + return status == Built || status == Substituted || status == AlreadyValid + || status == ResolvesToAlreadyValid; + } + }; + + struct Failure + { + /** + * @note This is directly used in the nix-store --serve protocol. + * That means we need to worry about compatibility across versions. + * Therefore, don't remove status codes, and only add new status + * codes at the end of the list. + * + * Must be disjoint with `Success::Status`. + */ + enum Status : uint8_t { + PermanentFailure = 3, + InputRejected = 4, + OutputRejected = 5, + /// possibly transient + TransientFailure = 6, + /// no longer used + CachedFailure = 7, + TimedOut = 8, + MiscFailure = 9, + DependencyFailed = 10, + LogLimitExceeded = 11, + NotDeterministic = 12, + NoSubstituters = 14, + /// A certain type of `OutputRejected`. The protocols do not yet + /// know about this one, so change it back to `OutputRejected` + /// before serialization. + HashMismatch = 15, + } status = MiscFailure; + + /** + * Information about the error if the build failed. + * + * @todo This should be an entire ErrorInfo object, not just a + * string, for richer information. + */ + std::string errorMsg; + + /** + * If timesBuilt > 1, whether some builds did not produce the same + * result. (Note that 'isNonDeterministic = false' does not mean + * the build is deterministic, just that we don't have evidence of + * non-determinism.) + */ + bool isNonDeterministic = false; + + bool operator==(const BuildResult::Failure &) const noexcept; + std::strong_ordering operator<=>(const BuildResult::Failure &) const noexcept; + + [[noreturn]] void rethrow() const + { + throw Error("%s", errorMsg); + } + }; + + std::variant inner = Failure{}; + /** - * @note This is directly used in the nix-store --serve protocol. - * That means we need to worry about compatibility across versions. - * Therefore, don't remove status codes, and only add new status - * codes at the end of the list. + * Convenience wrapper to avoid a longer `std::get_if` usage by the + * caller (which will have to add more `BuildResult::` than we do + * below also, do note.) */ - enum Status { - Built = 0, - Substituted, - AlreadyValid, - PermanentFailure, - InputRejected, - OutputRejected, - /// possibly transient - TransientFailure, - /// no longer used - CachedFailure, - TimedOut, - MiscFailure, - DependencyFailed, - LogLimitExceeded, - NotDeterministic, - ResolvesToAlreadyValid, - NoSubstituters, - } status = MiscFailure; + auto * tryGetSuccess(this auto & self) + { + return std::get_if(&self.inner); + } /** - * Information about the error if the build failed. - * - * @todo This should be an entire ErrorInfo object, not just a - * string, for richer information. + * Convenience wrapper to avoid a longer `std::get_if` usage by the + * caller (which will have to add more `BuildResult::` than we do + * below also, do note.) */ - std::string errorMsg; - - std::string toString() const + auto * tryGetFailure(this auto & self) { - auto strStatus = [&]() { - switch (status) { - case Built: - return "Built"; - case Substituted: - return "Substituted"; - case AlreadyValid: - return "AlreadyValid"; - case PermanentFailure: - return "PermanentFailure"; - case InputRejected: - return "InputRejected"; - case OutputRejected: - return "OutputRejected"; - case TransientFailure: - return "TransientFailure"; - case CachedFailure: - return "CachedFailure"; - case TimedOut: - return "TimedOut"; - case MiscFailure: - return "MiscFailure"; - case DependencyFailed: - return "DependencyFailed"; - case LogLimitExceeded: - return "LogLimitExceeded"; - case NotDeterministic: - return "NotDeterministic"; - case ResolvesToAlreadyValid: - return "ResolvesToAlreadyValid"; - case NoSubstituters: - return "NoSubstituters"; - default: - return "Unknown"; - }; - }(); - return strStatus + ((errorMsg == "") ? "" : " : " + errorMsg); + return std::get_if(&self.inner); } /** @@ -92,20 +129,6 @@ struct BuildResult */ unsigned int timesBuilt = 0; - /** - * If timesBuilt > 1, whether some builds did not produce the same - * result. (Note that 'isNonDeterministic = false' does not mean - * the build is deterministic, just that we don't have evidence of - * non-determinism.) - */ - bool isNonDeterministic = false; - - /** - * For derivations, a mapping from the names of the wanted outputs - * to actual paths. - */ - SingleDrvOutputs builtOutputs; - /** * The start/stop times of the build (or one of the rounds, if it * was repeated). @@ -119,15 +142,19 @@ struct BuildResult bool operator==(const BuildResult &) const noexcept; std::strong_ordering operator<=>(const BuildResult &) const noexcept; +}; - bool success() - { - return status == Built || status == Substituted || status == AlreadyValid || status == ResolvesToAlreadyValid; - } +/** + * denotes a permanent build failure + */ +struct BuildError : public Error +{ + BuildResult::Failure::Status status; - void rethrow() + BuildError(BuildResult::Failure::Status status, auto &&... args) + : Error{args...} + , status{status} { - throw Error("%s", errorMsg); } }; @@ -149,4 +176,7 @@ struct KeyedBuildResult : BuildResult } }; +void to_json(nlohmann::json & json, const BuildResult & buildResult); +void to_json(nlohmann::json & json, const KeyedBuildResult & buildResult); + } // namespace nix diff --git a/src/libstore/include/nix/store/build/derivation-builder.hh b/src/libstore/include/nix/store/build/derivation-builder.hh index 144ca27b12b..30a8e081c84 100644 --- a/src/libstore/include/nix/store/build/derivation-builder.hh +++ b/src/libstore/include/nix/store/build/derivation-builder.hh @@ -8,9 +8,33 @@ #include "nix/store/parsed-derivations.hh" #include "nix/util/processes.hh" #include "nix/store/restricted-store.hh" +#include "nix/store/build/derivation-env-desugar.hh" namespace nix { +/** + * Denotes a build failure that stemmed from the builder exiting with a + * failing exist status. + */ +struct BuilderFailureError : BuildError +{ + int builderStatus; + + std::string extraMsgAfter; + + BuilderFailureError(BuildResult::Failure::Status status, int builderStatus, std::string extraMsgAfter) + : BuildError{ + status, + /* No message for now, because the caller will make for + us, with extra context */ + "", + } + , builderStatus{std::move(builderStatus)} + , extraMsgAfter{std::move(extraMsgAfter)} + { + } +}; + /** * Stuff we need to pass to initChild(). */ @@ -52,10 +76,7 @@ struct DerivationBuilderParams */ const StorePathSet & inputPaths; - /** - * @note we do in fact mutate this - */ - std::map & initialOutputs; + const std::map & initialOutputs; const BuildMode & buildMode; @@ -65,60 +86,20 @@ struct DerivationBuilderParams */ PathsInChroot defaultPathsInChroot; - struct EnvEntry - { - /** - * Actually, this should be passed as a file, but with a custom - * name (rather than hash-derived name for usual "pass as file"). - */ - std::optional nameOfPassAsFile; - - /** - * String value of env var, or contents of the file - */ - std::string value; - }; - /** - * The final environment variables to additionally set, possibly - * indirectly via a file. + * May be used to control various platform-specific functionality. * - * This is used by the caller to desugar the "structured attrs" - * mechanism, so `DerivationBuilder` doesn't need to know about it. + * For example, on Linux, the `kvm` system feature controls whether + * `/dev/kvm` should be exposed to the builder within the sandbox. */ - std::map> finalEnv; + StringSet systemFeatures; + + DesugaredEnv desugaredEnv; /** - * Inserted in the temp dir, but no file names placed in env, unlike - * `EnvEntry::nameOfPassAsFile` above. + * The activity corresponding to the build. */ - StringMap extraFiles; - - DerivationBuilderParams( - const StorePath & drvPath, - const BuildMode & buildMode, - BuildResult & buildResult, - const Derivation & drv, - const DerivationOptions & drvOptions, - const StorePathSet & inputPaths, - std::map & initialOutputs, - PathsInChroot defaultPathsInChroot, - std::map> finalEnv, - StringMap extraFiles) - : drvPath{drvPath} - , buildResult{buildResult} - , drv{drv} - , drvOptions{drvOptions} - , inputPaths{inputPaths} - , initialOutputs{initialOutputs} - , buildMode{buildMode} - , defaultPathsInChroot{std::move(defaultPathsInChroot)} - , finalEnv{std::move(finalEnv)} - , extraFiles{std::move(extraFiles)} - { - } - - DerivationBuilderParams(DerivationBuilderParams &&) = default; + std::unique_ptr & act; }; /** @@ -138,24 +119,10 @@ struct DerivationBuilderCallbacks */ virtual void closeLogFile() = 0; - virtual void appendLogTailErrorMsg(std::string & msg) = 0; - - /** - * Hook up `builderOut` to some mechanism to ingest the log - * - * @todo this should be reworked - */ - virtual void childStarted(Descriptor builderOut) = 0; - /** * @todo this should be reworked */ virtual void childTerminated() = 0; - - virtual void noteHashMismatch(void) = 0; - virtual void noteCheckMismatch(void) = 0; - - virtual void markContentsGood(const StorePath & path) = 0; }; /** @@ -171,11 +138,6 @@ struct DerivationBuilderCallbacks */ struct DerivationBuilder : RestrictionContext { - /** - * The process ID of the builder. - */ - Pid pid; - DerivationBuilder() = default; virtual ~DerivationBuilder() = default; @@ -190,15 +152,15 @@ struct DerivationBuilder : RestrictionContext * locks as needed). After this is run, the builder should be * started. * - * @returns true if successful, false if we could not acquire a build - * user. In that case, the caller must wait and then try again. - */ - virtual bool prepareBuild() = 0; - - /** - * Start building a derivation. + * @returns logging pipe if successful, `std::nullopt` if we could + * not acquire a build user. In that case, the caller must wait and + * then try again. + * + * @note "success" just means that we were able to set up the environment + * and start the build. The builder could have immediately exited with + * failure, and that would still be considered a successful start. */ - virtual void startBuilder() = 0; + virtual std::optional startBuild() = 0; /** * Tear down build environment after the builder exits (either on @@ -208,25 +170,18 @@ struct DerivationBuilder : RestrictionContext * processing. A status code and exception are returned, providing * more information. The second case indicates success, and * realisations for each output of the derivation are returned. + * + * @throws BuildError */ - virtual std::variant, SingleDrvOutputs> unprepareBuild() = 0; - - /** - * Stop the in-process nix daemon thread. - * @see startDaemon - */ - virtual void stopDaemon() = 0; - - /** - * Delete the temporary directory, if we have one. - */ - virtual void deleteTmpDir(bool force) = 0; + virtual SingleDrvOutputs unprepareBuild() = 0; /** - * Kill any processes running under the build user UID or in the - * cgroup of the build. + * Forcibly kill the child process, if any. + * + * @returns whether the child was still alive and needed to be + * killed. */ - virtual void killSandbox(bool getStats) = 0; + virtual bool killChild() = 0; }; #ifndef _WIN32 // TODO enable `DerivationBuilder` on Windows diff --git a/src/libstore/include/nix/store/build/derivation-building-goal.hh b/src/libstore/include/nix/store/build/derivation-building-goal.hh index 95949649c83..edb49602489 100644 --- a/src/libstore/include/nix/store/build/derivation-building-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-building-goal.hh @@ -14,6 +14,7 @@ namespace nix { using std::map; +struct BuilderFailureError; #ifndef _WIN32 // TODO enable build hook on Windows struct HookInstance; struct DerivationBuilder; @@ -28,6 +29,12 @@ typedef enum { rpAccept, rpDecline, rpPostpone } HookReply; */ struct DerivationBuildingGoal : public Goal { + DerivationBuildingGoal( + const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode = bmNormal); + ~DerivationBuildingGoal(); + +private: + /** The path of the derivation. */ StorePath drvPath; @@ -42,19 +49,12 @@ struct DerivationBuildingGoal : public Goal * The remainder is state held during the build. */ - /** - * Locks on (fixed) output paths. - */ - PathLocks outputLocks; - /** * All input paths (that is, the union of FS closures of the * immediate input paths). */ StorePathSet inputPaths; - std::map initialOutputs; - /** * File descriptor for the log file. */ @@ -91,22 +91,8 @@ struct DerivationBuildingGoal : public Goal std::unique_ptr act; - /** - * Activity that denotes waiting for a lock. - */ - std::unique_ptr actLock; - std::map builderActivities; - /** - * The remote machine on which we're building. - */ - std::string machineName; - - DerivationBuildingGoal( - const StorePath & drvPath, const Derivation & drv, Worker & worker, BuildMode buildMode = bmNormal); - ~DerivationBuildingGoal(); - void timedOut(Error && ex) override; std::string key() override; @@ -116,12 +102,11 @@ struct DerivationBuildingGoal : public Goal */ Co gaveUpOnSubstitution(); Co tryToBuild(); - Co hookDone(); /** * Is the build hook willing to perform the build? */ - HookReply tryBuildHook(); + HookReply tryBuildHook(const std::map & initialOutputs); /** * Open a log file and a pipe to it. @@ -155,24 +140,18 @@ struct DerivationBuildingGoal : public Goal * whether all outputs are valid and non-corrupt, and a * 'SingleDrvOutputs' structure containing the valid outputs. */ - std::pair checkPathValidity(); - - /** - * Aborts if any output is not valid or corrupt, and otherwise - * returns a 'SingleDrvOutputs' structure containing all outputs. - */ - SingleDrvOutputs assertPathValidity(); + std::pair checkPathValidity(std::map & initialOutputs); /** * Forcibly kill the child process, if any. */ void killChild(); - void started(); + Done doneSuccess(BuildResult::Success::Status status, SingleDrvOutputs builtOutputs); - Done done(BuildResult::Status status, SingleDrvOutputs builtOutputs = {}, std::optional ex = {}); + Done doneFailure(BuildError ex); - void appendLogTailErrorMsg(std::string & msg); + BuildError fixupBuilderFailureErrorMessage(BuilderFailureError msg); JobCategory jobCategory() const override { diff --git a/src/libstore/include/nix/store/build/derivation-building-misc.hh b/src/libstore/include/nix/store/build/derivation-building-misc.hh index f9e96510468..2b68fa1782a 100644 --- a/src/libstore/include/nix/store/build/derivation-building-misc.hh +++ b/src/libstore/include/nix/store/build/derivation-building-misc.hh @@ -49,9 +49,6 @@ struct InitialOutput std::optional known; }; -void runPostBuildHook( - const StoreDirConfig & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths); - /** * Format the known outputs of a derivation for use in error messages. */ diff --git a/src/libstore/include/nix/store/build/derivation-env-desugar.hh b/src/libstore/include/nix/store/build/derivation-env-desugar.hh new file mode 100644 index 00000000000..6e2efa6bb4d --- /dev/null +++ b/src/libstore/include/nix/store/build/derivation-env-desugar.hh @@ -0,0 +1,83 @@ +#pragma once +///@file + +#include "nix/util/types.hh" +#include "nix/store/path.hh" + +namespace nix { + +class Store; +struct Derivation; +struct DerivationOptions; + +/** + * Derivations claim to "just" specify their environment variables, but + * actually do a number of different features, such as "structured + * attrs", "pass as file", and "export references graph", things are + * more complicated then they appear. + * + * The good news is that we can simplify all that to the following view, + * where environment variables and extra files are specified exactly, + * with no special cases. + * + * Because we have `DesugaredEnv`, `DerivationBuilder` doesn't need to + * know about any of those above features, and their special case. + */ +struct DesugaredEnv +{ + struct EnvEntry + { + /** + * Whether to prepend the (inside via) path to the sandbox build + * directory to `value`. This is useful for when the env var + * should point to a file visible to the builder. + */ + bool prependBuildDirectory = false; + + /** + * String value of env var, or contents of the file. + */ + std::string value; + }; + + /** + * The final environment variables to set. + */ + std::map> variables; + + /** + * Extra file to be placed in the build directory. + * + * @note `EnvEntry::prependBuildDirectory` can be used to refer to + * those files without knowing what the build directory is. + */ + StringMap extraFiles; + + /** + * A common case is to define an environment variable that points to + * a file, which contains some contents. + * + * In base: + * ``` + * export VAR=FILE_NAME + * echo CONTENTS >FILE_NAME + * ``` + * + * This function assists in doing both parts, so the file name is + * kept in sync. + */ + std::string & atFileEnvPair(std::string_view name, std::string fileName); + + /** + * Given a (resolved) derivation, its options, and the closure of + * its inputs (which we can get since the derivation is resolved), + * desugar the environment to create a `DesguaredEnv`. + * + * @todo drvOptions will go away as a separate argument when it is + * just part of `Derivation`. + */ + static DesugaredEnv create( + Store & store, const Derivation & drv, const DerivationOptions & drvOptions, const StorePathSet & inputPaths); +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/build/derivation-goal.hh b/src/libstore/include/nix/store/build/derivation-goal.hh index 589c3fd58b7..e05bf1c0b73 100644 --- a/src/libstore/include/nix/store/build/derivation-goal.hh +++ b/src/libstore/include/nix/store/build/derivation-goal.hh @@ -14,9 +14,6 @@ namespace nix { using std::map; -/** Used internally */ -void runPostBuildHook(Store & store, Logger & logger, const StorePath & drvPath, const StorePathSet & outputPaths); - /** * A goal for realising a single output of a derivation. Various sorts of * fetching (which will be done by other goal types) is tried, and if none of @@ -102,13 +99,9 @@ private: Co repairClosure(); - /** - * @param builtOutput Must be set if `status` is successful. - */ - Done done( - BuildResult::Status status, - std::optional builtOutput = std::nullopt, - std::optional ex = {}); + Done doneSuccess(BuildResult::Success::Status status, Realisation builtOutput); + + Done doneFailure(BuildError ex); }; } // namespace nix diff --git a/src/libstore/include/nix/store/build/substitution-goal.hh b/src/libstore/include/nix/store/build/substitution-goal.hh index 9fc6450b1b1..5f6cb6a18c7 100644 --- a/src/libstore/include/nix/store/build/substitution-goal.hh +++ b/src/libstore/include/nix/store/build/substitution-goal.hh @@ -41,7 +41,9 @@ struct PathSubstitutionGoal : public Goal */ std::optional ca; - Done done(ExitCode result, BuildResult::Status status, std::optional errorMsg = {}); + Done doneSuccess(BuildResult::Success::Status status); + + Done doneFailure(ExitCode result, BuildResult::Failure::Status status, std::string errorMsg); public: PathSubstitutionGoal( diff --git a/src/libstore/include/nix/store/builtins.hh b/src/libstore/include/nix/store/builtins.hh index cc164fe8273..0cdd3a2bcf0 100644 --- a/src/libstore/include/nix/store/builtins.hh +++ b/src/libstore/include/nix/store/builtins.hh @@ -3,8 +3,12 @@ #include "nix/store/derivations.hh" +#include + namespace nix { +struct StructuredAttrs; + struct BuiltinBuilderContext { const BasicDerivation & drv; diff --git a/src/libstore/include/nix/store/common-protocol.hh b/src/libstore/include/nix/store/common-protocol.hh index c1d22fa6c54..6139afc5d2e 100644 --- a/src/libstore/include/nix/store/common-protocol.hh +++ b/src/libstore/include/nix/store/common-protocol.hh @@ -30,6 +30,7 @@ struct CommonProto struct ReadConn { Source & from; + bool shortStorePaths = false; }; /** @@ -39,6 +40,7 @@ struct CommonProto struct WriteConn { Sink & to; + bool shortStorePaths = false; }; template diff --git a/src/libstore/include/nix/store/derivations.hh b/src/libstore/include/nix/store/derivations.hh index 18479b425df..b1739682c1d 100644 --- a/src/libstore/include/nix/store/derivations.hh +++ b/src/libstore/include/nix/store/derivations.hh @@ -11,12 +11,13 @@ #include "nix/util/sync.hh" #include "nix/util/variant-wrapper.hh" -#include +#include #include namespace nix { struct StoreDirConfig; +struct AsyncPathWriter; /* Abstract syntax of derivations. */ @@ -135,16 +136,12 @@ struct DerivationOutput std::optional path(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; - nlohmann::json toJSON(const StoreDirConfig & store, std::string_view drvName, OutputNameView outputName) const; + nlohmann::json toJSON() const; /** * @param xpSettings Stop-gap to avoid globals during unit tests. */ - static DerivationOutput fromJSON( - const StoreDirConfig & store, - std::string_view drvName, - OutputNameView outputName, - const nlohmann::json & json, - const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + static DerivationOutput + fromJSON(const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); }; typedef std::map DerivationOutputs; @@ -394,11 +391,9 @@ struct Derivation : BasicDerivation { } - nlohmann::json toJSON(const StoreDirConfig & store) const; - static Derivation fromJSON( - const StoreDirConfig & store, - const nlohmann::json & json, - const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); + nlohmann::json toJSON() const; + static Derivation + fromJSON(const nlohmann::json & json, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); bool operator==(const Derivation &) const = default; // TODO libc++ 16 (used by darwin) missing `std::map::operator <=>`, can't do yet. @@ -412,6 +407,16 @@ class Store; */ StorePath writeDerivation(Store & store, const Derivation & drv, RepairFlag repair = NoRepair, bool readOnly = false); +/** + * Asynchronously write a derivation to the Nix store, and return its path. + */ +StorePath writeDerivation( + Store & store, + AsyncPathWriter & asyncPathWriter, + const Derivation & drv, + RepairFlag repair = NoRepair, + bool readOnly = false); + /** * Read a derivation from a file. */ @@ -507,13 +512,23 @@ DrvHash hashDerivationModulo(Store & store, const Derivation & drv, bool maskOut */ std::map staticOutputHashes(Store & store, const Derivation & drv); +struct DrvHashFct +{ + using is_avalanching = std::true_type; + + std::size_t operator()(const StorePath & path) const noexcept + { + return std::hash{}(path.to_string()); + } +}; + /** * Memoisation of hashDerivationModulo(). */ -typedef std::map DrvHashes; +typedef boost::concurrent_flat_map DrvHashes; // FIXME: global, though at least thread-safe. -extern Sync drvHashes; +extern DrvHashes drvHashes; struct Source; struct Sink; @@ -532,3 +547,6 @@ void writeDerivation(Sink & out, const StoreDirConfig & store, const BasicDeriva std::string hashPlaceholder(const OutputNameView outputName); } // namespace nix + +JSON_IMPL(nix::DerivationOutput) +JSON_IMPL(nix::Derivation) diff --git a/src/libstore/include/nix/store/derived-path.hh b/src/libstore/include/nix/store/derived-path.hh index bc89b012eb7..47b29b2d6ca 100644 --- a/src/libstore/include/nix/store/derived-path.hh +++ b/src/libstore/include/nix/store/derived-path.hh @@ -5,6 +5,7 @@ #include "nix/store/outputs-spec.hh" #include "nix/util/configuration.hh" #include "nix/util/ref.hh" +#include "nix/util/json-impls.hh" #include @@ -14,9 +15,6 @@ namespace nix { struct StoreDirConfig; -// TODO stop needing this, `toJSON` below should be pure -class Store; - /** * An opaque derived path. * @@ -30,7 +28,6 @@ struct DerivedPathOpaque std::string to_string(const StoreDirConfig & store) const; static DerivedPathOpaque parse(const StoreDirConfig & store, std::string_view); - nlohmann::json toJSON(const StoreDirConfig & store) const; bool operator==(const DerivedPathOpaque &) const = default; auto operator<=>(const DerivedPathOpaque &) const = default; @@ -80,7 +77,6 @@ struct SingleDerivedPathBuilt ref drvPath, OutputNameView outputs, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); - nlohmann::json toJSON(Store & store) const; bool operator==(const SingleDerivedPathBuilt &) const noexcept; std::strong_ordering operator<=>(const SingleDerivedPathBuilt &) const noexcept; @@ -153,7 +149,6 @@ struct SingleDerivedPath : _SingleDerivedPathRaw const StoreDirConfig & store, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); - nlohmann::json toJSON(Store & store) const; }; static inline ref makeConstantStorePathRef(StorePath drvPath) @@ -208,7 +203,6 @@ struct DerivedPathBuilt ref, std::string_view, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); - nlohmann::json toJSON(Store & store) const; bool operator==(const DerivedPathBuilt &) const noexcept; // TODO libc++ 16 (used by darwin) missing `std::set::operator <=>`, can't do yet. @@ -287,8 +281,6 @@ struct DerivedPath : _DerivedPathRaw * Convert a `SingleDerivedPath` to a `DerivedPath`. */ static DerivedPath fromSingle(const SingleDerivedPath &); - - nlohmann::json toJSON(Store & store) const; }; typedef std::vector DerivedPaths; @@ -305,3 +297,9 @@ typedef std::vector DerivedPaths; void drvRequireExperiment( const SingleDerivedPath & drv, const ExperimentalFeatureSettings & xpSettings = experimentalFeatureSettings); } // namespace nix + +JSON_IMPL(nix::SingleDerivedPath::Opaque) +JSON_IMPL(nix::SingleDerivedPath::Built) +JSON_IMPL(nix::SingleDerivedPath) +JSON_IMPL(nix::DerivedPath::Built) +JSON_IMPL(nix::DerivedPath) diff --git a/src/libstore/include/nix/store/dummy-store-impl.hh b/src/libstore/include/nix/store/dummy-store-impl.hh new file mode 100644 index 00000000000..e05bb94ff76 --- /dev/null +++ b/src/libstore/include/nix/store/dummy-store-impl.hh @@ -0,0 +1,40 @@ +#pragma once +///@file + +#include "nix/store/dummy-store.hh" + +#include + +namespace nix { + +struct MemorySourceAccessor; + +/** + * Enough of the Dummy Store exposed for sake of writing unit tests + */ +struct DummyStore : virtual Store +{ + using Config = DummyStoreConfig; + + ref config; + + struct PathInfoAndContents + { + UnkeyedValidPathInfo info; + ref contents; + }; + + /** + * This is map conceptually owns the file system objects for each + * store object. + */ + boost::concurrent_flat_map contents; + + DummyStore(ref config) + : Store{*config} + , config(config) + { + } +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/dummy-store.hh b/src/libstore/include/nix/store/dummy-store.hh new file mode 100644 index 00000000000..95c09078c98 --- /dev/null +++ b/src/libstore/include/nix/store/dummy-store.hh @@ -0,0 +1,66 @@ +#pragma once +///@file + +#include "nix/store/store-api.hh" + +namespace nix { + +struct DummyStore; + +struct DummyStoreConfig : public std::enable_shared_from_this, virtual StoreConfig +{ + DummyStoreConfig(const Params & params) + : StoreConfig(params) + { + // Disable caching since this a temporary in-memory store. + pathInfoCacheSize = 0; + } + + DummyStoreConfig(std::string_view scheme, std::string_view authority, const Params & params) + : DummyStoreConfig(params) + { + if (!authority.empty()) + throw UsageError("`%s` store URIs must not contain an authority part %s", scheme, authority); + } + + Setting readOnly{ + this, + true, + "read-only", + R"( + Make any sort of write fail instead of succeeding. + No additional memory will be used, because no information needs to be stored. + )"}; + + static const std::string name() + { + return "Dummy Store"; + } + + static std::string doc(); + + static StringSet uriSchemes() + { + return {"dummy"}; + } + + /** + * Same as `openStore`, just with a more precise return type. + */ + ref openDummyStore() const; + + ref openStore() const override; + + StoreReference getReference() const override + { + return { + .variant = + StoreReference::Specified{ + .scheme = *uriSchemes().begin(), + }, + .params = getQueryParams(), + }; + } +}; + +} // namespace nix diff --git a/src/libstore/include/nix/store/export-import.hh b/src/libstore/include/nix/store/export-import.hh new file mode 100644 index 00000000000..4ea696f992f --- /dev/null +++ b/src/libstore/include/nix/store/export-import.hh @@ -0,0 +1,19 @@ +#pragma once + +#include "nix/store/store-api.hh" + +namespace nix { + +/** + * Export multiple paths in the format expected by `nix-store + * --import`. The paths will be sorted topologically. + */ +void exportPaths(Store & store, const StorePathSet & paths, Sink & sink, unsigned int version); + +/** + * Import a sequence of NAR dumps created by `exportPaths()` into the + * Nix store. + */ +StorePaths importPaths(Store & store, Source & source, CheckSigsFlag checkSigs = CheckSigs); + +} // namespace nix diff --git a/src/libstore/include/nix/store/filetransfer.hh b/src/libstore/include/nix/store/filetransfer.hh index 8ff0de5ef2b..3e011fddc52 100644 --- a/src/libstore/include/nix/store/filetransfer.hh +++ b/src/libstore/include/nix/store/filetransfer.hh @@ -9,6 +9,7 @@ #include "nix/util/ref.hh" #include "nix/util/configuration.hh" #include "nix/util/serialise.hh" +#include "nix/util/url.hh" namespace nix { @@ -30,9 +31,17 @@ struct FileTransferSettings : Config )", {"binary-caches-parallel-connections"}}; + /* Do not set this too low. On glibc, getaddrinfo() contains fallback code + paths that deal with ill-behaved DNS servers. Setting this too low + prevents some fallbacks from occurring. + + See description of options timeout, single-request, single-request-reopen + in resolv.conf(5). Also see https://github.com/NixOS/nix/pull/13985 for + details on the interaction between getaddrinfo(3) behavior and libcurl + CURLOPT_CONNECTTIMEOUT. */ Setting connectTimeout{ this, - 5, + 15, "connect-timeout", R"( The timeout (in seconds) for establishing connections in the @@ -70,7 +79,7 @@ extern const unsigned int RETRY_TIME_MS_DEFAULT; struct FileTransferRequest { - std::string uri; + VerbatimURL uri; Headers headers; std::string expectedETag; bool verifyTLS = true; @@ -84,8 +93,8 @@ struct FileTransferRequest std::string mimeType; std::function dataCallback; - FileTransferRequest(std::string_view uri) - : uri(uri) + FileTransferRequest(VerbatimURL uri) + : uri(std::move(uri)) , parentAct(getCurActivity()) { } @@ -111,6 +120,9 @@ struct FileTransferResult /** * All URLs visited in the redirect chain. + * + * @note Intentionally strings and not `ParsedURL`s so we faithfully + * return what cURL gave us. */ std::vector urls; @@ -179,6 +191,8 @@ ref getFileTransfer(); */ ref makeFileTransfer(); +std::shared_ptr resetFileTransfer(); + class FileTransferError : public Error { public: diff --git a/src/libstore/include/nix/store/gc-store.hh b/src/libstore/include/nix/store/gc-store.hh index 9f2255025cf..5691a3568c1 100644 --- a/src/libstore/include/nix/store/gc-store.hh +++ b/src/libstore/include/nix/store/gc-store.hh @@ -1,13 +1,21 @@ #pragma once ///@file -#include - #include "nix/store/store-api.hh" +#include +#include namespace nix { -typedef std::unordered_map> Roots; +// FIXME: should turn this into an std::variant to represent the +// several root types. +using GcRootInfo = std::string; + +typedef boost::unordered_flat_map< + StorePath, + boost::unordered_flat_set>, + std::hash> + Roots; struct GCOptions { @@ -51,6 +59,12 @@ struct GCOptions * Stop after at least `maxFreed` bytes have been freed. */ uint64_t maxFreed{std::numeric_limits::max()}; + + /** + * Whether to hide potentially sensitive information about GC + * roots (such as PIDs). + */ + bool censor = false; }; struct GCResults diff --git a/src/libstore/include/nix/store/globals.hh b/src/libstore/include/nix/store/globals.hh index 2cd92467c94..09c14bd900b 100644 --- a/src/libstore/include/nix/store/globals.hh +++ b/src/libstore/include/nix/store/globals.hh @@ -75,9 +75,9 @@ class Settings : public Config public: - Settings(); + static unsigned int getDefaultCores(); - unsigned int getDefaultCores() const; + Settings(); Path nixPrefix; @@ -427,7 +427,7 @@ public: R"( If set to `true`, Nix instructs [remote build machines](#conf-builders) to use their own [`substituters`](#conf-substituters) if available. - It means that remote build hosts fetches as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all. + It means that remote build hosts fetch as many dependencies as possible from their own substituters (e.g, from `cache.nixos.org`) instead of waiting for the local machine to upload them all. This can drastically reduce build times if the network connection between the local machine and the remote build host is slow. )"}; @@ -503,7 +503,7 @@ public: by the Nix account, its group should be the group specified here, and its mode should be `1775`. - If the build users group is empty, builds areperformed under + If the build users group is empty, builds are performed under the uid of the Nix process (that is, the uid of the caller if `NIX_REMOTE` is empty, the uid under which the Nix daemon runs if `NIX_REMOTE` is `daemon`). Obviously, this should not be used @@ -847,8 +847,8 @@ public: 4. The path to the build's scratch directory. This directory exists only if the build was run with `--keep-failed`. - The stderr and stdout output from the diff hook isn't - displayed to the user. Instead, it print to the nix-daemon's log. + The stderr and stdout output from the diff hook isn't displayed + to the user. Instead, it prints to the nix-daemon's log. When using the Nix daemon, `diff-hook` must be set in the `nix.conf` configuration file, and cannot be passed at the command line. @@ -1355,11 +1355,12 @@ public: Setting upgradeNixStorePathUrl{ this, - "https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix", + "", "upgrade-nix-store-path-url", R"( - Used by `nix upgrade-nix`, the URL of the file that contains the - store paths of the latest Nix release. + Deprecated. This option was used to configure how `nix upgrade-nix` operated. + + Using this setting has no effect. It will be removed in a future release of Determinate Nix. )"}; Setting warnLargePathThreshold{ @@ -1372,6 +1373,77 @@ public: Default is 0, which disables the warning. Set it to 1 to warn on all paths. )"}; + + struct ExternalBuilder + { + std::vector systems; + Path program; + std::vector args; + }; + + using ExternalBuilders = std::vector; + + Setting externalBuilders{ + this, + {}, + "external-builders", + R"( + Helper programs that execute derivations. + + The program is passed a JSON document that describes the build environment as the final argument. + The JSON document looks like this: + + { + "args": [ + "-e", + "/nix/store/vj1c3wf9…-source-stdenv.sh", + "/nix/store/shkw4qm9…-default-builder.sh" + ], + "builder": "/nix/store/s1qkj0ph…-bash-5.2p37/bin/bash", + "env": { + "HOME": "/homeless-shelter", + "builder": "/nix/store/s1qkj0ph…-bash-5.2p37/bin/bash", + "nativeBuildInputs": "/nix/store/l31j72f1…-version-check-hook", + "out": "/nix/store/2yx2prgx…-hello-2.12.2" + … + }, + "inputPaths": [ + "/nix/store/14dciax3…-glibc-2.32-54-dev", + "/nix/store/1azs5s8z…-gettext-0.21", + … + ], + "outputs": { + "out": "/nix/store/2yx2prgx…-hello-2.12.2" + }, + "realStoreDir": "/nix/store", + "storeDir": "/nix/store", + "system": "aarch64-linux", + "tmpDir": "/private/tmp/nix-build-hello-2.12.2.drv-0/build", + "tmpDirInSandbox": "/build", + "topTmpDir": "/private/tmp/nix-build-hello-2.12.2.drv-0", + "version": 1 + } + )", + {}, // aliases + true, // document default + // NOTE(cole-h): even though we can make the experimental feature required here, the errors + // are not as good (it just becomes a warning if you try to use this setting without the + // experimental feature) + // + // With this commented out: + // + // error: experimental Nix feature 'external-builders' is disabled; add '--extra-experimental-features + // external-builders' to enable it + // + // With this uncommented: + // + // warning: Ignoring setting 'external-builders' because experimental feature 'external-builders' is not enabled + // error: Cannot build '/nix/store/vwsp4qd8…-opentofu-1.10.2.drv'. + // Reason: required system or feature not available + // Required system: 'aarch64-linux' with features {} + // Current system: 'aarch64-darwin' with features {apple-virt, benchmark, big-parallel, nixos-test} + // Xp::ExternalBuilders + }; }; // FIXME: don't use a global variable. @@ -1398,6 +1470,8 @@ std::vector getUserConfigFiles(); */ extern std::string nixVersion; +extern const std::string determinateNixVersion; + /** * @param loadConfig Whether to load configuration from `nix.conf`, `NIX_CONFIG`, etc. May be disabled for unit tests. * @note When using libexpr, and/or libmain, This is not sufficient. See initNix(). diff --git a/src/libstore/include/nix/store/http-binary-cache-store.hh b/src/libstore/include/nix/store/http-binary-cache-store.hh index e0f6ce42fdf..e0b7ac1ea32 100644 --- a/src/libstore/include/nix/store/http-binary-cache-store.hh +++ b/src/libstore/include/nix/store/http-binary-cache-store.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "nix/util/url.hh" #include "nix/store/binary-cache-store.hh" @@ -14,6 +17,21 @@ struct HttpBinaryCacheStoreConfig : std::enable_shared_from_this narinfoCompression{ + this, "", "narinfo-compression", "Compression method for `.narinfo` files."}; + + const Setting lsCompression{this, "", "ls-compression", "Compression method for `.ls` files."}; + + const Setting logCompression{ + this, + "", + "log-compression", + R"( + Compression method for `log/*` files. It is recommended to + use a compression method supported by most web browsers + (e.g. `brotli`). + )"}; + static const std::string name() { return "HTTP Binary Cache Store"; diff --git a/src/libstore/include/nix/store/legacy-ssh-store.hh b/src/libstore/include/nix/store/legacy-ssh-store.hh index 91e021433e5..c91f88a8478 100644 --- a/src/libstore/include/nix/store/legacy-ssh-store.hh +++ b/src/libstore/include/nix/store/legacy-ssh-store.hh @@ -109,7 +109,7 @@ struct LegacySSHStore : public virtual Store unsupported("addToStore"); } - virtual StorePath addToStoreFromDump( + StorePath addToStoreFromDump( Source & dump, std::string_view name, FileSerialisationMethod dumpMethod = FileSerialisationMethod::NixArchive, @@ -121,6 +121,11 @@ struct LegacySSHStore : public virtual Store unsupported("addToStore"); } + void registerDrvOutput(const Realisation & output) override + { + unsupported("registerDrvOutput"); + } + public: BuildResult buildDerivation(const StorePath & drvPath, const BasicDerivation & drv, BuildMode buildMode) override; @@ -142,7 +147,12 @@ public: unsupported("ensurePath"); } - virtual ref getFSAccessor(bool requireValidPath) override + ref getFSAccessor(bool requireValidPath) override + { + unsupported("getFSAccessor"); + } + + std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath) override { unsupported("getFSAccessor"); } @@ -179,12 +189,6 @@ public: */ StorePathSet queryValidPaths(const StorePathSet & paths, bool lock, SubstituteFlag maybeSubstitute = NoSubstitute); - /** - * Just exists because this is exactly what Hydra was doing, and we - * don't yet want an algorithmic change. - */ - void addMultipleToStoreLegacy(Store & srcStore, const StorePathSet & paths); - void connect() override; unsigned int getProtocol() override; diff --git a/src/libstore/include/nix/store/local-fs-store.hh b/src/libstore/include/nix/store/local-fs-store.hh index 84777f3d78c..08f8e165646 100644 --- a/src/libstore/include/nix/store/local-fs-store.hh +++ b/src/libstore/include/nix/store/local-fs-store.hh @@ -9,6 +9,18 @@ namespace nix { struct LocalFSStoreConfig : virtual StoreConfig { +private: + static OptionalPathSetting makeRootDirSetting(LocalFSStoreConfig & self, std::optional defaultValue) + { + return { + &self, + std::move(defaultValue), + "root", + "Directory prefixed to all other paths.", + }; + } + +public: using StoreConfig::StoreConfig; /** @@ -20,7 +32,7 @@ struct LocalFSStoreConfig : virtual StoreConfig */ LocalFSStoreConfig(PathView path, const Params & params); - OptionalPathSetting rootDir{this, std::nullopt, "root", "Directory prefixed to all other paths."}; + OptionalPathSetting rootDir = makeRootDirSetting(*this, std::nullopt); private: @@ -68,6 +80,7 @@ struct LocalFSStore : virtual Store, virtual GcStore, virtual LogStore void narFromPath(const StorePath & path, Sink & sink) override; ref getFSAccessor(bool requireValidPath = true) override; + std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath = true) override; /** * Creates symlink from the `gcRoot` to the `storePath` and diff --git a/src/libstore/include/nix/store/local-store.hh b/src/libstore/include/nix/store/local-store.hh index f7dfcb5ad7e..b871aaee2ce 100644 --- a/src/libstore/include/nix/store/local-store.hh +++ b/src/libstore/include/nix/store/local-store.hh @@ -11,7 +11,7 @@ #include #include #include -#include +#include namespace nix { @@ -174,7 +174,11 @@ private: std::unique_ptr publicKeys; }; - Sync _state; + /** + * Mutable state. It's behind a `ref` to reduce false sharing + * between immutable and mutable fields. + */ + ref> _state; public: @@ -438,7 +442,7 @@ private: std::pair createTempDirInStore(); - typedef std::unordered_set InodeHash; + typedef boost::unordered_flat_set InodeHash; InodeHash loadInodeHash(); Strings readDirectoryIgnoringInodes(const Path & path, const InodeHash & inodeHash); diff --git a/src/libstore/include/nix/store/meson.build b/src/libstore/include/nix/store/meson.build index cba5d9ca51b..8e88ec51f66 100644 --- a/src/libstore/include/nix/store/meson.build +++ b/src/libstore/include/nix/store/meson.build @@ -10,11 +10,14 @@ config_pub_h = configure_file( ) headers = [ config_pub_h ] + files( + 'async-path-writer.hh', + 'aws-creds.hh', 'binary-cache-store.hh', 'build-result.hh', 'build/derivation-builder.hh', 'build/derivation-building-goal.hh', 'build/derivation-building-misc.hh', + 'build/derivation-env-desugar.hh', 'build/derivation-goal.hh', 'build/derivation-trampoline-goal.hh', 'build/drv-output-substitution-goal.hh', @@ -33,6 +36,9 @@ headers = [ config_pub_h ] + files( 'derived-path-map.hh', 'derived-path.hh', 'downstream-placeholder.hh', + 'dummy-store-impl.hh', + 'dummy-store.hh', + 'export-import.hh', 'filetransfer.hh', 'gc-store.hh', 'globals.hh', @@ -69,6 +75,7 @@ headers = [ config_pub_h ] + files( 'remote-store.hh', 'restricted-store.hh', 's3-binary-cache-store.hh', + 's3-url.hh', 's3.hh', 'serve-protocol-connection.hh', 'serve-protocol-impl.hh', diff --git a/src/libstore/include/nix/store/nar-info.hh b/src/libstore/include/nix/store/nar-info.hh index 39d75b0a90d..1684837c690 100644 --- a/src/libstore/include/nix/store/nar-info.hh +++ b/src/libstore/include/nix/store/nar-info.hh @@ -18,19 +18,20 @@ struct NarInfo : ValidPathInfo NarInfo() = delete; - NarInfo(const StoreDirConfig & store, std::string name, ContentAddressWithReferences ca, Hash narHash) - : ValidPathInfo(store, std::move(name), std::move(ca), narHash) + NarInfo(ValidPathInfo info) + : ValidPathInfo{std::move(info)} { } NarInfo(StorePath path, Hash narHash) - : ValidPathInfo(std::move(path), narHash) + : NarInfo{ValidPathInfo{std::move(path), UnkeyedValidPathInfo(narHash)}} { } - NarInfo(const ValidPathInfo & info) - : ValidPathInfo(info) + static NarInfo + makeFromCA(const StoreDirConfig & store, std::string_view name, ContentAddressWithReferences ca, Hash narHash) { + return ValidPathInfo::makeFromCA(store, std::move(name), std::move(ca), narHash); } NarInfo(const StoreDirConfig & store, const std::string & s, const std::string & whence); diff --git a/src/libstore/include/nix/store/path-info.hh b/src/libstore/include/nix/store/path-info.hh index 9f341198c51..cbc5abdb442 100644 --- a/src/libstore/include/nix/store/path-info.hh +++ b/src/libstore/include/nix/store/path-info.hh @@ -179,8 +179,8 @@ struct ValidPathInfo : UnkeyedValidPathInfo : UnkeyedValidPathInfo(info) , path(path) {}; - ValidPathInfo( - const StoreDirConfig & store, std::string_view name, ContentAddressWithReferences && ca, Hash narHash); + static ValidPathInfo + makeFromCA(const StoreDirConfig & store, std::string_view name, ContentAddressWithReferences && ca, Hash narHash); }; static_assert(std::is_move_assignable_v); diff --git a/src/libstore/include/nix/store/path.hh b/src/libstore/include/nix/store/path.hh index 784298daaac..74ee0422bee 100644 --- a/src/libstore/include/nix/store/path.hh +++ b/src/libstore/include/nix/store/path.hh @@ -4,6 +4,8 @@ #include #include "nix/util/types.hh" +#include "nix/util/json-impls.hh" +#include "nix/util/json-non-null.hh" namespace nix { @@ -87,6 +89,10 @@ typedef std::vector StorePaths; */ constexpr std::string_view drvExtension = ".drv"; +template<> +struct json_avoids_null : std::true_type +{}; + } // namespace nix namespace std { @@ -101,3 +107,14 @@ struct hash }; } // namespace std + +namespace nix { + +inline std::size_t hash_value(const StorePath & path) +{ + return std::hash{}(path); +} + +} // namespace nix + +JSON_IMPL(nix::StorePath) diff --git a/src/libstore/include/nix/store/realisation.hh b/src/libstore/include/nix/store/realisation.hh index 6eb3eecf3f6..3424a39c9c8 100644 --- a/src/libstore/include/nix/store/realisation.hh +++ b/src/libstore/include/nix/store/realisation.hh @@ -64,9 +64,6 @@ struct Realisation */ std::map dependentRealisations; - nlohmann::json toJSON() const; - static Realisation fromJSON(const nlohmann::json & json, const std::string & whence); - std::string fingerprint() const; void sign(const Signer &); bool checkSignature(const PublicKeys & publicKeys, const std::string & sig) const; @@ -169,3 +166,5 @@ public: }; } // namespace nix + +JSON_IMPL(nix::Realisation) diff --git a/src/libstore/include/nix/store/remote-fs-accessor.hh b/src/libstore/include/nix/store/remote-fs-accessor.hh index fa0555d9b71..9e1999cc061 100644 --- a/src/libstore/include/nix/store/remote-fs-accessor.hh +++ b/src/libstore/include/nix/store/remote-fs-accessor.hh @@ -27,6 +27,11 @@ class RemoteFSAccessor : public SourceAccessor public: + /** + * @return nullptr if the store does not contain any object at that path. + */ + std::shared_ptr accessObject(const StorePath & path); + RemoteFSAccessor( ref store, bool requireValidPath = true, const /* FIXME: use std::optional */ Path & cacheDir = ""); diff --git a/src/libstore/include/nix/store/remote-store.hh b/src/libstore/include/nix/store/remote-store.hh index 76591cf9390..df9162d134c 100644 --- a/src/libstore/include/nix/store/remote-store.hh +++ b/src/libstore/include/nix/store/remote-store.hh @@ -16,13 +16,14 @@ struct FdSink; struct FdSource; template class Pool; +class RemoteFSAccessor; struct RemoteStoreConfig : virtual StoreConfig { using StoreConfig::StoreConfig; const Setting maxConnections{ - this, 1, "max-connections", "Maximum number of concurrent connections to the Nix daemon."}; + this, 64, "max-connections", "Maximum number of concurrent connections to the Nix daemon."}; const Setting maxConnectionAge{ this, @@ -176,10 +177,18 @@ protected: virtual ref getFSAccessor(bool requireValidPath = true) override; + virtual std::shared_ptr + getFSAccessor(const StorePath & path, bool requireValidPath = true) override; + virtual void narFromPath(const StorePath & path, Sink & sink) override; private: + /** + * Same as the default implemenation of `RemoteStore::getFSAccessor`, but with a more preceise return type. + */ + ref getRemoteFSAccessor(bool requireValidPath = true); + std::atomic_bool failed{false}; void copyDrvsFromEvalStore(const std::vector & paths, std::shared_ptr evalStore); diff --git a/src/libstore/include/nix/store/s3-url.hh b/src/libstore/include/nix/store/s3-url.hh new file mode 100644 index 00000000000..45c3b2d1c7b --- /dev/null +++ b/src/libstore/include/nix/store/s3-url.hh @@ -0,0 +1,60 @@ +#pragma once +///@file +#include "nix/store/config.hh" + +#if NIX_WITH_S3_SUPPORT || NIX_WITH_CURL_S3 + +# include "nix/util/url.hh" +# include "nix/util/util.hh" + +# include +# include +# include +# include + +namespace nix { + +/** + * Parsed S3 URL. + */ +struct ParsedS3URL +{ + std::string bucket; + /** + * @see ParsedURL::path. This is a vector for the same reason. + * Unlike ParsedURL::path this doesn't include the leading empty segment, + * since the bucket name is necessary. + */ + std::vector key; + std::optional profile; + std::optional region; + std::optional scheme; + /** + * The endpoint can be either missing, be an absolute URI (with a scheme like `http:`) + * or an authority (so an IP address or a registered name). + */ + std::variant endpoint; + + std::optional getEncodedEndpoint() const + { + return std::visit( + overloaded{ + [](std::monostate) -> std::optional { return std::nullopt; }, + [](const auto & authorityOrUrl) -> std::optional { return authorityOrUrl.to_string(); }, + }, + endpoint); + } + + static ParsedS3URL parse(const ParsedURL & uri); + + /** + * Convert this ParsedS3URL to HTTPS ParsedURL for use with curl's AWS SigV4 authentication + */ + ParsedURL toHttpsUrl() const; + + auto operator<=>(const ParsedS3URL & other) const = default; +}; + +} // namespace nix + +#endif diff --git a/src/libstore/include/nix/store/s3.hh b/src/libstore/include/nix/store/s3.hh index 51782595287..ba3adbc2a28 100644 --- a/src/libstore/include/nix/store/s3.hh +++ b/src/libstore/include/nix/store/s3.hh @@ -4,12 +4,9 @@ #if NIX_WITH_S3_SUPPORT # include "nix/util/ref.hh" -# include "nix/util/url.hh" -# include "nix/util/util.hh" +# include "nix/store/s3-url.hh" -# include # include -# include namespace Aws { namespace Client { @@ -48,36 +45,6 @@ struct S3Helper FileTransferResult getObject(const std::string & bucketName, const std::string & key); }; -/** - * Parsed S3 URL. - */ -struct ParsedS3URL -{ - std::string bucket; - std::string key; - std::optional profile; - std::optional region; - std::optional scheme; - /** - * The endpoint can be either missing, be an absolute URI (with a scheme like `http:`) - * or an authority (so an IP address or a registered name). - */ - std::variant endpoint; - - std::optional getEncodedEndpoint() const - { - return std::visit( - overloaded{ - [](std::monostate) -> std::optional { return std::nullopt; }, - [](const auto & authorityOrUrl) -> std::optional { return authorityOrUrl.to_string(); }, - }, - endpoint); - } - - static ParsedS3URL parse(std::string_view uri); - auto operator<=>(const ParsedS3URL & other) const = default; -}; - } // namespace nix #endif diff --git a/src/libstore/include/nix/store/serve-protocol-connection.hh b/src/libstore/include/nix/store/serve-protocol-connection.hh index fa50132c88b..873277db902 100644 --- a/src/libstore/include/nix/store/serve-protocol-connection.hh +++ b/src/libstore/include/nix/store/serve-protocol-connection.hh @@ -82,8 +82,6 @@ struct ServeProto::BasicClientConnection BuildResult getBuildDerivationResponse(const StoreDirConfig & store); void narFromPath(const StoreDirConfig & store, const StorePath & path, std::function fun); - - void importPaths(const StoreDirConfig & store, std::function fun); }; struct ServeProto::BasicServerConnection diff --git a/src/libstore/include/nix/store/serve-protocol.hh b/src/libstore/include/nix/store/serve-protocol.hh index c8f3560d181..5cb8a89a8d3 100644 --- a/src/libstore/include/nix/store/serve-protocol.hh +++ b/src/libstore/include/nix/store/serve-protocol.hh @@ -108,8 +108,8 @@ enum struct ServeProto::Command : uint64_t { QueryValidPaths = 1, QueryPathInfos = 2, DumpStorePath = 3, - ImportPaths = 4, - ExportPaths = 5, + // ImportPaths = 4, // removed + // ExportPaths = 5, // removed BuildPaths = 6, QueryClosure = 7, BuildDerivation = 8, diff --git a/src/libstore/include/nix/store/ssh.hh b/src/libstore/include/nix/store/ssh.hh index c7228464b66..574cb5cf414 100644 --- a/src/libstore/include/nix/store/ssh.hh +++ b/src/libstore/include/nix/store/ssh.hh @@ -1,6 +1,7 @@ #pragma once ///@file +#include "nix/util/ref.hh" #include "nix/util/sync.hh" #include "nix/util/url.hh" #include "nix/util/processes.hh" @@ -26,12 +27,13 @@ private: const bool compress; const Descriptor logFD; + const ref tmpDir; + struct State { #ifndef _WIN32 // TODO re-enable on Windows, once we can start processes. Pid sshMaster; #endif - std::unique_ptr tmpDir; Path socketPath; }; diff --git a/src/libstore/include/nix/store/store-api.hh b/src/libstore/include/nix/store/store-api.hh index 987ed4d4869..09c8799e1f8 100644 --- a/src/libstore/include/nix/store/store-api.hh +++ b/src/libstore/include/nix/store/store-api.hh @@ -24,11 +24,6 @@ namespace nix { -MakeError(SubstError, Error); -/** - * denotes a permanent build failure - */ -MakeError(BuildError, Error); MakeError(InvalidPath, Error); MakeError(Unsupported, Error); MakeError(SubstituteGone, Error); @@ -53,11 +48,6 @@ enum CheckSigsFlag : bool { NoCheckSigs = false, CheckSigs = true }; enum SubstituteFlag : bool { NoSubstitute = false, Substitute = true }; -/** - * Magic header of exportPath() output (obsolete). - */ -const uint32_t exportMagic = 0x4558494e; - enum BuildMode : uint8_t { bmNormal, bmRepair, bmCheck }; enum TrustedFlag : bool { NotTrusted = false, Trusted = true }; @@ -315,14 +305,11 @@ protected: } }; - struct State - { - LRUCache pathInfoCache; - }; - void invalidatePathInfoCacheFor(const StorePath & path); - SharedSync state; + // Note: this is a `ref` to avoid false sharing with immutable + // bits of `Store`. + ref>> pathInfoCache; std::shared_ptr diskCache; @@ -349,7 +336,9 @@ public: StorePath followLinksToStorePath(std::string_view path) const; /** - * Check whether a path is valid. + * Check whether a path is valid. NOTE: this function does not + * generally cache whether a path is valid. You may want to use + * `maybeQueryPathInfo()`, which does cache. */ bool isValidPath(const StorePath & path); @@ -389,10 +378,17 @@ public: /** * Query information about a valid path. It is permitted to omit - * the name part of the store path. + * the name part of the store path. Throws an exception if the + * path is not valid. */ ref queryPathInfo(const StorePath & path); + /** + * Like `queryPathInfo()`, but returns `nullptr` if the path is + * not valid. + */ + std::shared_ptr maybeQueryPathInfo(const StorePath & path); + /** * Asynchronous version of queryPathInfo(). */ @@ -611,10 +607,7 @@ public: * floating-ca derivations and their dependencies as there's no way to * retrieve this information otherwise. */ - virtual void registerDrvOutput(const Realisation & output) - { - unsupported("registerDrvOutput"); - } + virtual void registerDrvOutput(const Realisation & output) = 0; virtual void registerDrvOutput(const Realisation & output, CheckSigsFlag checkSigs) { @@ -730,10 +723,20 @@ public: }; /** - * @return An object to access files in the Nix store. + * @return An object to access files in the Nix store, across all + * store objects. */ virtual ref getFSAccessor(bool requireValidPath = true) = 0; + /** + * @return An object to access files for a specific store object in + * the Nix store. + * + * @return nullptr if the store doesn't contain an object at the + * givine path. + */ + virtual std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath = true) = 0; + /** * Repair the contents of the given path by redownloading it using * a substituter (if available). @@ -812,21 +815,6 @@ public: */ StorePaths topoSortPaths(const StorePathSet & paths); - /** - * Export multiple paths in the format expected by ‘nix-store - * --import’. - */ - void exportPaths(const StorePathSet & paths, Sink & sink); - - void exportPath(const StorePath & path, Sink & sink); - - /** - * Import a sequence of NAR dumps created by exportPaths() into the - * Nix store. Optionally, the contents of the NARs are preloaded - * into the specified FS accessor to speed up subsequent access. - */ - StorePaths importPaths(Source & source, CheckSigsFlag checkSigs = CheckSigs); - struct Stats { std::atomic narInfoRead{0}; @@ -865,7 +853,7 @@ public: */ void clearPathInfoCache() { - state.lock()->pathInfoCache.clear(); + pathInfoCache->lock()->clear(); } /** diff --git a/src/libstore/include/nix/store/store-reference.hh b/src/libstore/include/nix/store/store-reference.hh index 5cf1e9a11e9..dc34500d9cb 100644 --- a/src/libstore/include/nix/store/store-reference.hh +++ b/src/libstore/include/nix/store/store-reference.hh @@ -64,7 +64,29 @@ struct StoreReference auto operator<=>(const Specified & rhs) const = default; }; - typedef std::variant Variant; + /** + * Special case for `daemon` to avoid normalization. + */ + struct Daemon : Specified + { + Daemon() + : Specified({.scheme = "unix"}) + { + } + }; + + /** + * Special case for `local` to avoid normalization. + */ + struct Local : Specified + { + Local() + : Specified({.scheme = "local"}) + { + } + }; + + typedef std::variant Variant; Variant variant; @@ -77,12 +99,22 @@ struct StoreReference */ std::string render(bool withParams = true) const; + std::string to_string() const + { + return render(); + } + /** * Parse a URI into a store reference. */ static StoreReference parse(const std::string & uri, const Params & extraParams = Params{}); }; +static inline std::ostream & operator<<(std::ostream & os, const StoreReference & ref) +{ + return os << ref.render(); +} + /** * Split URI into protocol+hierarchy part and its parameter set. */ diff --git a/src/libstore/include/nix/store/uds-remote-store.hh b/src/libstore/include/nix/store/uds-remote-store.hh index 37c239796d9..fe6e486f412 100644 --- a/src/libstore/include/nix/store/uds-remote-store.hh +++ b/src/libstore/include/nix/store/uds-remote-store.hh @@ -61,6 +61,11 @@ struct UDSRemoteStore : virtual IndirectRootStore, virtual RemoteStore return LocalFSStore::getFSAccessor(requireValidPath); } + std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath = true) override + { + return LocalFSStore::getFSAccessor(path, requireValidPath); + } + void narFromPath(const StorePath & path, Sink & sink) override { LocalFSStore::narFromPath(path, sink); diff --git a/src/libstore/include/nix/store/worker-protocol-connection.hh b/src/libstore/include/nix/store/worker-protocol-connection.hh index 73dd507192c..31436395fe7 100644 --- a/src/libstore/include/nix/store/worker-protocol-connection.hh +++ b/src/libstore/include/nix/store/worker-protocol-connection.hh @@ -130,8 +130,6 @@ struct WorkerProto::BasicClientConnection : WorkerProto::BasicConnection bool * daemonException, const StorePath & path, std::function fun); - - void importPaths(const StoreDirConfig & store, bool * daemonException, Source & source); }; struct WorkerProto::BasicServerConnection : WorkerProto::BasicConnection diff --git a/src/libstore/include/nix/store/worker-protocol-impl.hh b/src/libstore/include/nix/store/worker-protocol-impl.hh index 26f6b9d44e4..c36145d620d 100644 --- a/src/libstore/include/nix/store/worker-protocol-impl.hh +++ b/src/libstore/include/nix/store/worker-protocol-impl.hh @@ -45,12 +45,14 @@ struct WorkerProto::Serialise { static T read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { - return CommonProto::Serialise::read(store, CommonProto::ReadConn{.from = conn.from}); + return CommonProto::Serialise::read( + store, CommonProto::ReadConn{.from = conn.from, .shortStorePaths = conn.shortStorePaths}); } static void write(const StoreDirConfig & store, WorkerProto::WriteConn conn, const T & t) { - CommonProto::Serialise::write(store, CommonProto::WriteConn{.to = conn.to}, t); + CommonProto::Serialise::write( + store, CommonProto::WriteConn{.to = conn.to, .shortStorePaths = conn.shortStorePaths}, t); } }; diff --git a/src/libstore/include/nix/store/worker-protocol.hh b/src/libstore/include/nix/store/worker-protocol.hh index c7f8d589100..79d59144cdf 100644 --- a/src/libstore/include/nix/store/worker-protocol.hh +++ b/src/libstore/include/nix/store/worker-protocol.hh @@ -13,6 +13,7 @@ namespace nix { /* Note: you generally shouldn't change the protocol version. Define a new `WorkerProto::Feature` instead. */ #define PROTOCOL_VERSION (1 << 8 | 38) +#define MINIMUM_PROTOCOL_VERSION (1 << 8 | 18) #define GET_PROTOCOL_MAJOR(x) ((x) & 0xff00) #define GET_PROTOCOL_MINOR(x) ((x) & 0x00ff) @@ -65,6 +66,7 @@ struct WorkerProto { Source & from; Version version; + bool shortStorePaths = false; }; /** @@ -75,6 +77,7 @@ struct WorkerProto { Sink & to; Version version; + bool shortStorePaths = false; }; /** @@ -152,7 +155,7 @@ enum struct WorkerProto::Op : uint64_t { AddIndirectRoot = 12, SyncWithGC = 13, FindRoots = 14, - ExportPath = 16, // obsolete + // ExportPath = 16, // removed QueryDeriver = 18, // obsolete SetOptions = 19, CollectGarbage = 20, @@ -162,7 +165,7 @@ enum struct WorkerProto::Op : uint64_t { QueryFailedPaths = 24, ClearFailedPaths = 25, QueryPathInfo = 26, - ImportPaths = 27, // obsolete + // ImportPaths = 27, // removed QueryDerivationOutputNames = 28, // obsolete QueryPathFromHashPart = 29, QuerySubstitutablePathInfos = 30, diff --git a/src/libstore/legacy-ssh-store.cc b/src/libstore/legacy-ssh-store.cc index 0e9ee35bf3a..3b466c9bb8b 100644 --- a/src/libstore/legacy-ssh-store.cc +++ b/src/libstore/legacy-ssh-store.cc @@ -105,9 +105,6 @@ std::map LegacySSHStore::queryPathInfosUncached { auto conn(connections->get()); - /* No longer support missing NAR hash */ - assert(GET_PROTOCOL_MINOR(conn->remoteVersion) >= 4); - debug( "querying remote host '%s' for info on '%s'", config->authority.host, @@ -152,40 +149,21 @@ void LegacySSHStore::addToStore(const ValidPathInfo & info, Source & source, Rep auto conn(connections->get()); - if (GET_PROTOCOL_MINOR(conn->remoteVersion) >= 5) { - - conn->to << ServeProto::Command::AddToStoreNar << printStorePath(info.path) - << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(HashFormat::Base16, false); - ServeProto::write(*this, *conn, info.references); - conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs - << renderContentAddress(info.ca); - try { - copyNAR(source, conn->to); - } catch (...) { - conn->good = false; - throw; - } - conn->to.flush(); - - if (readInt(conn->from) != 1) - throw Error( - "failed to add path '%s' to remote host '%s'", printStorePath(info.path), config->authority.host); - - } else { - - conn->importPaths(*this, [&](Sink & sink) { - try { - copyNAR(source, sink); - } catch (...) { - conn->good = false; - throw; - } - sink << exportMagic << printStorePath(info.path); - ServeProto::write(*this, *conn, info.references); - sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 << 0; - }); + conn->to << ServeProto::Command::AddToStoreNar << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(HashFormat::Base16, false); + ServeProto::write(*this, *conn, info.references); + conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca); + try { + copyNAR(source, conn->to); + } catch (...) { + conn->good = false; + throw; } + conn->to.flush(); + + if (readInt(conn->from) != 1) + throw Error("failed to add path '%s' to remote host '%s'", printStorePath(info.path), config->authority.host); } void LegacySSHStore::narFromPath(const StorePath & path, Sink & sink) @@ -263,12 +241,13 @@ void LegacySSHStore::buildPaths( conn->to.flush(); - BuildResult result; - result.status = (BuildResult::Status) readInt(conn->from); - - if (!result.success()) { - conn->from >> result.errorMsg; - throw Error(result.status, result.errorMsg); + auto status = readInt(conn->from); + if (!BuildResult::Success::statusIs(status)) { + BuildResult::Failure failure{ + .status = (BuildResult::Failure::Status) status, + }; + conn->from >> failure.errorMsg; + throw Error(failure.status, std::move(failure.errorMsg)); } } @@ -302,22 +281,6 @@ StorePathSet LegacySSHStore::queryValidPaths(const StorePathSet & paths, bool lo return conn->queryValidPaths(*this, lock, paths, maybeSubstitute); } -void LegacySSHStore::addMultipleToStoreLegacy(Store & srcStore, const StorePathSet & paths) -{ - auto conn(connections->get()); - conn->to << ServeProto::Command::ImportPaths; - try { - srcStore.exportPaths(paths, conn->to); - } catch (...) { - conn->good = false; - throw; - } - conn->to.flush(); - - if (readInt(conn->from) != 1) - throw Error("remote machine failed to import closure"); -} - void LegacySSHStore::connect() { auto conn(connections->get()); diff --git a/src/libstore/local-fs-store.cc b/src/libstore/local-fs-store.cc index e0f07b91b66..67024c4e365 100644 --- a/src/libstore/local-fs-store.cc +++ b/src/libstore/local-fs-store.cc @@ -20,13 +20,17 @@ Path LocalFSStoreConfig::getDefaultLogDir() LocalFSStoreConfig::LocalFSStoreConfig(PathView rootDir, const Params & params) : StoreConfig(params) - // Default `?root` from `rootDir` if non set - // FIXME don't duplicate description once we don't have root setting - , rootDir{ - this, - !rootDir.empty() && params.count("root") == 0 ? (std::optional{rootDir}) : std::nullopt, - "root", - "Directory prefixed to all other paths."} + /* Default `?root` from `rootDir` if non set + * NOTE: We would like to just do rootDir.set(...), which would take care of + * all normalization and error checking for us. Unfortunately we cannot do + * that because of the complicated initialization order of other fields with + * the virtual class hierarchy of nix store configs, and the design of the + * settings system. As such, we have no choice but to redefine the field and + * manually repeat the same normalization logic. + */ + , rootDir{makeRootDirSetting( + *this, + !rootDir.empty() && params.count("root") == 0 ? std::optional{canonPath(rootDir)} : std::nullopt)} { } @@ -51,7 +55,7 @@ struct LocalStoreAccessor : PosixSourceAccessor void requireStoreObject(const CanonPath & path) { auto [storePath, rest] = store->toStorePath(store->storeDir + path.abs()); - if (requireValidPath && !store->isValidPath(storePath)) + if (requireValidPath && !store->maybeQueryPathInfo(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); } @@ -91,6 +95,23 @@ ref LocalFSStore::getFSAccessor(bool requireValidPath) ref(std::dynamic_pointer_cast(shared_from_this())), requireValidPath); } +std::shared_ptr LocalFSStore::getFSAccessor(const StorePath & path, bool requireValidPath) +{ + auto absPath = std::filesystem::path{config.realStoreDir.get()} / path.to_string(); + if (requireValidPath) { + /* Only return non-null if the store object is a fully-valid + member of the store. */ + if (!isValidPath(path)) + return nullptr; + } else { + /* Return non-null as long as the some file system data exists, + even if the store object is not fully registered. */ + if (!pathExists(absPath)) + return nullptr; + } + return std::make_shared(std::move(absPath)); +} + void LocalFSStore::narFromPath(const StorePath & path, Sink & sink) { if (!isValidPath(path)) diff --git a/src/libstore/local-store.cc b/src/libstore/local-store.cc index a66a9786677..5c8c5f1575a 100644 --- a/src/libstore/local-store.cc +++ b/src/libstore/local-store.cc @@ -118,6 +118,7 @@ LocalStore::LocalStore(ref config) : Store{*config} , LocalFSStore{*config} , config{config} + , _state(make_ref>()) , dbDir(config->stateDir + "/db") , linksDir(config->realStoreDir + "/.links") , reservedPath(dbDir + "/reserved") @@ -125,7 +126,7 @@ LocalStore::LocalStore(ref config) , tempRootsDir(config->stateDir + "/temproots") , fnTempRoots(fmt("%s/%d", tempRootsDir, getpid())) { - auto state(_state.lock()); + auto state(_state->lock()); state->stmts = std::make_unique(); /* Create missing state directories if they don't already exist. */ @@ -433,7 +434,7 @@ LocalStore::~LocalStore() std::shared_future future; { - auto state(_state.lock()); + auto state(_state->lock()); if (state->gcRunning) future = state->gcFuture; } @@ -456,12 +457,17 @@ LocalStore::~LocalStore() StoreReference LocalStoreConfig::getReference() const { + auto params = getQueryParams(); + /* Back-compatibility kludge. Tools like nix-output-monitor expect 'local' + and can't parse 'local://'. */ + if (params.empty()) + return {.variant = StoreReference::Local{}}; return { .variant = StoreReference::Specified{ .scheme = *uriSchemes().begin(), }, - .params = getQueryParams(), + .params = std::move(params), }; } @@ -624,7 +630,7 @@ void LocalStore::registerDrvOutput(const Realisation & info) { experimentalFeatureSettings.require(Xp::CaDerivations); retrySQLite([&]() { - auto state(_state.lock()); + auto state(_state->lock()); if (auto oldR = queryRealisation_(*state, info.id)) { if (info.isCompatibleWith(*oldR)) { auto combinedSignatures = oldR->signatures; @@ -716,12 +722,8 @@ uint64_t LocalStore::addValidPath(State & state, const ValidPathInfo & info, boo } } - { - auto state_(Store::state.lock()); - state_->pathInfoCache.upsert( - std::string(info.path.to_string()), - PathInfoCacheValue{.value = std::make_shared(info)}); - } + pathInfoCache->lock()->upsert( + std::string(info.path.to_string()), PathInfoCacheValue{.value = std::make_shared(info)}); return id; } @@ -731,8 +733,7 @@ void LocalStore::queryPathInfoUncached( { try { callback(retrySQLite>([&]() { - auto state(_state.lock()); - return queryPathInfoInternal(*state, path); + return queryPathInfoInternal(*_state->lock(), path); })); } catch (...) { @@ -814,10 +815,7 @@ bool LocalStore::isValidPath_(State & state, const StorePath & path) bool LocalStore::isValidPathUncached(const StorePath & path) { - return retrySQLite([&]() { - auto state(_state.lock()); - return isValidPath_(*state, path); - }); + return retrySQLite([&]() { return isValidPath_(*_state->lock(), path); }); } StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) @@ -832,7 +830,7 @@ StorePathSet LocalStore::queryValidPaths(const StorePathSet & paths, SubstituteF StorePathSet LocalStore::queryAllValidPaths() { return retrySQLite([&]() { - auto state(_state.lock()); + auto state(_state->lock()); auto use(state->stmts->QueryValidPaths.use()); StorePathSet res; while (use.next()) @@ -851,16 +849,13 @@ void LocalStore::queryReferrers(State & state, const StorePath & path, StorePath void LocalStore::queryReferrers(const StorePath & path, StorePathSet & referrers) { - return retrySQLite([&]() { - auto state(_state.lock()); - queryReferrers(*state, path, referrers); - }); + return retrySQLite([&]() { queryReferrers(*_state->lock(), path, referrers); }); } StorePathSet LocalStore::queryValidDerivers(const StorePath & path) { return retrySQLite([&]() { - auto state(_state.lock()); + auto state(_state->lock()); auto useQueryValidDerivers(state->stmts->QueryValidDerivers.use()(printStorePath(path))); @@ -876,7 +871,7 @@ std::map> LocalStore::queryStaticPartialDerivationOutputMap(const StorePath & path) { return retrySQLite>>([&]() { - auto state(_state.lock()); + auto state(_state->lock()); std::map> outputs; uint64_t drvId; drvId = queryValidPathId(*state, path); @@ -896,7 +891,7 @@ std::optional LocalStore::queryPathFromHashPart(const std::string & h Path prefix = storeDir + "/" + hashPart; return retrySQLite>([&]() -> std::optional { - auto state(_state.lock()); + auto state(_state->lock()); auto useQueryPathFromHashPart(state->stmts->QueryPathFromHashPart.use()(prefix)); @@ -961,7 +956,7 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) #endif return retrySQLite([&]() { - auto state(_state.lock()); + auto state(_state->lock()); SQLiteTxn txn(state->db); StorePathSet paths; @@ -1002,7 +997,10 @@ void LocalStore::registerValidPaths(const ValidPathInfos & infos) }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( - "cycle detected in the references of '%s' from '%s'", printStorePath(path), printStorePath(parent)); + BuildResult::Failure::OutputRejected, + "cycle detected in the references of '%s' from '%s'", + printStorePath(path), + printStorePath(parent)); }}); txn.commit(); @@ -1020,15 +1018,12 @@ void LocalStore::invalidatePath(State & state, const StorePath & path) /* Note that the foreign key constraints on the Refs table take care of deleting the references entries for `path'. */ - { - auto state_(Store::state.lock()); - state_->pathInfoCache.erase(std::string(path.to_string())); - } + pathInfoCache->lock()->erase(std::string(path.to_string())); } const PublicKeys & LocalStore::getPublicKeys() { - auto state(_state.lock()); + auto state(_state->lock()); if (!state->publicKeys) state->publicKeys = std::make_unique(getDefaultPublicKeys()); return *state->publicKeys; @@ -1053,15 +1048,13 @@ void LocalStore::addToStore(const ValidPathInfo & info, Source & source, RepairF /* In case we are not interested in reading the NAR: discard it. */ bool narRead = false; Finally cleanup = [&]() { - if (!narRead) { - NullFileSystemObjectSink sink; + if (!narRead) try { - parseDump(sink, source); + source.skip(info.narSize); } catch (...) { // TODO: should Interrupted be handled here? ignoreExceptionInDestructor(); } - } }; addTempRoot(info.path); @@ -1316,7 +1309,7 @@ StorePath LocalStore::addToStoreFromDump( syncParent(realPath); } - ValidPathInfo info{*this, name, std::move(desc), narHash.hash}; + auto info = ValidPathInfo::makeFromCA(*this, name, std::move(desc), narHash.hash); info.narSize = narHash.numBytesDigested; registerValidPath(info); } @@ -1351,7 +1344,7 @@ std::pair LocalStore::createTempDirInStore() void LocalStore::invalidatePathChecked(const StorePath & path) { retrySQLite([&]() { - auto state(_state.lock()); + auto state(_state->lock()); SQLiteTxn txn(state->db); @@ -1388,7 +1381,7 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) for (auto & link : DirectoryIterator{linksDir}) { checkInterrupt(); auto name = link.path().filename(); - printMsg(lvlTalkative, "checking contents of '%s'", name); + printMsg(lvlTalkative, "checking contents of %s", name); PosixSourceAccessor accessor; std::string hash = hashPath( PosixSourceAccessor::createAtRoot(link.path()), @@ -1396,10 +1389,10 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) HashAlgorithm::SHA256) .first.to_string(HashFormat::Nix32, false); if (hash != name.string()) { - printError("link '%s' was modified! expected hash '%s', got '%s'", link.path(), name, hash); + printError("link %s was modified! expected hash %s, got '%s'", link.path(), name, hash); if (repair) { std::filesystem::remove(link.path()); - printInfo("removed link '%s'", link.path()); + printInfo("removed link %s", link.path()); } else { errors = true; } @@ -1451,10 +1444,8 @@ bool LocalStore::verifyStore(bool checkContents, RepairFlag repair) update = true; } - if (update) { - auto state(_state.lock()); - updatePathInfo(*state, *info); - } + if (update) + updatePathInfo(*_state->lock(), *info); } } catch (Error & e) { @@ -1541,8 +1532,7 @@ void LocalStore::verifyPath( if (canInvalidate) { printInfo("path '%s' disappeared, removing from database...", pathS); - auto state(_state.lock()); - invalidatePath(*state, path); + invalidatePath(*_state->lock(), path); } else { printError("path '%s' disappeared, but it still has valid referrers!", pathS); if (repair) @@ -1574,14 +1564,13 @@ std::optional LocalStore::isTrustedClient() void LocalStore::vacuumDB() { - auto state(_state.lock()); - state->db.exec("vacuum"); + _state->lock()->db.exec("vacuum"); } void LocalStore::addSignatures(const StorePath & storePath, const StringSet & sigs) { retrySQLite([&]() { - auto state(_state.lock()); + auto state(_state->lock()); SQLiteTxn txn(state->db); @@ -1643,10 +1632,8 @@ void LocalStore::queryRealisationUncached( const DrvOutput & id, Callback> callback) noexcept { try { - auto maybeRealisation = retrySQLite>([&]() { - auto state(_state.lock()); - return queryRealisation_(*state, id); - }); + auto maybeRealisation = + retrySQLite>([&]() { return queryRealisation_(*_state->lock(), id); }); if (maybeRealisation) callback(std::make_shared(maybeRealisation.value())); else diff --git a/src/libstore/make-content-addressed.cc b/src/libstore/make-content-addressed.cc index ce4a36849d8..4a7b21c3b12 100644 --- a/src/libstore/make-content-addressed.cc +++ b/src/libstore/make-content-addressed.cc @@ -45,7 +45,7 @@ std::map makeContentAddressed(Store & srcStore, Store & ds auto narModuloHash = hashModuloSink.finish().hash; - ValidPathInfo info{ + auto info = ValidPathInfo::makeFromCA( dstStore, path.name(), FixedOutputInfo{ @@ -53,8 +53,7 @@ std::map makeContentAddressed(Store & srcStore, Store & ds .hash = narModuloHash, .references = std::move(refs), }, - Hash::dummy, - }; + Hash::dummy); printInfo("rewriting '%s' to '%s'", pathS, dstStore.printStorePath(info.path)); diff --git a/src/libstore/meson.build b/src/libstore/meson.build index ad130945e18..5ca54930df6 100644 --- a/src/libstore/meson.build +++ b/src/libstore/meson.build @@ -13,6 +13,8 @@ project( license : 'LGPL-2.1-or-later', ) +fs = import('fs') + cxx = meson.get_compiler('cpp') subdir('nix-meson-build-support/deps-lists') @@ -23,6 +25,11 @@ configdata_priv = configuration_data() # TODO rename, because it will conflict with downstream projects configdata_priv.set_quoted('PACKAGE_VERSION', meson.project_version()) +configdata_priv.set_quoted( + 'DETERMINATE_NIX_VERSION', + fs.read('../../.version-determinate').strip(), +) + subdir('nix-meson-build-support/default-system-cpu') # Used in public header. @@ -101,7 +108,12 @@ subdir('nix-meson-build-support/libatomic') boost = dependency( 'boost', - modules : [ 'container', 'regex' ], + modules : [ + 'container', + # Shouldn't list, because can header-only, and Meson currently looks for libs + #'regex', + 'url', + ], include_type : 'system', ) # boost is a public dependency, but not a pkg-config dependency unfortunately, so we @@ -159,6 +171,33 @@ if aws_s3.found() endif deps_other += aws_s3 +# Curl-based S3 store support (alternative to AWS SDK) +# Check if curl supports AWS SigV4 (requires >= 7.75.0) +curl_supports_aws_sigv4 = curl.version().version_compare('>= 7.75.0') +# AWS CRT C++ for lightweight credential management +aws_crt_cpp = cxx.find_library('aws-crt-cpp', required : false) + +curl_s3_store_opt = get_option('curl-s3-store').require( + curl_supports_aws_sigv4, + error_message : 'curl-based S3 support requires curl >= 7.75.0', +).require( + aws_crt_cpp.found(), + error_message : 'curl-based S3 support requires aws-crt-cpp', +) + +# Make AWS SDK and curl-based S3 mutually exclusive +if aws_s3.found() and curl_s3_store_opt.enabled() + error( + 'Cannot enable both AWS SDK S3 support and curl-based S3 support. Please choose one.', + ) +endif + +if curl_s3_store_opt.enabled() + deps_other += aws_crt_cpp +endif + +configdata_pub.set('NIX_WITH_CURL_S3', curl_s3_store_opt.enabled().to_int()) + subdir('nix-meson-build-support/generate-header') generated_headers = [] @@ -195,8 +234,6 @@ if get_option('embedded-sandbox-shell') generated_headers += embedded_sandbox_shell_gen endif -fs = import('fs') - prefix = get_option('prefix') # For each of these paths, assume that it is relative to the prefix unless # it is already an absolute path (which is the default for store-dir, localstatedir, and log-dir). @@ -260,11 +297,16 @@ config_priv_h = configure_file( ) subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( + 'async-path-writer.cc', + 'aws-creds.cc', 'binary-cache-store.cc', 'build-result.cc', 'build/derivation-building-goal.cc', + 'build/derivation-check.cc', + 'build/derivation-env-desugar.cc', 'build/derivation-goal.cc', 'build/derivation-trampoline-goal.cc', 'build/drv-output-substitution-goal.cc', @@ -321,7 +363,7 @@ sources = files( 'remote-store.cc', 'restricted-store.cc', 's3-binary-cache-store.cc', - 's3.cc', + 's3-url.cc', 'serve-protocol-connection.cc', 'serve-protocol.cc', 'sqlite.cc', @@ -356,6 +398,7 @@ this_library = library( generated_headers, sources, config_priv_h, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, diff --git a/src/libstore/meson.options b/src/libstore/meson.options index b8414068de1..edc43bd4513 100644 --- a/src/libstore/meson.options +++ b/src/libstore/meson.options @@ -33,3 +33,10 @@ option( value : '/nix/var/log/nix', description : 'path to store logs in for Nix', ) + +option( + 'curl-s3-store', + type : 'feature', + value : 'disabled', + description : 'Enable curl-based S3 binary cache store support (requires aws-crt-cpp and curl >= 7.75.0)', +) diff --git a/src/libstore/misc.cc b/src/libstore/misc.cc index c794f8d068b..7efaa4f860e 100644 --- a/src/libstore/misc.cc +++ b/src/libstore/misc.cc @@ -1,5 +1,3 @@ -#include - #include "nix/store/derivations.hh" #include "nix/store/parsed-derivations.hh" #include "nix/store/derivation-options.hh" @@ -13,6 +11,8 @@ #include "nix/store/filetransfer.hh" #include "nix/util/strings.hh" +#include + namespace nix { void Store::computeFSClosure( @@ -106,7 +106,7 @@ MissingPaths Store::queryMissing(const std::vector & targets) struct State { - std::unordered_set done; + boost::unordered_flat_set done; MissingPaths res; }; @@ -322,7 +322,10 @@ StorePaths Store::topoSortPaths(const StorePathSet & paths) }}, {[&](const StorePath & path, const StorePath & parent) { return BuildError( - "cycle detected in the references of '%s' from '%s'", printStorePath(path), printStorePath(parent)); + BuildResult::Failure::OutputRejected, + "cycle detected in the references of '%s' from '%s'", + printStorePath(path), + printStorePath(parent)); }}); } diff --git a/src/libstore/nar-info-disk-cache.cc b/src/libstore/nar-info-disk-cache.cc index 69d8d2e14d4..11608a667b3 100644 --- a/src/libstore/nar-info-disk-cache.cc +++ b/src/libstore/nar-info-disk-cache.cc @@ -304,10 +304,15 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache if (queryRealisation.isNull(0)) return {oInvalid, 0}; - auto realisation = std::make_shared( - Realisation::fromJSON(nlohmann::json::parse(queryRealisation.getStr(0)), "Local disk cache")); - - return {oValid, realisation}; + try { + return { + oValid, + std::make_shared(nlohmann::json::parse(queryRealisation.getStr(0))), + }; + } catch (Error & e) { + e.addTrace({}, "while parsing the local disk cache"); + throw; + } }); } @@ -349,7 +354,8 @@ class NarInfoDiskCacheImpl : public NarInfoDiskCache auto & cache(getCache(*state, uri)); - state->insertRealisation.use()(cache.id)(realisation.id.to_string())(realisation.toJSON().dump())(time(0)) + state->insertRealisation + .use()(cache.id)(realisation.id.to_string())(static_cast(realisation).dump())(time(0)) .exec(); }); } diff --git a/src/libstore/optimise-store.cc b/src/libstore/optimise-store.cc index 1cf28e022e6..8f28781362e 100644 --- a/src/libstore/optimise-store.cc +++ b/src/libstore/optimise-store.cc @@ -202,7 +202,7 @@ void LocalStore::optimisePath_( full. When that happens, it's fine to ignore it: we just effectively disable deduplication of this file. */ - printInfo("cannot link '%s' to '%s': %s", linkPath, path, strerror(errno)); + printInfo("cannot link %s to '%s': %s", linkPath, path, strerror(errno)); return; } @@ -216,11 +216,11 @@ void LocalStore::optimisePath_( auto stLink = lstat(linkPath.string()); if (st.st_ino == stLink.st_ino) { - debug("'%1%' is already linked to '%2%'", path, linkPath); + debug("'%1%' is already linked to %2%", path, linkPath); return; } - printMsg(lvlTalkative, "linking '%1%' to '%2%'", path, linkPath); + printMsg(lvlTalkative, "linking '%1%' to %2%", path, linkPath); /* Make the containing directory writable, but only if it's not the store itself (we don't want or need to mess with its @@ -245,7 +245,7 @@ void LocalStore::optimisePath_( systems). This is likely to happen with empty files. Just shrug and ignore. */ if (st.st_size) - printInfo("'%1%' has maximum number of links", linkPath); + printInfo("%1% has maximum number of links", linkPath); return; } throw; @@ -256,13 +256,13 @@ void LocalStore::optimisePath_( std::filesystem::rename(tempLink, path); } catch (std::filesystem::filesystem_error & e) { std::filesystem::remove(tempLink); - printError("unable to unlink '%1%'", tempLink); + printError("unable to unlink %1%", tempLink); if (e.code() == std::errc::too_many_links) { /* Some filesystems generate too many links on the rename, rather than on the original link. (Probably it temporarily increases the st_nlink field before decreasing it again.) */ - debug("'%s' has reached maximum number of links", linkPath); + debug("%s has reached maximum number of links", linkPath); return; } throw; diff --git a/src/libstore/outputs-spec.cc b/src/libstore/outputs-spec.cc index 7f73c7d35dd..aacc964cdbb 100644 --- a/src/libstore/outputs-spec.cc +++ b/src/libstore/outputs-spec.cc @@ -150,7 +150,7 @@ OutputsSpec adl_serializer::from_json(const json & json) return OutputsSpec::Names{std::move(names)}; } -void adl_serializer::to_json(json & json, OutputsSpec t) +void adl_serializer::to_json(json & json, const OutputsSpec & t) { std::visit( overloaded{ @@ -169,7 +169,7 @@ ExtendedOutputsSpec adl_serializer::from_json(const json & } } -void adl_serializer::to_json(json & json, ExtendedOutputsSpec t) +void adl_serializer::to_json(json & json, const ExtendedOutputsSpec & t) { std::visit( overloaded{ diff --git a/src/libstore/package.nix b/src/libstore/package.nix index 47805547b8e..ba16c6ab803 100644 --- a/src/libstore/package.nix +++ b/src/libstore/package.nix @@ -4,12 +4,13 @@ mkMesonLibrary, unixtools, - darwin, + apple-sdk, nix-util, boost, curl, aws-sdk-cpp, + aws-crt-cpp, libseccomp, nlohmann_json, sqlite, @@ -25,6 +26,8 @@ withAWS ? # Default is this way because there have been issues building this dependency stdenv.hostPlatform == stdenv.buildPlatform && (stdenv.isLinux || stdenv.isDarwin), + + withCurlS3 ? false, }: let @@ -32,15 +35,17 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-store"; + pname = "determinate-nix-store"; inherit version; workDir = ./.; fileset = fileset.unions [ ../../nix-meson-build-support ./nix-meson-build-support + # FIXME: get rid of these symlinks. ../../.version ./.version + ../../.version-determinate ./meson.build ./meson.options ./include/nix/store/meson.build @@ -64,9 +69,8 @@ mkMesonLibrary (finalAttrs: { sqlite ] ++ lib.optional stdenv.hostPlatform.isLinux libseccomp - # There have been issues building these dependencies - ++ lib.optional stdenv.hostPlatform.isDarwin darwin.apple_sdk.libs.sandbox - ++ lib.optional withAWS aws-sdk-cpp; + ++ lib.optional withAWS aws-sdk-cpp + ++ lib.optional withCurlS3 aws-crt-cpp; propagatedBuildInputs = [ nix-util @@ -76,6 +80,7 @@ mkMesonLibrary (finalAttrs: { mesonFlags = [ (lib.mesonEnable "seccomp-sandboxing" stdenv.hostPlatform.isLinux) (lib.mesonBool "embedded-sandbox-shell" embeddedSandboxShell) + (lib.mesonEnable "curl-s3-store" withCurlS3) ] ++ lib.optionals stdenv.hostPlatform.isLinux [ (lib.mesonOption "sandbox-shell" "${busybox-sandbox-shell}/bin/busybox") diff --git a/src/libstore/path-info.cc b/src/libstore/path-info.cc index e3de5949dbe..270c532bb31 100644 --- a/src/libstore/path-info.cc +++ b/src/libstore/path-info.cc @@ -124,25 +124,29 @@ Strings ValidPathInfo::shortRefs() const return refs; } -ValidPathInfo::ValidPathInfo( +ValidPathInfo ValidPathInfo::makeFromCA( const StoreDirConfig & store, std::string_view name, ContentAddressWithReferences && ca, Hash narHash) - : UnkeyedValidPathInfo(narHash) - , path(store.makeFixedOutputPathFromCA(name, ca)) { - this->ca = ContentAddress{ + ValidPathInfo res{ + store.makeFixedOutputPathFromCA(name, ca), + narHash, + }; + res.ca = ContentAddress{ .method = ca.getMethod(), .hash = ca.getHash(), }; - std::visit( + res.references = std::visit( overloaded{ - [this](TextInfo && ti) { this->references = std::move(ti.references); }, - [this](FixedOutputInfo && foi) { - this->references = std::move(foi.references.others); + [&](TextInfo && ti) { return std::move(ti.references); }, + [&](FixedOutputInfo && foi) { + auto references = std::move(foi.references.others); if (foi.references.self) - this->references.insert(path); + references.insert(res.path); + return references; }, }, std::move(ca).raw); + return res; } nlohmann::json diff --git a/src/libstore/path.cc b/src/libstore/path.cc index 516b01571e9..fa430ce94d7 100644 --- a/src/libstore/path.cc +++ b/src/libstore/path.cc @@ -1,4 +1,7 @@ +#include + #include "nix/store/store-dir-config.hh" +#include "nix/util/json-utils.hh" namespace nix { @@ -75,3 +78,19 @@ StorePath StorePath::random(std::string_view name) } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +StorePath adl_serializer::from_json(const json & json) +{ + return StorePath{getString(json)}; +} + +void adl_serializer::to_json(json & json, const StorePath & storePath) +{ + json = storePath.to_string(); +} + +} // namespace nlohmann diff --git a/src/libstore/posix-fs-canonicalise.cc b/src/libstore/posix-fs-canonicalise.cc index a889938c9fe..a274468c329 100644 --- a/src/libstore/posix-fs-canonicalise.cc +++ b/src/libstore/posix-fs-canonicalise.cc @@ -98,7 +98,7 @@ static void canonicalisePathMetaData_( (i.e. "touch $out/foo; ln $out/foo $out/bar"). */ if (uidRange && (st.st_uid < uidRange->first || st.st_uid > uidRange->second)) { if (S_ISDIR(st.st_mode) || !inodesSeen.count(Inode(st.st_dev, st.st_ino))) - throw BuildError("invalid ownership on file '%1%'", path); + throw BuildError(BuildResult::Failure::OutputRejected, "invalid ownership on file '%1%'", path); mode_t mode = st.st_mode & ~S_IFMT; assert( S_ISLNK(st.st_mode) diff --git a/src/libstore/realisation.cc b/src/libstore/realisation.cc index 8c3baa73b8d..febd67bd2d5 100644 --- a/src/libstore/realisation.cc +++ b/src/libstore/realisation.cc @@ -2,6 +2,7 @@ #include "nix/store/store-api.hh" #include "nix/util/closure.hh" #include "nix/util/signature/local-keys.hh" +#include "nix/util/json-utils.hh" #include namespace nix { @@ -60,54 +61,9 @@ void Realisation::closure(Store & store, const std::set & startOutp }); } -nlohmann::json Realisation::toJSON() const -{ - auto jsonDependentRealisations = nlohmann::json::object(); - for (auto & [depId, depOutPath] : dependentRealisations) - jsonDependentRealisations.emplace(depId.to_string(), depOutPath.to_string()); - return nlohmann::json{ - {"id", id.to_string()}, - {"outPath", outPath.to_string()}, - {"signatures", signatures}, - {"dependentRealisations", jsonDependentRealisations}, - }; -} - -Realisation Realisation::fromJSON(const nlohmann::json & json, const std::string & whence) -{ - auto getOptionalField = [&](std::string fieldName) -> std::optional { - auto fieldIterator = json.find(fieldName); - if (fieldIterator == json.end()) - return std::nullopt; - return {*fieldIterator}; - }; - auto getField = [&](std::string fieldName) -> std::string { - if (auto field = getOptionalField(fieldName)) - return *field; - else - throw Error("Drv output info file '%1%' is corrupt, missing field %2%", whence, fieldName); - }; - - StringSet signatures; - if (auto signaturesIterator = json.find("signatures"); signaturesIterator != json.end()) - signatures.insert(signaturesIterator->begin(), signaturesIterator->end()); - - std::map dependentRealisations; - if (auto jsonDependencies = json.find("dependentRealisations"); jsonDependencies != json.end()) - for (auto & [jsonDepId, jsonDepOutPath] : jsonDependencies->get()) - dependentRealisations.insert({DrvOutput::parse(jsonDepId), StorePath(jsonDepOutPath)}); - - return Realisation{ - .id = DrvOutput::parse(getField("id")), - .outPath = StorePath(getField("outPath")), - .signatures = signatures, - .dependentRealisations = dependentRealisations, - }; -} - std::string Realisation::fingerprint() const { - auto serialized = toJSON(); + nlohmann::json serialized = *this; serialized.erase("signatures"); return serialized.dump(); } @@ -183,3 +139,43 @@ RealisedPath::Set RealisedPath::closure(Store & store) const } } // namespace nix + +namespace nlohmann { + +using namespace nix; + +Realisation adl_serializer::from_json(const json & json0) +{ + auto json = getObject(json0); + + StringSet signatures; + if (auto signaturesOpt = optionalValueAt(json, "signatures")) + signatures = *signaturesOpt; + + std::map dependentRealisations; + if (auto jsonDependencies = optionalValueAt(json, "dependentRealisations")) + for (auto & [jsonDepId, jsonDepOutPath] : getObject(*jsonDependencies)) + dependentRealisations.insert({DrvOutput::parse(jsonDepId), jsonDepOutPath}); + + return Realisation{ + .id = DrvOutput::parse(valueAt(json, "id")), + .outPath = valueAt(json, "outPath"), + .signatures = signatures, + .dependentRealisations = dependentRealisations, + }; +} + +void adl_serializer::to_json(json & json, const Realisation & r) +{ + auto jsonDependentRealisations = nlohmann::json::object(); + for (auto & [depId, depOutPath] : r.dependentRealisations) + jsonDependentRealisations.emplace(depId.to_string(), depOutPath); + json = { + {"id", r.id.to_string()}, + {"outPath", r.outPath}, + {"signatures", r.signatures}, + {"dependentRealisations", jsonDependentRealisations}, + }; +} + +} // namespace nlohmann diff --git a/src/libstore/remote-fs-accessor.cc b/src/libstore/remote-fs-accessor.cc index 12c810eca39..e6715cbdfb0 100644 --- a/src/libstore/remote-fs-accessor.cc +++ b/src/libstore/remote-fs-accessor.cc @@ -51,15 +51,17 @@ ref RemoteFSAccessor::addToCache(std::string_view hashPart, std: std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPath & path) { - auto [storePath, restPath_] = store->toStorePath(store->storeDir + path.abs()); - auto restPath = CanonPath(restPath_); - + auto [storePath, restPath] = store->toStorePath(store->storeDir + path.abs()); if (requireValidPath && !store->isValidPath(storePath)) throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); + return {ref{accessObject(storePath)}, CanonPath{restPath}}; +} +std::shared_ptr RemoteFSAccessor::accessObject(const StorePath & storePath) +{ auto i = nars.find(std::string(storePath.hashPart())); if (i != nars.end()) - return {i->second, restPath}; + return i->second; std::string listing; Path cacheFile; @@ -90,7 +92,7 @@ std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPat }); nars.emplace(storePath.hashPart(), narAccessor); - return {narAccessor, restPath}; + return narAccessor; } catch (SystemError &) { } @@ -98,14 +100,14 @@ std::pair, CanonPath> RemoteFSAccessor::fetch(const CanonPat try { auto narAccessor = makeNarAccessor(nix::readFile(cacheFile)); nars.emplace(storePath.hashPart(), narAccessor); - return {narAccessor, restPath}; + return narAccessor; } catch (SystemError &) { } } StringSink sink; store->narFromPath(storePath, sink); - return {addToCache(storePath.hashPart(), std::move(sink.s)), restPath}; + return addToCache(storePath.hashPart(), std::move(sink.s)); } std::optional RemoteFSAccessor::maybeLstat(const CanonPath & path) diff --git a/src/libstore/remote-store.cc b/src/libstore/remote-store.cc index 5694fa466a1..e78d8e2ff67 100644 --- a/src/libstore/remote-store.cc +++ b/src/libstore/remote-store.cc @@ -73,6 +73,8 @@ void RemoteStore::initConnection(Connection & conn) try { auto [protoVersion, features] = WorkerProto::BasicClientConnection::handshake(conn.to, tee, PROTOCOL_VERSION, WorkerProto::allFeatures); + if (protoVersion < MINIMUM_PROTOCOL_VERSION) + throw Error("the Nix daemon version is too old"); conn.protoVersion = protoVersion; conn.features = features; } catch (SerialisationError & e) { @@ -109,24 +111,22 @@ void RemoteStore::setOptions(Connection & conn) << 0 /* obsolete print build trace */ << settings.buildCores << settings.useSubstitutes; - if (GET_PROTOCOL_MINOR(conn.protoVersion) >= 12) { - std::map overrides; - settings.getSettings(overrides, true); // libstore settings - fileTransferSettings.getSettings(overrides, true); - overrides.erase(settings.keepFailed.name); - overrides.erase(settings.keepGoing.name); - overrides.erase(settings.tryFallback.name); - overrides.erase(settings.maxBuildJobs.name); - overrides.erase(settings.maxSilentTime.name); - overrides.erase(settings.buildCores.name); - overrides.erase(settings.useSubstitutes.name); - overrides.erase(loggerSettings.showTrace.name); - overrides.erase(experimentalFeatureSettings.experimentalFeatures.name); - overrides.erase("plugin-files"); - conn.to << overrides.size(); - for (auto & i : overrides) - conn.to << i.first << i.second.value; - } + std::map overrides; + settings.getSettings(overrides, true); // libstore settings + fileTransferSettings.getSettings(overrides, true); + overrides.erase(settings.keepFailed.name); + overrides.erase(settings.keepGoing.name); + overrides.erase(settings.tryFallback.name); + overrides.erase(settings.maxBuildJobs.name); + overrides.erase(settings.maxSilentTime.name); + overrides.erase(settings.buildCores.name); + overrides.erase(settings.useSubstitutes.name); + overrides.erase(loggerSettings.showTrace.name); + overrides.erase(experimentalFeatureSettings.experimentalFeatures.name); + overrides.erase("plugin-files"); + conn.to << overrides.size(); + for (auto & i : overrides) + conn.to << i.first << i.second.value; auto ex = conn.processStderrReturn(); if (ex) @@ -167,15 +167,7 @@ bool RemoteStore::isValidPathUncached(const StorePath & path) StorePathSet RemoteStore::queryValidPaths(const StorePathSet & paths, SubstituteFlag maybeSubstitute) { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { - StorePathSet res; - for (auto & i : paths) - if (isValidPath(i)) - res.insert(i); - return res; - } else { - return conn->queryValidPaths(*this, &conn.daemonException, paths, maybeSubstitute); - } + return conn->queryValidPaths(*this, &conn.daemonException, paths, maybeSubstitute); } StorePathSet RemoteStore::queryAllValidPaths() @@ -189,21 +181,10 @@ StorePathSet RemoteStore::queryAllValidPaths() StorePathSet RemoteStore::querySubstitutablePaths(const StorePathSet & paths) { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { - StorePathSet res; - for (auto & i : paths) { - conn->to << WorkerProto::Op::HasSubstitutes << printStorePath(i); - conn.processStderr(); - if (readInt(conn->from)) - res.insert(i); - } - return res; - } else { - conn->to << WorkerProto::Op::QuerySubstitutablePaths; - WorkerProto::write(*this, *conn, paths); - conn.processStderr(); - return WorkerProto::Serialise::read(*this, *conn); - } + conn->to << WorkerProto::Op::QuerySubstitutablePaths; + WorkerProto::write(*this, *conn, paths); + conn.processStderr(); + return WorkerProto::Serialise::read(*this, *conn); } void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, SubstitutablePathInfos & infos) @@ -213,45 +194,24 @@ void RemoteStore::querySubstitutablePathInfos(const StorePathCAMap & pathsMap, S auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 12) { - - for (auto & i : pathsMap) { - SubstitutablePathInfo info; - conn->to << WorkerProto::Op::QuerySubstitutablePathInfo << printStorePath(i.first); - conn.processStderr(); - unsigned int reply = readInt(conn->from); - if (reply == 0) - continue; - auto deriver = readString(conn->from); - if (deriver != "") - info.deriver = parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*this, *conn); - info.downloadSize = readLongLong(conn->from); - info.narSize = readLongLong(conn->from); - infos.insert_or_assign(i.first, std::move(info)); - } - - } else { - - conn->to << WorkerProto::Op::QuerySubstitutablePathInfos; - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 22) { - StorePathSet paths; - for (auto & path : pathsMap) - paths.insert(path.first); - WorkerProto::write(*this, *conn, paths); - } else - WorkerProto::write(*this, *conn, pathsMap); - conn.processStderr(); - size_t count = readNum(conn->from); - for (size_t n = 0; n < count; n++) { - SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); - auto deriver = readString(conn->from); - if (deriver != "") - info.deriver = parseStorePath(deriver); - info.references = WorkerProto::Serialise::read(*this, *conn); - info.downloadSize = readLongLong(conn->from); - info.narSize = readLongLong(conn->from); - } + conn->to << WorkerProto::Op::QuerySubstitutablePathInfos; + if (GET_PROTOCOL_MINOR(conn->protoVersion) < 22) { + StorePathSet paths; + for (auto & path : pathsMap) + paths.insert(path.first); + WorkerProto::write(*this, *conn, paths); + } else + WorkerProto::write(*this, *conn, pathsMap); + conn.processStderr(); + size_t count = readNum(conn->from); + for (size_t n = 0; n < count; n++) { + SubstitutablePathInfo & info(infos[parseStorePath(readString(conn->from))]); + auto deriver = readString(conn->from); + if (deriver != "") + info.deriver = parseStorePath(deriver); + info.references = WorkerProto::Serialise::read(*this, *conn); + info.downloadSize = readLongLong(conn->from); + info.narSize = readLongLong(conn->from); } } @@ -466,36 +426,20 @@ void RemoteStore::addToStore(const ValidPathInfo & info, Source & source, Repair { auto conn(getConnection()); - if (GET_PROTOCOL_MINOR(conn->protoVersion) < 18) { - auto source2 = sinkToSource([&](Sink & sink) { - sink << 1 // == path follows - ; - copyNAR(source, sink); - sink << exportMagic << printStorePath(info.path); - WorkerProto::write(*this, *conn, info.references); - sink << (info.deriver ? printStorePath(*info.deriver) : "") << 0 // == no legacy signature - << 0 // == no path follows - ; - }); - conn->importPaths(*this, &conn.daemonException, *source2); - } - - else { - conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path) - << (info.deriver ? printStorePath(*info.deriver) : "") - << info.narHash.to_string(HashFormat::Base16, false); - WorkerProto::write(*this, *conn, info.references); - conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) - << repair << !checkSigs; - - if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 23) { - conn.withFramedSink([&](Sink & sink) { copyNAR(source, sink); }); - } else if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 21) { - conn.processStderr(0, &source); - } else { - copyNAR(source, conn->to); - conn.processStderr(0, nullptr); - } + conn->to << WorkerProto::Op::AddToStoreNar << printStorePath(info.path) + << (info.deriver ? printStorePath(*info.deriver) : "") + << info.narHash.to_string(HashFormat::Base16, false); + WorkerProto::write(*this, *conn, info.references); + conn->to << info.registrationTime << info.narSize << info.ultimate << info.sigs << renderContentAddress(info.ca) + << repair << !checkSigs; + + if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 23) { + conn.withFramedSink([&](Sink & sink) { copyNAR(source, sink); }); + } else if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 21) { + conn.processStderr(0, &source); + } else { + copyNAR(source, conn->to); + conn.processStderr(0, nullptr); } } @@ -618,15 +562,8 @@ void RemoteStore::buildPaths( auto conn(getConnection()); conn->to << WorkerProto::Op::BuildPaths; - assert(GET_PROTOCOL_MINOR(conn->protoVersion) >= 13); WorkerProto::write(*this, *conn, drvPaths); - if (GET_PROTOCOL_MINOR(conn->protoVersion) >= 15) - conn->to << buildMode; - else - /* Old daemons did not take a 'buildMode' parameter, so we - need to validate it here on the client side. */ - if (buildMode != bmNormal) - throw Error("repairing or checking is not supported when building through the Nix daemon"); + conn->to << buildMode; conn.processStderr(); readInt(conn->from); } @@ -661,16 +598,15 @@ std::vector RemoteStore::buildPathsWithResults( [&](const DerivedPath::Opaque & bo) { results.push_back( KeyedBuildResult{ - { - .status = BuildResult::Substituted, - }, + {.inner{BuildResult::Success{ + .status = BuildResult::Success::Substituted, + }}}, /* .path = */ bo, }); }, [&](const DerivedPath::Built & bfd) { - KeyedBuildResult res{ - {.status = BuildResult::Built}, - /* .path = */ bfd, + BuildResult::Success success{ + .status = BuildResult::Success::Built, }; OutputPathMap outputs; @@ -690,9 +626,9 @@ std::vector RemoteStore::buildPathsWithResults( auto realisation = queryRealisation(outputId); if (!realisation) throw MissingRealisation(outputId); - res.builtOutputs.emplace(output, *realisation); + success.builtOutputs.emplace(output, *realisation); } else { - res.builtOutputs.emplace( + success.builtOutputs.emplace( output, Realisation{ .id = outputId, @@ -701,7 +637,11 @@ std::vector RemoteStore::buildPathsWithResults( } } - results.push_back(res); + results.push_back( + KeyedBuildResult{ + {.inner = std::move(success)}, + /* .path = */ bfd, + }); }}, path.raw()); } @@ -764,10 +704,7 @@ void RemoteStore::collectGarbage(const GCOptions & options, GCResults & results) results.bytesFreed = readLongLong(conn->from); readLongLong(conn->from); // obsolete - { - auto state_(Store::state.lock()); - state_->pathInfoCache.clear(); - } + pathInfoCache->lock()->clear(); } void RemoteStore::optimiseStore() @@ -860,9 +797,19 @@ void RemoteStore::narFromPath(const StorePath & path, Sink & sink) conn->narFromPath(*this, &conn.daemonException, path, [&](Source & source) { copyNAR(conn->from, sink); }); } +ref RemoteStore::getRemoteFSAccessor(bool requireValidPath) +{ + return make_ref(ref(shared_from_this()), requireValidPath); +} + ref RemoteStore::getFSAccessor(bool requireValidPath) { - return make_ref(ref(shared_from_this())); + return getRemoteFSAccessor(requireValidPath); +} + +std::shared_ptr RemoteStore::getFSAccessor(const StorePath & path, bool requireValidPath) +{ + return getRemoteFSAccessor(requireValidPath)->accessObject(path); } void RemoteStore::ConnectionHandle::withFramedSink(std::function fun) diff --git a/src/libstore/restricted-store.cc b/src/libstore/restricted-store.cc index e0f43ab6c47..a1cb4160638 100644 --- a/src/libstore/restricted-store.cc +++ b/src/libstore/restricted-store.cc @@ -257,8 +257,8 @@ void RestrictedStore::buildPaths( const std::vector & paths, BuildMode buildMode, std::shared_ptr evalStore) { for (auto & result : buildPathsWithResults(paths, buildMode, evalStore)) - if (!result.success()) - result.rethrow(); + if (auto * failureP = result.tryGetFailure()) + failureP->rethrow(); } std::vector RestrictedStore::buildPathsWithResults( @@ -280,9 +280,11 @@ std::vector RestrictedStore::buildPathsWithResults( auto results = next->buildPathsWithResults(paths, buildMode); for (auto & result : results) { - for (auto & [outputName, output] : result.builtOutputs) { - newPaths.insert(output.outPath); - newRealisations.insert(output); + if (auto * successP = result.tryGetSuccess()) { + for (auto & [outputName, output] : successP->builtOutputs) { + newPaths.insert(output.outPath); + newRealisations.insert(output); + } } } diff --git a/src/libstore/s3-binary-cache-store.cc b/src/libstore/s3-binary-cache-store.cc index 4ad09aff22c..b70f04be782 100644 --- a/src/libstore/s3-binary-cache-store.cc +++ b/src/libstore/s3-binary-cache-store.cc @@ -262,6 +262,7 @@ StoreReference S3BinaryCacheStoreConfig::getReference() const .scheme = *uriSchemes().begin(), .authority = bucketName, }, + .params = getQueryParams(), }; } diff --git a/src/libstore/s3-url.cc b/src/libstore/s3-url.cc new file mode 100644 index 00000000000..baefe5cba5e --- /dev/null +++ b/src/libstore/s3-url.cc @@ -0,0 +1,121 @@ +#include "nix/store/s3-url.hh" + +#if NIX_WITH_S3_SUPPORT || NIX_WITH_CURL_S3 + +# include "nix/util/error.hh" +# include "nix/util/split.hh" +# include "nix/util/strings-inline.hh" + +# include +# include + +using namespace std::string_view_literals; + +namespace nix { + +ParsedS3URL ParsedS3URL::parse(const ParsedURL & parsed) +try { + if (parsed.scheme != "s3"sv) + throw BadURL("URI scheme '%s' is not 's3'", parsed.scheme); + + /* Yeah, S3 URLs in Nix have the bucket name as authority. Luckily registered name type + authority has the same restrictions (mostly) as S3 bucket names. + TODO: Validate against: + https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html#general-purpose-bucket-names + */ + if (!parsed.authority || parsed.authority->host.empty() + || parsed.authority->hostType != ParsedURL::Authority::HostType::Name) + throw BadURL("URI has a missing or invalid bucket name"); + + /* TODO: Validate the key against: + * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html#object-key-guidelines + */ + + auto getOptionalParam = [&](std::string_view key) -> std::optional { + const auto & query = parsed.query; + auto it = query.find(key); + if (it == query.end()) + return std::nullopt; + return it->second; + }; + + auto endpoint = getOptionalParam("endpoint"); + if (parsed.path.size() <= 1 || !parsed.path.front().empty()) + throw BadURL("URI has a missing or invalid key"); + + auto path = std::views::drop(parsed.path, 1) | std::ranges::to>(); + + return ParsedS3URL{ + .bucket = parsed.authority->host, + .key = std::move(path), + .profile = getOptionalParam("profile"), + .region = getOptionalParam("region"), + .scheme = getOptionalParam("scheme"), + .endpoint = [&]() -> decltype(ParsedS3URL::endpoint) { + if (!endpoint) + return std::monostate(); + + /* Try to parse the endpoint as a full-fledged URL with a scheme. */ + try { + return parseURL(*endpoint); + } catch (BadURL &) { + } + + return ParsedURL::Authority::parse(*endpoint); + }(), + }; +} catch (BadURL & e) { + e.addTrace({}, "while parsing S3 URI: '%s'", parsed.to_string()); + throw; +} + +ParsedURL ParsedS3URL::toHttpsUrl() const +{ + auto toView = [](const auto & x) { return std::string_view{x}; }; + + auto regionStr = region.transform(toView).value_or("us-east-1"); + auto schemeStr = scheme.transform(toView).value_or("https"); + + // Handle endpoint configuration using std::visit + return std::visit( + overloaded{ + [&](const std::monostate &) { + // No custom endpoint, use standard AWS S3 endpoint + std::vector path{""}; + path.push_back(bucket); + path.insert(path.end(), key.begin(), key.end()); + return ParsedURL{ + .scheme = std::string{schemeStr}, + .authority = ParsedURL::Authority{.host = "s3." + regionStr + ".amazonaws.com"}, + .path = std::move(path), + }; + }, + [&](const ParsedURL::Authority & auth) { + // Endpoint is just an authority (hostname/port) + std::vector path{""}; + path.push_back(bucket); + path.insert(path.end(), key.begin(), key.end()); + return ParsedURL{ + .scheme = std::string{schemeStr}, + .authority = auth, + .path = std::move(path), + }; + }, + [&](const ParsedURL & endpointUrl) { + // Endpoint is already a ParsedURL (e.g., http://server:9000) + auto path = endpointUrl.path; + path.push_back(bucket); + path.insert(path.end(), key.begin(), key.end()); + return ParsedURL{ + .scheme = endpointUrl.scheme, + .authority = endpointUrl.authority, + .path = std::move(path), + }; + }, + }, + endpoint); +} + +} // namespace nix + +#endif diff --git a/src/libstore/s3.cc b/src/libstore/s3.cc deleted file mode 100644 index 9ed4e7fd9ce..00000000000 --- a/src/libstore/s3.cc +++ /dev/null @@ -1,71 +0,0 @@ -#include "nix/store/s3.hh" -#include "nix/util/split.hh" -#include "nix/util/url.hh" - -namespace nix { - -using namespace std::string_view_literals; - -#if NIX_WITH_S3_SUPPORT - -ParsedS3URL ParsedS3URL::parse(std::string_view uri) -try { - auto parsed = parseURL(uri); - - if (parsed.scheme != "s3"sv) - throw BadURL("URI scheme '%s' is not 's3'", parsed.scheme); - - /* Yeah, S3 URLs in Nix have the bucket name as authority. Luckily registered name type - authority has the same restrictions (mostly) as S3 bucket names. - TODO: Validate against: - https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html#general-purpose-bucket-names - */ - if (!parsed.authority || parsed.authority->host.empty() - || parsed.authority->hostType != ParsedURL::Authority::HostType::Name) - throw BadURL("URI has a missing or invalid bucket name"); - - std::string_view key = parsed.path; - /* Make the key a relative path. */ - splitPrefix(key, "/"); - - /* TODO: Validate the key against: - * https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html#object-key-guidelines - */ - - auto getOptionalParam = [&](std::string_view key) -> std::optional { - const auto & query = parsed.query; - auto it = query.find(key); - if (it == query.end()) - return std::nullopt; - return it->second; - }; - - auto endpoint = getOptionalParam("endpoint"); - - return ParsedS3URL{ - .bucket = std::move(parsed.authority->host), - .key = std::string{key}, - .profile = getOptionalParam("profile"), - .region = getOptionalParam("region"), - .scheme = getOptionalParam("scheme"), - .endpoint = [&]() -> decltype(ParsedS3URL::endpoint) { - if (!endpoint) - return std::monostate(); - - /* Try to parse the endpoint as a full-fledged URL with a scheme. */ - try { - return parseURL(*endpoint); - } catch (BadURL &) { - } - - return ParsedURL::Authority::parse(*endpoint); - }(), - }; -} catch (BadURL & e) { - e.addTrace({}, "while parsing S3 URI: '%s'", uri); - throw; -} - -#endif - -} // namespace nix diff --git a/src/libstore/serve-protocol-connection.cc b/src/libstore/serve-protocol-connection.cc index 908994f4e9a..a90b104a68e 100644 --- a/src/libstore/serve-protocol-connection.cc +++ b/src/libstore/serve-protocol-connection.cc @@ -15,7 +15,7 @@ ServeProto::Version ServeProto::BasicClientConnection::handshake( if (magic != SERVE_MAGIC_2) throw Error("'nix-store --serve' protocol mismatch from '%s'", host); auto remoteVersion = readInt(from); - if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200) + if (GET_PROTOCOL_MAJOR(remoteVersion) != 0x200 || GET_PROTOCOL_MINOR(remoteVersion) < 5) throw Error("unsupported 'nix-store --serve' protocol version on '%s'", host); return std::min(remoteVersion, localVersion); } @@ -93,14 +93,4 @@ void ServeProto::BasicClientConnection::narFromPath( fun(from); } -void ServeProto::BasicClientConnection::importPaths(const StoreDirConfig & store, std::function fun) -{ - to << ServeProto::Command::ImportPaths; - fun(to); - to.flush(); - - if (readInt(from) != 1) - throw Error("remote machine failed to import closure"); -} - } // namespace nix diff --git a/src/libstore/serve-protocol.cc b/src/libstore/serve-protocol.cc index 7cf5e699716..51b575fcd5a 100644 --- a/src/libstore/serve-protocol.cc +++ b/src/libstore/serve-protocol.cc @@ -16,32 +16,62 @@ namespace nix { BuildResult ServeProto::Serialise::read(const StoreDirConfig & store, ServeProto::ReadConn conn) { BuildResult status; - status.status = (BuildResult::Status) readInt(conn.from); - conn.from >> status.errorMsg; + BuildResult::Success success; + BuildResult::Failure failure; + + auto rawStatus = readInt(conn.from); + conn.from >> failure.errorMsg; if (GET_PROTOCOL_MINOR(conn.version) >= 3) - conn.from >> status.timesBuilt >> status.isNonDeterministic >> status.startTime >> status.stopTime; + conn.from >> status.timesBuilt >> failure.isNonDeterministic >> status.startTime >> status.stopTime; if (GET_PROTOCOL_MINOR(conn.version) >= 6) { auto builtOutputs = ServeProto::Serialise::read(store, conn); for (auto && [output, realisation] : builtOutputs) - status.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + } + + if (BuildResult::Success::statusIs(rawStatus)) { + success.status = static_cast(rawStatus); + status.inner = std::move(success); + } else { + failure.status = static_cast(rawStatus); + status.inner = std::move(failure); } + return status; } void ServeProto::Serialise::write( - const StoreDirConfig & store, ServeProto::WriteConn conn, const BuildResult & status) + const StoreDirConfig & store, ServeProto::WriteConn conn, const BuildResult & res) { - conn.to << status.status << status.errorMsg; - - if (GET_PROTOCOL_MINOR(conn.version) >= 3) - conn.to << status.timesBuilt << status.isNonDeterministic << status.startTime << status.stopTime; - if (GET_PROTOCOL_MINOR(conn.version) >= 6) { - DrvOutputs builtOutputs; - for (auto & [output, realisation] : status.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - ServeProto::write(store, conn, builtOutputs); - } + /* The protocol predates the use of sum types (std::variant) to + separate the success or failure cases. As such, it transits some + success- or failure-only fields in both cases. This helper + function helps support this: in each case, we just pass the old + default value for the fields that don't exist in that case. */ + auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { + conn.to << errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 3) + conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; + if (GET_PROTOCOL_MINOR(conn.version) >= 6) { + DrvOutputs builtOutputsFullKey; + for (auto & [output, realisation] : builtOutputs) + builtOutputsFullKey.insert_or_assign(realisation.id, realisation); + ServeProto::write(store, conn, builtOutputsFullKey); + } + }; + std::visit( + overloaded{ + [&](const BuildResult::Failure & failure) { + conn.to << failure.status; + common(failure.errorMsg, failure.isNonDeterministic, decltype(BuildResult::Success::builtOutputs){}); + }, + [&](const BuildResult::Success & success) { + conn.to << success.status; + common(/*errorMsg=*/"", /*isNonDeterministic=*/false, success.builtOutputs); + }, + }, + res.inner); } UnkeyedValidPathInfo ServeProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) diff --git a/src/libstore/ssh-store.cc b/src/libstore/ssh-store.cc index dafe14fea76..a7e28017fad 100644 --- a/src/libstore/ssh-store.cc +++ b/src/libstore/ssh-store.cc @@ -151,6 +151,11 @@ struct MountedSSHStore : virtual SSHStore, virtual LocalFSStore return LocalFSStore::getFSAccessor(requireValidPath); } + std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath) override + { + return LocalFSStore::getFSAccessor(path, requireValidPath); + } + std::optional getBuildLogExact(const StorePath & path) override { return LocalFSStore::getBuildLogExact(path); diff --git a/src/libstore/ssh.cc b/src/libstore/ssh.cc index 8a4614a0d60..1a99083669c 100644 --- a/src/libstore/ssh.cc +++ b/src/libstore/ssh.cc @@ -78,29 +78,26 @@ SSHMaster::SSHMaster( oss << authority.host; return std::move(oss).str(); }()) - , fakeSSH(authority.host == "localhost") + , fakeSSH(authority.to_string() == "localhost") , keyFile(keyFile) , sshPublicHostKey(parsePublicHostKey(authority.host, sshPublicHostKey)) , useMaster(useMaster && !fakeSSH) , compress(compress) , logFD(logFD) + , tmpDir(make_ref(createTempDir("", "nix", 0700))) { checkValidAuthority(authority); - auto state(state_.lock()); - state->tmpDir = std::make_unique(createTempDir("", "nix", 0700)); } void SSHMaster::addCommonSSHOpts(Strings & args) { - auto state(state_.lock()); - auto sshArgs = getNixSshOpts(); args.insert(args.end(), sshArgs.begin(), sshArgs.end()); if (!keyFile.empty()) args.insert(args.end(), {"-i", keyFile}); if (!sshPublicHostKey.empty()) { - std::filesystem::path fileName = state->tmpDir->path() / "host-key"; + std::filesystem::path fileName = tmpDir->path() / "host-key"; writeFile(fileName.string(), authority.host + " " + sshPublicHostKey + "\n"); args.insert(args.end(), {"-oUserKnownHostsFile=" + fileName.string()}); } @@ -241,7 +238,7 @@ Path SSHMaster::startMaster() if (state->sshMaster != INVALID_DESCRIPTOR) return state->socketPath; - state->socketPath = (Path) *state->tmpDir + "/ssh.sock"; + state->socketPath = (Path) *tmpDir + "/ssh.sock"; Pipe out; out.create(); diff --git a/src/libstore/store-api.cc b/src/libstore/store-api.cc index fad79a83e0d..6d011d324d2 100644 --- a/src/libstore/store-api.cc +++ b/src/libstore/store-api.cc @@ -1,3 +1,4 @@ +#include "nix/util/logging.hh" #include "nix/util/signature/local-keys.hh" #include "nix/util/source-accessor.hh" #include "nix/store/globals.hh" @@ -57,12 +58,22 @@ std::pair StoreDirConfig::toStorePath(PathView path) const Path Store::followLinksToStore(std::string_view _path) const { Path path = absPath(std::string(_path)); + + // Limit symlink follows to prevent infinite loops + unsigned int followCount = 0; + const unsigned int maxFollow = 1024; + while (!isInStore(path)) { if (!std::filesystem::is_symlink(path)) break; + + if (++followCount >= maxFollow) + throw Error("too many symbolic links encountered while resolving '%s'", _path); + auto target = readLink(path); path = absPath(target, dirOf(path)); } + if (!isInStore(path)) throw BadStorePath("path '%1%' is not in the Nix store", path); return path; @@ -99,8 +110,12 @@ StorePath Store::addToStore( auto sink = sourceToSink([&](Source & source) { LengthSource lengthSource(source); storePath = addToStoreFromDump(lengthSource, name, fsm, method, hashAlgo, references, repair); - if (settings.warnLargePathThreshold && lengthSource.total >= settings.warnLargePathThreshold) - warn("copied large path '%s' to the store (%s)", path, renderSize(lengthSource.total)); + if (settings.warnLargePathThreshold && lengthSource.total >= settings.warnLargePathThreshold) { + static bool failOnLargePath = getEnv("_NIX_TEST_FAIL_ON_LARGE_PATH").value_or("") == "1"; + if (failOnLargePath) + throw Error("doesn't copy large path '%s' to the store (%d)", path, renderSize(lengthSource.total)); + warn("copied large path '%s' to the store (%d)", path, renderSize(lengthSource.total)); + } }); dumpPath(path, *sink, fsm, filter); sink->finish(); @@ -268,7 +283,7 @@ ValidPathInfo Store::addToStoreSlow( if (expectedCAHash && expectedCAHash != hash) throw Error("hash mismatch for '%s'", srcPath); - ValidPathInfo info{ + auto info = ValidPathInfo::makeFromCA( *this, name, ContentAddressWithReferences::fromParts( @@ -278,8 +293,7 @@ ValidPathInfo Store::addToStoreSlow( .others = references, .self = false, }), - narHash, - }; + narHash); info.narSize = narSize; if (!isValidPath(info.path)) { @@ -306,7 +320,7 @@ StringSet Store::Config::getDefaultSystemFeatures() Store::Store(const Store::Config & config) : StoreDirConfig{config} , config{config} - , state({(size_t) config.pathInfoCacheSize}) + , pathInfoCache(make_ref((size_t) config.pathInfoCacheSize)) { assertLibStoreInitialized(); } @@ -326,7 +340,7 @@ bool Store::PathInfoCacheValue::isKnownNow() void Store::invalidatePathInfoCacheFor(const StorePath & path) { - state.lock()->pathInfoCache.erase(path.to_string()); + pathInfoCache->lock()->erase(path.to_string()); } std::map> Store::queryStaticPartialDerivationOutputMap(const StorePath & path) @@ -392,11 +406,14 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta { if (!settings.useSubstitutes) return; - for (auto & sub : getDefaultSubstituters()) { - for (auto & path : paths) { - if (infos.count(path.first)) - // Choose first succeeding substituter. - continue; + + for (auto & path : paths) { + std::optional lastStoresException = std::nullopt; + for (auto & sub : getDefaultSubstituters()) { + if (lastStoresException.has_value()) { + logError(lastStoresException->info()); + lastStoresException.reset(); + } auto subPath(path.first); @@ -437,24 +454,24 @@ void Store::querySubstitutablePathInfos(const StorePathCAMap & paths, Substituta } catch (InvalidPath &) { } catch (SubstituterDisabled &) { } catch (Error & e) { - if (settings.tryFallback) - logError(e.info()); - else - throw; + lastStoresException = std::make_optional(std::move(e)); } } + if (lastStoresException.has_value()) { + if (!settings.tryFallback) { + throw *lastStoresException; + } else + logError(lastStoresException->info()); + } } } bool Store::isValidPath(const StorePath & storePath) { - { - auto state_(state.lock()); - auto res = state_->pathInfoCache.get(storePath.to_string()); - if (res && res->isKnownNow()) { - stats.narInfoReadAverted++; - return res->didExist(); - } + auto res = pathInfoCache->lock()->get(storePath.to_string()); + if (res && res->isKnownNow()) { + stats.narInfoReadAverted++; + return res->didExist(); } if (diskCache) { @@ -462,8 +479,7 @@ bool Store::isValidPath(const StorePath & storePath) config.getReference().render(/*FIXME withParams=*/false), std::string(storePath.hashPart())); if (res.first != NarInfoDiskCache::oUnknown) { stats.narInfoReadAverted++; - auto state_(state.lock()); - state_->pathInfoCache.upsert( + pathInfoCache->lock()->upsert( storePath.to_string(), res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} : PathInfoCacheValue{.value = res.second}); @@ -508,6 +524,23 @@ ref Store::queryPathInfo(const StorePath & storePath) return promise.get_future().get(); } +std::shared_ptr Store::maybeQueryPathInfo(const StorePath & storePath) +{ + std::promise> promise; + + queryPathInfo(storePath, {[&](std::future> result) { + try { + promise.set_value(result.get()); + } catch (InvalidPath &) { + promise.set_value(nullptr); + } catch (...) { + promise.set_exception(std::current_exception()); + } + }}); + + return promise.get_future().get(); +} + static bool goodStorePath(const StorePath & expected, const StorePath & actual) { return expected.hashPart() == actual.hashPart() @@ -518,30 +551,25 @@ std::optional> Store::queryPathInfoFromClie { auto hashPart = std::string(storePath.hashPart()); - { - auto res = state.lock()->pathInfoCache.get(storePath.to_string()); - if (res && res->isKnownNow()) { - stats.narInfoReadAverted++; - if (res->didExist()) - return std::make_optional(res->value); - else - return std::make_optional(nullptr); - } + auto res = pathInfoCache->lock()->get(storePath.to_string()); + if (res && res->isKnownNow()) { + stats.narInfoReadAverted++; + if (res->didExist()) + return std::make_optional(res->value); + else + return std::make_optional(nullptr); } if (diskCache) { auto res = diskCache->lookupNarInfo(config.getReference().render(/*FIXME withParams=*/false), hashPart); if (res.first != NarInfoDiskCache::oUnknown) { stats.narInfoReadAverted++; - { - auto state_(state.lock()); - state_->pathInfoCache.upsert( - storePath.to_string(), - res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} - : PathInfoCacheValue{.value = res.second}); - if (res.first == NarInfoDiskCache::oInvalid || !goodStorePath(storePath, res.second->path)) - return std::make_optional(nullptr); - } + pathInfoCache->lock()->upsert( + storePath.to_string(), + res.first == NarInfoDiskCache::oInvalid ? PathInfoCacheValue{} + : PathInfoCacheValue{.value = res.second}); + if (res.first == NarInfoDiskCache::oInvalid || !goodStorePath(storePath, res.second->path)) + return std::make_optional(nullptr); assert(res.second); return std::make_optional(res.second); } @@ -577,10 +605,7 @@ void Store::queryPathInfo(const StorePath & storePath, CallbackupsertNarInfo(config.getReference().render(/*FIXME withParams=*/false), hashPart, info); - { - auto state_(state.lock()); - state_->pathInfoCache.upsert(storePath.to_string(), PathInfoCacheValue{.value = info}); - } + pathInfoCache->lock()->upsert(storePath.to_string(), PathInfoCacheValue{.value = info}); if (!info || !goodStorePath(storePath, info->path)) { stats.narInfoMissing++; @@ -770,6 +795,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor for (auto & storePath : storePaths) { if (!inputPaths.count(storePath)) throw BuildError( + BuildResult::Failure::InputRejected, "cannot export references of path '%s' because it is not in the input closure of the derivation", printStorePath(storePath)); @@ -802,10 +828,7 @@ StorePathSet Store::exportReferences(const StorePathSet & storePaths, const Stor const Store::Stats & Store::getStats() { - { - auto state_(state.readLock()); - stats.pathInfoCacheSize = state_->pathInfoCache.size(); - } + stats.pathInfoCacheSize = pathInfoCache->readLock()->size(); return stats; } @@ -817,7 +840,13 @@ makeCopyPathMessage(const StoreConfig & srcCfg, const StoreConfig & dstCfg, std: auto isShorthand = [](const StoreReference & ref) { /* At this point StoreReference **must** be resolved. */ - const auto & specified = std::get(ref.variant); + const auto & specified = std::visit( + overloaded{ + [](const StoreReference::Auto &) -> const StoreReference::Specified & { unreachable(); }, + [](const StoreReference::Specified & specified) -> const StoreReference::Specified & { + return specified; + }}, + ref.variant); const auto & scheme = specified.scheme; return (scheme == "local" || scheme == "unix") && specified.authority.empty(); }; @@ -1122,10 +1151,9 @@ Derivation Store::derivationFromPath(const StorePath & drvPath) static Derivation readDerivationCommon(Store & store, const StorePath & drvPath, bool requireValidPath) { - auto accessor = store.getFSAccessor(requireValidPath); + auto accessor = store.getFSAccessor(drvPath, requireValidPath); try { - return parseDerivation( - store, accessor->readFile(CanonPath(drvPath.to_string())), Derivation::nameFromPath(drvPath)); + return parseDerivation(store, accessor->readFile(CanonPath::root), Derivation::nameFromPath(drvPath)); } catch (FormatError & e) { throw Error("error parsing derivation '%s': %s", store.printStorePath(drvPath), e.msg()); } diff --git a/src/libstore/store-reference.cc b/src/libstore/store-reference.cc index adc60b39135..96ee829d037 100644 --- a/src/libstore/store-reference.cc +++ b/src/libstore/store-reference.cc @@ -1,11 +1,12 @@ -#include - #include "nix/util/error.hh" +#include "nix/util/split.hh" #include "nix/util/url.hh" #include "nix/store/store-reference.hh" #include "nix/util/file-system.hh" #include "nix/util/util.hh" +#include + namespace nix { static bool isNonUriPath(const std::string & spec) @@ -25,6 +26,8 @@ std::string StoreReference::render(bool withParams) const std::visit( overloaded{ [&](const StoreReference::Auto &) { res = "auto"; }, + [&](const StoreReference::Daemon &) { res = "daemon"; }, + [&](const StoreReference::Local &) { res = "local"; }, [&](const StoreReference::Specified & g) { res = g.scheme; res += "://"; @@ -41,6 +44,29 @@ std::string StoreReference::render(bool withParams) const return res; } +namespace { + +struct SchemeAndAuthorityWithPath +{ + std::string_view scheme; + std::string_view authority; +}; + +} // namespace + +/** + * Return the 'scheme' and remove the '://' or ':' separator. + */ +static std::optional splitSchemePrefixTo(std::string_view string) +{ + auto scheme = splitPrefixTo(string, ':'); + if (!scheme) + return std::nullopt; + + splitPrefix(string, "//"); + return SchemeAndAuthorityWithPath{.scheme = *scheme, .authority = string}; +} + StoreReference StoreReference::parse(const std::string & uri, const StoreReference::Params & extraParams) { auto params = extraParams; @@ -48,13 +74,11 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen auto parsedUri = parseURL(uri, /*lenient=*/true); params.insert(parsedUri.query.begin(), parsedUri.query.end()); - auto baseURI = parsedUri.authority.value_or(ParsedURL::Authority{}).to_string() + parsedUri.path; - return { .variant = Specified{ .scheme = std::move(parsedUri.scheme), - .authority = std::move(baseURI), + .authority = parsedUri.renderAuthorityAndPath(), }, .params = std::move(params), }; @@ -68,21 +92,17 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen .params = std::move(params), }; } else if (baseURI == "daemon") { + if (params.empty()) + return {.variant = Daemon{}}; return { - .variant = - Specified{ - .scheme = "unix", - .authority = "", - }, + .variant = Specified{.scheme = "unix", .authority = ""}, .params = std::move(params), }; } else if (baseURI == "local") { + if (params.empty()) + return {.variant = Local{}}; return { - .variant = - Specified{ - .scheme = "local", - .authority = "", - }, + .variant = Specified{.scheme = "local", .authority = ""}, .params = std::move(params), }; } else if (isNonUriPath(baseURI)) { @@ -94,6 +114,32 @@ StoreReference StoreReference::parse(const std::string & uri, const StoreReferen }, .params = std::move(params), }; + } else if (auto schemeAndAuthority = splitSchemePrefixTo(baseURI)) { + /* Back-compatibility shim to accept unbracketed IPv6 addresses after the scheme. + * Old versions of nix allowed that. Note that this is ambiguous and does not allow + * specifying the port number. For that the address must be bracketed, otherwise it's + * greedily assumed to be the part of the host address. */ + auto authorityString = schemeAndAuthority->authority; + auto userinfo = splitPrefixTo(authorityString, '@'); + auto maybeIpv6 = boost::urls::parse_ipv6_address(authorityString); + if (maybeIpv6) { + std::string fixedAuthority; + if (userinfo) { + fixedAuthority += *userinfo; + fixedAuthority += '@'; + } + fixedAuthority += '['; + fixedAuthority += authorityString; + fixedAuthority += ']'; + return { + .variant = + Specified{ + .scheme = std::string(schemeAndAuthority->scheme), + .authority = fixedAuthority, + }, + .params = std::move(params), + }; + } } } diff --git a/src/libstore/uds-remote-store.cc b/src/libstore/uds-remote-store.cc index 4871b491399..6106a99ce38 100644 --- a/src/libstore/uds-remote-store.cc +++ b/src/libstore/uds-remote-store.cc @@ -57,16 +57,21 @@ UDSRemoteStore::UDSRemoteStore(ref config) StoreReference UDSRemoteStoreConfig::getReference() const { + /* We specifically return "daemon" here instead of "unix://" or "unix://${path}" + * to be more compatible with older versions of nix. Some tooling out there + * tries hard to parse store references and it might not be able to handle "unix://". */ + if (path == settings.nixDaemonSocketFile) + return { + .variant = StoreReference::Daemon{}, + .params = getQueryParams(), + }; return { .variant = StoreReference::Specified{ .scheme = *uriSchemes().begin(), - // We return the empty string when the path looks like the - // default path, but we could also just return the path - // verbatim always, to be robust to overall config changes - // at the cost of some verbosity. - .authority = path == settings.nixDaemonSocketFile ? "" : path, + .authority = path, }, + .params = getQueryParams(), }; } diff --git a/src/libstore/unix/build/chroot-derivation-builder.cc b/src/libstore/unix/build/chroot-derivation-builder.cc index 887bb47f081..8c93595334c 100644 --- a/src/libstore/unix/build/chroot-derivation-builder.cc +++ b/src/libstore/unix/build/chroot-derivation-builder.cc @@ -22,13 +22,6 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl PathsInChroot pathsInChroot; - void deleteTmpDir(bool force) override - { - autoDelChroot.reset(); /* this runs the destructor */ - - DerivationBuilderImpl::deleteTmpDir(force); - } - bool needsHashRewrite() override { return false; @@ -166,13 +159,13 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl return !needsHashRewrite() ? chrootRootDir + p : store.toRealPath(p); } - void cleanupBuild() override + void cleanupBuild(bool force) override { - DerivationBuilderImpl::cleanupBuild(); + DerivationBuilderImpl::cleanupBuild(force); /* Move paths out of the chroot for easier debugging of build failures. */ - if (buildMode == bmNormal) + if (!force && buildMode == bmNormal) for (auto & [_, status] : initialOutputs) { if (!status.known) continue; @@ -182,6 +175,8 @@ struct ChrootDerivationBuilder : virtual DerivationBuilderImpl if (pathExists(chrootRootDir + p)) std::filesystem::rename((chrootRootDir + p), p); } + + autoDelChroot.reset(); /* this runs the destructor */ } std::pair addDependencyPrep(const StorePath & path) diff --git a/src/libstore/unix/build/derivation-builder.cc b/src/libstore/unix/build/derivation-builder.cc index 15c99e3c002..75766437a91 100644 --- a/src/libstore/unix/build/derivation-builder.cc +++ b/src/libstore/unix/build/derivation-builder.cc @@ -17,6 +17,8 @@ #include "nix/store/restricted-store.hh" #include "nix/store/user-lock.hh" #include "nix/store/globals.hh" +#include "nix/store/build/derivation-env-desugar.hh" +#include "nix/util/terminal.hh" #include @@ -42,10 +44,17 @@ #include "nix/util/signals.hh" #include "store-config-private.hh" +#include "build/derivation-check.hh" namespace nix { -MakeError(NotDeterministic, BuildError); +struct NotDeterministic : BuildError +{ + NotDeterministic(auto &&... args) + : BuildError(BuildResult::Failure::NotDeterministic, args...) + { + } +}; /** * This class represents the state for building locally. @@ -63,6 +72,11 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder { protected: + /** + * The process ID of the builder. + */ + Pid pid; + LocalStore & store; std::unique_ptr miscMethods; @@ -78,6 +92,27 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder { } + ~DerivationBuilderImpl() + { + /* Careful: we should never ever throw an exception from a + destructor. */ + try { + killChild(); + } catch (...) { + ignoreExceptionInDestructor(); + } + try { + stopDaemon(); + } catch (...) { + ignoreExceptionInDestructor(); + } + try { + cleanupBuild(false); + } catch (...) { + ignoreExceptionInDestructor(); + } + } + protected: /** @@ -180,11 +215,9 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder public: - bool prepareBuild() override; + std::optional startBuild() override; - void startBuilder() override; - - std::variant, SingleDrvOutputs> unprepareBuild() override; + SingleDrvOutputs unprepareBuild() override; protected: @@ -196,6 +229,12 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder return acquireUserLock(1, false); } + /** + * Throw an exception if we can't do this derivation because of + * missing system features. + */ + virtual void checkSystem(); + /** * Return the paths that should be made available in the sandbox. * This includes: @@ -278,9 +317,11 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder */ void startDaemon(); -public: - - void stopDaemon() override; + /** + * Stop the in-process nix daemon thread. + * @see startDaemon + */ + void stopDaemon(); protected: @@ -335,22 +376,25 @@ class DerivationBuilderImpl : public DerivationBuilder, public DerivationBuilder */ SingleDrvOutputs registerOutputs(); +protected: + /** - * Check that an output meets the requirements specified by the - * 'outputChecks' attribute (or the legacy - * '{allowed,disallowed}{References,Requisites}' attributes). + * Delete the temporary directory, if we have one. + * + * @param force We know the build suceeded, so don't attempt to + * preseve anything for debugging. */ - void checkOutputs(const std::map & outputs); - -public: - - void deleteTmpDir(bool force) override; + virtual void cleanupBuild(bool force); - void killSandbox(bool getStats) override; + /** + * Kill any processes running under the build user UID or in the + * cgroup of the build. + */ + virtual void killSandbox(bool getStats); -protected: +public: - virtual void cleanupBuild(); + bool killChild() override; private: @@ -413,29 +457,26 @@ void DerivationBuilderImpl::killSandbox(bool getStats) } } -bool DerivationBuilderImpl::prepareBuild() +bool DerivationBuilderImpl::killChild() { - if (useBuildUsers()) { - if (!buildUser) - buildUser = getBuildUser(); - - if (!buildUser) - return false; + bool ret = pid != -1; + if (ret) { + /* If we're using a build user, then there is a tricky race + condition: if we kill the build user before the child has + done its setuid() to the build user uid, then it won't be + killed, and we'll potentially lock up in pid.wait(). So + also send a conventional kill to the child. */ + ::kill(-pid, SIGKILL); /* ignore the result */ + + killSandbox(true); + + pid.wait(); } - - return true; + return ret; } -std::variant, SingleDrvOutputs> DerivationBuilderImpl::unprepareBuild() +SingleDrvOutputs DerivationBuilderImpl::unprepareBuild() { - // FIXME: get rid of this, rely on RAII. - Finally releaseBuildUser([&]() { - /* Release the build user at the end of this function. We don't do - it right away because we don't want another build grabbing this - uid and then messing around with our output. */ - buildUser.reset(); - }); - /* Since we got an EOF on the logger pipe, the builder is presumed to have terminated. In fact, the builder could also have simply have closed its end of the pipe, so just to be sure, @@ -475,63 +516,29 @@ std::variant, SingleDrvOutputs> Derivation ((double) buildResult.cpuSystem->count()) / 1000000); } - bool diskFull = false; - - try { - - /* Check the exit status. */ - if (!statusOk(status)) { - - diskFull |= decideWhetherDiskFull(); - - cleanupBuild(); - - auto msg = - fmt("Cannot build '%s'.\n" - "Reason: " ANSI_RED "builder %s" ANSI_NORMAL ".", - Magenta(store.printStorePath(drvPath)), - statusToString(status)); - - msg += showKnownOutputs(store, drv); - - miscMethods->appendLogTailErrorMsg(msg); - - if (diskFull) - msg += "\nnote: build failure may have been caused by lack of free disk space"; - - throw BuildError(msg); - } - - /* Compute the FS closure of the outputs and register them as - being valid. */ - auto builtOutputs = registerOutputs(); - - StorePathSet outputPaths; - for (auto & [_, output] : builtOutputs) - outputPaths.insert(output.outPath); - runPostBuildHook(store, *logger, drvPath, outputPaths); + /* Check the exit status. */ + if (!statusOk(status)) { - /* Delete unused redirected outputs (when doing hash rewriting). */ - for (auto & i : redirectedOutputs) - deletePath(store.Store::toRealPath(i.second)); + /* Check *before* cleaning up. */ + bool diskFull = decideWhetherDiskFull(); - deleteTmpDir(true); + cleanupBuild(false); - return std::move(builtOutputs); + throw BuilderFailureError{ + !derivationType.isSandboxed() || diskFull ? BuildResult::Failure::TransientFailure + : BuildResult::Failure::PermanentFailure, + status, + diskFull ? "\nnote: build failure may have been caused by lack of free disk space" : "", + }; + } - } catch (BuildError & e) { - BuildResult::Status st = dynamic_cast(&e) ? BuildResult::NotDeterministic - : statusOk(status) ? BuildResult::OutputRejected - : !derivationType.isSandboxed() || diskFull ? BuildResult::TransientFailure - : BuildResult::PermanentFailure; + /* Compute the FS closure of the outputs and register them as + being valid. */ + auto builtOutputs = registerOutputs(); - return std::pair{std::move(st), std::move(e)}; - } -} + cleanupBuild(true); -void DerivationBuilderImpl::cleanupBuild() -{ - deleteTmpDir(false); + return builtOutputs; } static void chmod_(const Path & path, mode_t mode) @@ -665,13 +672,8 @@ static bool checkNotWorldWritable(std::filesystem::path path) return true; } -void DerivationBuilderImpl::startBuilder() +void DerivationBuilderImpl::checkSystem() { - /* Make sure that no other processes are executing under the - sandbox uids. This must be done before any chownToBuilder() - calls. */ - prepareUser(); - /* Right platform? */ if (!drvOptions.canBuildLocally(store, drv)) { auto msg = @@ -693,9 +695,27 @@ void DerivationBuilderImpl::startBuilder() fmt("\nNote: run `%s` to run programs for x86_64-darwin", Magenta("/usr/sbin/softwareupdate --install-rosetta && launchctl stop org.nixos.nix-daemon")); - throw BuildError(msg); + throw BuildError(BuildResult::Failure::InputRejected, msg); + } +} + +std::optional DerivationBuilderImpl::startBuild() +{ + if (useBuildUsers()) { + if (!buildUser) + buildUser = getBuildUser(); + + if (!buildUser) + return std::nullopt; } + checkSystem(); + + /* Make sure that no other processes are executing under the + sandbox uids. This must be done before any chownToBuilder() + calls. */ + prepareUser(); + auto buildDir = store.config->getBuildDir(); createDirs(buildDir); @@ -706,7 +726,7 @@ void DerivationBuilderImpl::startBuilder() /* Create a temporary directory where the build will take place. */ - topTmpDir = createTempDir(buildDir, "nix-build-" + std::string(drvPath.name()), 0700); + topTmpDir = createTempDir(buildDir, "nix", 0700); setBuildTmpDir(); assert(!tmpDir.empty()); @@ -801,8 +821,7 @@ void DerivationBuilderImpl::startBuilder() if (!builderOut) throw SysError("opening pseudoterminal master"); - // FIXME: not thread-safe, use ptsname_r - std::string slaveName = ptsname(builderOut.get()); + std::string slaveName = getPtsName(builderOut.get()); if (buildUser) { if (chmod(slaveName.c_str(), 0600)) @@ -827,9 +846,10 @@ void DerivationBuilderImpl::startBuilder() startChild(); pid.setSeparatePG(true); - miscMethods->childStarted(builderOut.get()); processSandboxSetupMessages(); + + return builderOut.get(); } PathsInChroot DerivationBuilderImpl::getPathsInSandbox() @@ -838,6 +858,11 @@ PathsInChroot DerivationBuilderImpl::getPathsInSandbox() host file system. */ PathsInChroot pathsInChroot = defaultPathsInChroot; + for (auto & p : pathsInChroot) + if (!p.second.optional && !maybeLstat(p.second.source)) + throw SysError( + "path '%s' is configured as part of the `sandbox-paths` option, but is inaccessible", p.second.source); + if (hasPrefix(store.storeDir, tmpDirInSandbox())) { throw Error("`sandbox-build-dir` must not contain the storeDir"); } @@ -915,7 +940,7 @@ void DerivationBuilderImpl::prepareSandbox() void DerivationBuilderImpl::openSlave() { - std::string slaveName = ptsname(builderOut.get()); + std::string slaveName = getPtsName(builderOut.get()); AutoCloseFD builderOut = open(slaveName.c_str(), O_RDWR | O_NOCTTY); if (!builderOut) @@ -957,7 +982,7 @@ void DerivationBuilderImpl::processSandboxSetupMessages() "while waiting for the build environment for '%s' to initialize (%s, previous messages: %s)", store.printStorePath(drvPath), statusToString(status), - concatStringsSep("|", msgs)); + concatStringsSep("\n", msgs)); throw; } }(); @@ -1003,19 +1028,13 @@ void DerivationBuilderImpl::initEnv() /* Write the final environment. Note that this is intentionally *not* `drv.env`, because we've desugared things like like "passAFile", "expandReferencesGraph", structured attrs, etc. */ - for (const auto & [name, info] : finalEnv) { - if (info.nameOfPassAsFile) { - auto & fileName = *info.nameOfPassAsFile; - writeBuilderFile(fileName, rewriteStrings(info.value, inputRewrites)); - env[name] = tmpDirInSandbox() + "/" + fileName; - } else { - env[name] = info.value; - } + for (const auto & [name, info] : desugaredEnv.variables) { + env[name] = info.prependBuildDirectory ? tmpDirInSandbox() + "/" + info.value : info.value; } /* Add extra files, similar to `finalEnv` */ - for (const auto & [fileName, value] : extraFiles) { - writeBuilderFile(fileName, value); + for (const auto & [fileName, value] : desugaredEnv.extraFiles) { + writeBuilderFile(fileName, rewriteStrings(value, inputRewrites)); } /* For convenience, set an environment pointing to the top build @@ -1334,8 +1353,6 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() outputs to allow hard links between outputs. */ InodesSeen inodesSeen; - std::exception_ptr delayedException; - /* The paths that can be referenced are the input closures, the output paths, and any paths that have been built via recursive Nix calls. */ @@ -1389,6 +1406,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() auto optSt = maybeLstat(actualPath.c_str()); if (!optSt) throw BuildError( + BuildResult::Failure::OutputRejected, "builder for '%s' failed to produce output path for output '%s' at '%s'", store.printStorePath(drvPath), outputName, @@ -1403,6 +1421,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() if ((!S_ISLNK(st.st_mode) && (st.st_mode & (S_IWGRP | S_IWOTH))) || (buildUser && st.st_uid != buildUser->getUID())) throw BuildError( + BuildResult::Failure::OutputRejected, "suspicious ownership or permission on '%s' for output '%s'; rejecting this build output", actualPath, outputName); @@ -1439,7 +1458,11 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() {[&](const std::string & name) { auto orifu = get(outputReferencesIfUnregistered, name); if (!orifu) - throw BuildError("no output reference for '%s' in build of '%s'", name, store.printStorePath(drvPath)); + throw BuildError( + BuildResult::Failure::OutputRejected, + "no output reference for '%s' in build of '%s'", + name, + store.printStorePath(drvPath)); return std::visit( overloaded{ /* Since we'll use the already installed versions of these, we @@ -1461,6 +1484,7 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() {[&](const std::string & path, const std::string & parent) { // TODO with more -vvvv also show the temporary paths for manual inspection. return BuildError( + BuildResult::Failure::OutputRejected, "cycle detected in build of '%s' in the references of output '%s' from output '%s'", store.printStorePath(drvPath), path, @@ -1554,11 +1578,13 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() auto newInfoFromCA = [&](const DerivationOutput::CAFloating outputHash) -> ValidPathInfo { auto st = get(outputStats, outputName); if (!st) - throw BuildError("output path %1% without valid stats info", actualPath); + throw BuildError( + BuildResult::Failure::OutputRejected, "output path %1% without valid stats info", actualPath); if (outputHash.method.getFileIngestionMethod() == FileIngestionMethod::Flat) { /* The output path should be a regular file without execute permission. */ if (!S_ISREG(st->st_mode) || (st->st_mode & S_IXUSR) != 0) throw BuildError( + BuildResult::Failure::OutputRejected, "output path '%1%' should be a non-executable regular file " "since recursive hashing is not enabled (one of outputHashMode={flat,text} is true)", actualPath); @@ -1583,12 +1609,11 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() assert(false); }(); - ValidPathInfo newInfo0{ + auto newInfo0 = ValidPathInfo::makeFromCA( store, outputPathName(drv.name, outputName), ContentAddressWithReferences::fromParts(outputHash.method, std::move(got), rewriteRefs()), - Hash::dummy, - }; + Hash::dummy); if (*scratchPath != newInfo0.path) { // If the path has some self-references, we need to rewrite // them. @@ -1646,35 +1671,11 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() std::filesystem::rename(tmpOutput, actualPath); - auto newInfo0 = newInfoFromCA( + return newInfoFromCA( DerivationOutput::CAFloating{ .method = dof.ca.method, .hashAlgo = wanted.algo, }); - - /* Check wanted hash */ - assert(newInfo0.ca); - auto & got = newInfo0.ca->hash; - if (wanted != got) { - /* Throw an error after registering the path as - valid. */ - miscMethods->noteHashMismatch(); - delayedException = std::make_exception_ptr(BuildError( - "hash mismatch in fixed-output derivation '%s':\n specified: %s\n got: %s", - store.printStorePath(drvPath), - wanted.to_string(HashFormat::SRI, true), - got.to_string(HashFormat::SRI, true))); - } - if (!newInfo0.references.empty()) { - auto numViolations = newInfo.references.size(); - delayedException = std::make_exception_ptr(BuildError( - "fixed-output derivations must not reference store paths: '%s' references %d distinct paths, e.g. '%s'", - store.printStorePath(drvPath), - numViolations, - store.printStorePath(*newInfo.references.begin()))); - } - - return newInfo0; }, [&](const DerivationOutput::CAFloating & dof) { return newInfoFromCA(dof); }, @@ -1729,6 +1730,8 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() /* Path already exists because CA path produced by something else. No moving needed. */ assert(newInfo.ca); + /* Can delete our scratch copy now. */ + deletePath(actualPath); } else { auto destPath = store.toRealPath(finalDestPath); deletePath(destPath); @@ -1738,85 +1741,90 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() } if (buildMode == bmCheck) { + /* Check against already registered outputs */ + + if (store.isValidPath(newInfo.path)) { + ValidPathInfo oldInfo(*store.queryPathInfo(newInfo.path)); + if (newInfo.narHash != oldInfo.narHash) { + if (settings.runDiffHook || settings.keepFailed) { + auto dst = store.toRealPath(finalDestPath + ".check"); + deletePath(dst); + movePath(actualPath, dst); + + handleDiffHook( + buildUser ? buildUser->getUID() : getuid(), + buildUser ? buildUser->getGID() : getgid(), + finalDestPath, + dst, + store.printStorePath(drvPath), + tmpDir); - if (!store.isValidPath(newInfo.path)) - continue; - ValidPathInfo oldInfo(*store.queryPathInfo(newInfo.path)); - if (newInfo.narHash != oldInfo.narHash) { - miscMethods->noteCheckMismatch(); - if (settings.runDiffHook || settings.keepFailed) { - auto dst = store.toRealPath(finalDestPath + ".check"); - deletePath(dst); - movePath(actualPath, dst); - - handleDiffHook( - buildUser ? buildUser->getUID() : getuid(), - buildUser ? buildUser->getGID() : getgid(), - finalDestPath, - dst, - store.printStorePath(drvPath), - tmpDir); - - throw NotDeterministic( - "derivation '%s' may not be deterministic: output '%s' differs from '%s'", - store.printStorePath(drvPath), - store.toRealPath(finalDestPath), - dst); - } else - throw NotDeterministic( - "derivation '%s' may not be deterministic: output '%s' differs", - store.printStorePath(drvPath), - store.toRealPath(finalDestPath)); - } + throw NotDeterministic( + "derivation '%s' may not be deterministic: output '%s' differs from '%s'", + store.printStorePath(drvPath), + store.toRealPath(finalDestPath), + dst); + } else + throw NotDeterministic( + "derivation '%s' may not be deterministic: output '%s' differs", + store.printStorePath(drvPath), + store.toRealPath(finalDestPath)); + } - /* Since we verified the build, it's now ultimately trusted. */ - if (!oldInfo.ultimate) { - oldInfo.ultimate = true; - store.signPathInfo(oldInfo); - store.registerValidPaths({{oldInfo.path, oldInfo}}); + /* Since we verified the build, it's now ultimately trusted. */ + if (!oldInfo.ultimate) { + oldInfo.ultimate = true; + store.signPathInfo(oldInfo); + store.registerValidPaths({{oldInfo.path, oldInfo}}); + } } + } else { + /* do tasks relating to registering these outputs */ - continue; - } + /* For debugging, print out the referenced and unreferenced paths. */ + for (auto & i : inputPaths) { + if (references.count(i)) + debug("referenced input: '%1%'", store.printStorePath(i)); + else + debug("unreferenced input: '%1%'", store.printStorePath(i)); + } - /* For debugging, print out the referenced and unreferenced paths. */ - for (auto & i : inputPaths) { - if (references.count(i)) - debug("referenced input: '%1%'", store.printStorePath(i)); - else - debug("unreferenced input: '%1%'", store.printStorePath(i)); - } + store.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() - store.optimisePath(actualPath, NoRepair); // FIXME: combine with scanForReferences() - miscMethods->markContentsGood(newInfo.path); + newInfo.deriver = drvPath; + newInfo.ultimate = true; + store.signPathInfo(newInfo); - newInfo.deriver = drvPath; - newInfo.ultimate = true; - store.signPathInfo(newInfo); + finish(newInfo.path); - finish(newInfo.path); + /* If it's a CA path, register it right away. This is necessary if it + isn't statically known so that we can safely unlock the path before + the next iteration - /* If it's a CA path, register it right away. This is necessary if it - isn't statically known so that we can safely unlock the path before - the next iteration */ - if (newInfo.ca) - store.registerValidPaths({{newInfo.path, newInfo}}); + This is also good so that if a fixed-output produces the + wrong path, we still store the result (just don't consider + the derivation sucessful, so if someone fixes the problem by + just changing the wanted hash, the redownload (or whateer + possibly quite slow thing it was) doesn't have to be done + again. */ + if (newInfo.ca) + store.registerValidPaths({{newInfo.path, newInfo}}); + } + /* Do this in both the check and non-check cases, because we + want `checkOutputs` below to work, which needs these path + infos. */ infos.emplace(outputName, std::move(newInfo)); } + /* Apply output checks. This includes checking of the wanted vs got + hash of fixed-outputs. */ + checkOutputs(store, drvPath, drv.outputs, drvOptions.outputChecks, infos, *act); + if (buildMode == bmCheck) { - /* In case of fixed-output derivations, if there are - mismatches on `--check` an error must be thrown as this is - also a source for non-determinism. */ - if (delayedException) - std::rethrow_exception(delayedException); return {}; } - /* Apply output checks. */ - checkOutputs(infos); - /* Register each output path as valid, and register the sets of paths referenced by each of them. If there are cycles in the outputs, this will fail. */ @@ -1828,16 +1836,11 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() store.registerValidPaths(infos2); } - /* In case of a fixed-output derivation hash mismatch, throw an - exception now that we have registered the output as valid. */ - if (delayedException) - std::rethrow_exception(delayedException); - - /* If we made it this far, we are sure the output matches the derivation - (since the delayedException would be a fixed output CA mismatch). That - means it's safe to link the derivation to the output hash. We must do - that for floating CA derivations, which otherwise couldn't be cached, - but it's fine to do in all cases. */ + /* If we made it this far, we are sure the output matches the + derivation That means it's safe to link the derivation to the + output hash. We must do that for floating CA derivations, which + otherwise couldn't be cached, but it's fine to do in all cases. + */ SingleDrvOutputs builtOutputs; for (auto & [outputName, newInfo] : infos) { @@ -1854,151 +1857,14 @@ SingleDrvOutputs DerivationBuilderImpl::registerOutputs() return builtOutputs; } -void DerivationBuilderImpl::checkOutputs(const std::map & outputs) +void DerivationBuilderImpl::cleanupBuild(bool force) { - std::map outputsByPath; - for (auto & output : outputs) - outputsByPath.emplace(store.printStorePath(output.second.path), output.second); - - for (auto & output : outputs) { - auto & outputName = output.first; - auto & info = output.second; - - /* Compute the closure and closure size of some output. This - is slightly tricky because some of its references (namely - other outputs) may not be valid yet. */ - auto getClosure = [&](const StorePath & path) { - uint64_t closureSize = 0; - StorePathSet pathsDone; - std::queue pathsLeft; - pathsLeft.push(path); - - while (!pathsLeft.empty()) { - auto path = pathsLeft.front(); - pathsLeft.pop(); - if (!pathsDone.insert(path).second) - continue; - - auto i = outputsByPath.find(store.printStorePath(path)); - if (i != outputsByPath.end()) { - closureSize += i->second.narSize; - for (auto & ref : i->second.references) - pathsLeft.push(ref); - } else { - auto info = store.queryPathInfo(path); - closureSize += info->narSize; - for (auto & ref : info->references) - pathsLeft.push(ref); - } - } - - return std::make_pair(std::move(pathsDone), closureSize); - }; - - auto applyChecks = [&](const DerivationOptions::OutputChecks & checks) { - if (checks.maxSize && info.narSize > *checks.maxSize) - throw BuildError( - "path '%s' is too large at %d bytes; limit is %d bytes", - store.printStorePath(info.path), - info.narSize, - *checks.maxSize); - - if (checks.maxClosureSize) { - uint64_t closureSize = getClosure(info.path).second; - if (closureSize > *checks.maxClosureSize) - throw BuildError( - "closure of path '%s' is too large at %d bytes; limit is %d bytes", - store.printStorePath(info.path), - closureSize, - *checks.maxClosureSize); - } - - auto checkRefs = [&](const StringSet & value, bool allowed, bool recursive) { - /* Parse a list of reference specifiers. Each element must - either be a store path, or the symbolic name of the output - of the derivation (such as `out'). */ - StorePathSet spec; - for (auto & i : value) { - if (store.isStorePath(i)) - spec.insert(store.parseStorePath(i)); - else if (auto output = get(outputs, i)) - spec.insert(output->path); - else { - std::string outputsListing = - concatMapStringsSep(", ", outputs, [](auto & o) { return o.first; }); - throw BuildError( - "derivation '%s' output check for '%s' contains an illegal reference specifier '%s'," - " expected store path or output name (one of [%s])", - store.printStorePath(drvPath), - outputName, - i, - outputsListing); - } - } - - auto used = recursive ? getClosure(info.path).first : info.references; - - if (recursive && checks.ignoreSelfRefs) - used.erase(info.path); - - StorePathSet badPaths; - - for (auto & i : used) - if (allowed) { - if (!spec.count(i)) - badPaths.insert(i); - } else { - if (spec.count(i)) - badPaths.insert(i); - } - - if (!badPaths.empty()) { - std::string badPathsStr; - for (auto & i : badPaths) { - badPathsStr += "\n "; - badPathsStr += store.printStorePath(i); - } - throw BuildError( - "output '%s' is not allowed to refer to the following paths:%s", - store.printStorePath(info.path), - badPathsStr); - } - }; - - /* Mandatory check: absent whitelist, and present but empty - whitelist mean very different things. */ - if (auto & refs = checks.allowedReferences) { - checkRefs(*refs, true, false); - } - if (auto & refs = checks.allowedRequisites) { - checkRefs(*refs, true, true); - } - - /* Optimization: don't need to do anything when - disallowed and empty set. */ - if (!checks.disallowedReferences.empty()) { - checkRefs(checks.disallowedReferences, false, false); - } - if (!checks.disallowedRequisites.empty()) { - checkRefs(checks.disallowedRequisites, false, true); - } - }; - - std::visit( - overloaded{ - [&](const DerivationOptions::OutputChecks & checks) { applyChecks(checks); }, - [&](const std::map & checksPerOutput) { - if (auto outputChecks = get(checksPerOutput, outputName)) - - applyChecks(*outputChecks); - }, - }, - drvOptions.outputChecks); + if (force) { + /* Delete unused redirected outputs (when doing hash rewriting). */ + for (auto & i : redirectedOutputs) + deletePath(store.Store::toRealPath(i.second)); } -} -void DerivationBuilderImpl::deleteTmpDir(bool force) -{ if (topTmpDir != "") { /* As an extra precaution, even in the event of `deletePath` failing to * clean up, the `tmpDir` will be chowned as if we were to move @@ -2054,12 +1920,16 @@ StorePath DerivationBuilderImpl::makeFallbackPath(const StorePath & path) #include "chroot-derivation-builder.cc" #include "linux-derivation-builder.cc" #include "darwin-derivation-builder.cc" +#include "external-derivation-builder.cc" namespace nix { std::unique_ptr makeDerivationBuilder( LocalStore & store, std::unique_ptr miscMethods, DerivationBuilderParams params) { + if (auto builder = ExternalDerivationBuilder::newIfSupported(store, miscMethods, params)) + return builder; + bool useSandbox = false; /* Are we doing a sandboxed build? */ diff --git a/src/libstore/unix/build/external-derivation-builder.cc b/src/libstore/unix/build/external-derivation-builder.cc new file mode 100644 index 00000000000..71cfd1a62c0 --- /dev/null +++ b/src/libstore/unix/build/external-derivation-builder.cc @@ -0,0 +1,123 @@ +namespace nix { + +struct ExternalDerivationBuilder : DerivationBuilderImpl +{ + Settings::ExternalBuilder externalBuilder; + + ExternalDerivationBuilder( + LocalStore & store, + std::unique_ptr miscMethods, + DerivationBuilderParams params, + Settings::ExternalBuilder externalBuilder) + : DerivationBuilderImpl(store, std::move(miscMethods), std::move(params)) + , externalBuilder(std::move(externalBuilder)) + { + experimentalFeatureSettings.require(Xp::ExternalBuilders); + } + + static std::unique_ptr newIfSupported( + LocalStore & store, std::unique_ptr & miscMethods, DerivationBuilderParams & params) + { + for (auto & handler : settings.externalBuilders.get()) { + for (auto & system : handler.systems) + if (params.drv.platform == system) + return std::make_unique( + store, std::move(miscMethods), std::move(params), handler); + } + return {}; + } + + Path tmpDirInSandbox() override + { + /* In a sandbox, for determinism, always use the same temporary + directory. */ + return "/build"; + } + + void setBuildTmpDir() override + { + tmpDir = topTmpDir + "/build"; + createDir(tmpDir, 0700); + } + + void checkSystem() override {} + + void startChild() override + { + if (drvOptions.getRequiredSystemFeatures(drv).count("recursive-nix")) + throw Error("'recursive-nix' is not supported yet by external derivation builders"); + + auto json = nlohmann::json::object(); + + json.emplace("version", 1); + json.emplace("builder", drv.builder); + { + auto l = nlohmann::json::array(); + for (auto & i : drv.args) + l.push_back(rewriteStrings(i, inputRewrites)); + json.emplace("args", std::move(l)); + } + { + auto j = nlohmann::json::object(); + for (auto & [name, value] : env) + j.emplace(name, rewriteStrings(value, inputRewrites)); + json.emplace("env", std::move(j)); + } + json.emplace("topTmpDir", topTmpDir); + json.emplace("tmpDir", tmpDir); + json.emplace("tmpDirInSandbox", tmpDirInSandbox()); + json.emplace("storeDir", store.storeDir); + json.emplace("realStoreDir", store.config->realStoreDir.get()); + json.emplace("system", drv.platform); + { + auto l = nlohmann::json::array(); + for (auto & i : inputPaths) + l.push_back(store.printStorePath(i)); + json.emplace("inputPaths", std::move(l)); + } + { + auto l = nlohmann::json::object(); + for (auto & i : scratchOutputs) + l.emplace(i.first, store.printStorePath(i.second)); + json.emplace("outputs", std::move(l)); + } + + // TODO(cole-h): writing this to stdin is too much effort right now, if we want to revisit + // that, see this comment by Eelco about how to make it not suck: + // https://github.com/DeterminateSystems/nix-src/pull/141#discussion_r2205493257 + auto jsonFile = std::filesystem::path{topTmpDir} / "build.json"; + writeFile(jsonFile, json.dump()); + + pid = startProcess([&]() { + openSlave(); + try { + commonChildInit(); + + Strings args = {externalBuilder.program}; + + if (!externalBuilder.args.empty()) { + args.insert(args.end(), externalBuilder.args.begin(), externalBuilder.args.end()); + } + + args.insert(args.end(), jsonFile); + + if (chdir(tmpDir.c_str()) == -1) + throw SysError("changing into '%1%'", tmpDir); + + chownToBuilder(topTmpDir); + + setUser(); + + debug("executing external builder: %s", concatStringsSep(" ", args)); + execv(externalBuilder.program.c_str(), stringsToCharPtrs(args).data()); + + throw SysError("executing '%s'", externalBuilder.program); + } catch (...) { + handleChildException(true); + _exit(1); + } + }); + } +}; + +} // namespace nix diff --git a/src/libstore/unix/build/linux-derivation-builder.cc b/src/libstore/unix/build/linux-derivation-builder.cc index 0d9dc4a8579..f6e910d08a9 100644 --- a/src/libstore/unix/build/linux-derivation-builder.cc +++ b/src/libstore/unix/build/linux-derivation-builder.cc @@ -3,6 +3,7 @@ # include "nix/store/personality.hh" # include "nix/util/cgroup.hh" # include "nix/util/linux-namespaces.hh" +# include "nix/util/logging.hh" # include "linux/fchmodat2-compat.hh" # include @@ -362,9 +363,21 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu userNamespaceSync.readSide = -1; - /* Close the write side to prevent runChild() from hanging - reading from this. */ - Finally cleanup([&]() { userNamespaceSync.writeSide = -1; }); + /* Make sure that we write *something* to the child in case of + an exception. Note that merely closing + `userNamespaceSync.writeSide` doesn't work in + multi-threaded Nix, since several child processes may have + inherited `writeSide` (and O_CLOEXEC doesn't help because + the children may not do an execve). */ + bool userNamespaceSyncDone = false; + Finally cleanup([&]() { + try { + if (!userNamespaceSyncDone) + writeFull(userNamespaceSync.writeSide.get(), "0\n"); + } catch (...) { + } + userNamespaceSync.writeSide = -1; + }); auto ss = tokenizeString>(readLine(sendPid.readSide.get())); assert(ss.size() == 1); @@ -419,14 +432,15 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu writeFile(*cgroup + "/cgroup.procs", fmt("%d", (pid_t) pid)); /* Signal the builder that we've updated its user namespace. */ - writeFull(userNamespaceSync.writeSide.get(), "1"); + writeFull(userNamespaceSync.writeSide.get(), "1\n"); + userNamespaceSyncDone = true; } void enterChroot() override { userNamespaceSync.writeSide = -1; - if (drainFD(userNamespaceSync.readSide.get()) != "1") + if (readLine(userNamespaceSync.readSide.get()) != "1") throw Error("user namespace initialisation failed"); userNamespaceSync.readSide = -1; @@ -492,8 +506,16 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu createDirs(chrootRootDir + "/dev/shm"); createDirs(chrootRootDir + "/dev/pts"); ss.push_back("/dev/full"); - if (store.Store::config.systemFeatures.get().count("kvm") && pathExists("/dev/kvm")) - ss.push_back("/dev/kvm"); + if (systemFeatures.count("kvm")) { + if (pathExists("/dev/kvm")) { + ss.push_back("/dev/kvm"); + } else { + warn( + "KVM is enabled in system-features but /dev/kvm is not available. " + "QEMU builds may fall back to slow emulation. " + "Consider removing 'kvm' from system-features in nix.conf if KVM is not supported on this system."); + } + } ss.push_back("/dev/null"); ss.push_back("/dev/random"); ss.push_back("/dev/tty"); @@ -659,7 +681,7 @@ struct ChrootLinuxDerivationBuilder : ChrootDerivationBuilder, LinuxDerivationBu throw SysError("setuid failed"); } - std::variant, SingleDrvOutputs> unprepareBuild() override + SingleDrvOutputs unprepareBuild() override { sandboxMountNamespace = -1; sandboxUserNamespace = -1; diff --git a/src/libstore/unix/include/nix/store/build/hook-instance.hh b/src/libstore/unix/include/nix/store/build/hook-instance.hh index 87e03665c72..7657d5dbd08 100644 --- a/src/libstore/unix/include/nix/store/build/hook-instance.hh +++ b/src/libstore/unix/include/nix/store/build/hook-instance.hh @@ -7,6 +7,14 @@ namespace nix { +/** + * @note Sometimes this is owned by the `Worker`, and sometimes it is + * owned by a `Goal`. This is for efficiency: rather than starting the + * hook every time we want to ask whether we can run a remote build + * (which can be very often), we reuse a hook process for answering + * those queries until it accepts a build. So if there are N + * derivations to be built, at most N hooks will be started. + */ struct HookInstance { /** @@ -29,6 +37,15 @@ struct HookInstance */ Pid pid; + /** + * The remote machine on which we're building. + * + * @Invariant When the hook instance is owned by the `Worker`, this + * is the empty string. When it is owned by a `Goal`, this should be + * set. + */ + std::string machineName; + FdSink sink; std::map activities; diff --git a/src/libstore/worker-protocol-connection.cc b/src/libstore/worker-protocol-connection.cc index 987d0c8dde8..8a37662904d 100644 --- a/src/libstore/worker-protocol-connection.cc +++ b/src/libstore/worker-protocol-connection.cc @@ -313,12 +313,4 @@ void WorkerProto::BasicClientConnection::narFromPath( fun(from); } -void WorkerProto::BasicClientConnection::importPaths( - const StoreDirConfig & store, bool * daemonException, Source & source) -{ - to << WorkerProto::Op::ImportPaths; - processStderr(daemonException, 0, &source); - auto importedPaths = WorkerProto::Serialise::read(store, *this); - assert(importedPaths.size() <= importedPaths.size()); -} } // namespace nix diff --git a/src/libstore/worker-protocol.cc b/src/libstore/worker-protocol.cc index 1bbff64a25b..a17d2c02857 100644 --- a/src/libstore/worker-protocol.cc +++ b/src/libstore/worker-protocol.cc @@ -165,10 +165,14 @@ void WorkerProto::Serialise::write( BuildResult WorkerProto::Serialise::read(const StoreDirConfig & store, WorkerProto::ReadConn conn) { BuildResult res; - res.status = static_cast(readInt(conn.from)); - conn.from >> res.errorMsg; + BuildResult::Success success; + BuildResult::Failure failure; + + auto rawStatus = readInt(conn.from); + conn.from >> failure.errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { - conn.from >> res.timesBuilt >> res.isNonDeterministic >> res.startTime >> res.stopTime; + conn.from >> res.timesBuilt >> failure.isNonDeterministic >> res.startTime >> res.stopTime; } if (GET_PROTOCOL_MINOR(conn.version) >= 37) { res.cpuUser = WorkerProto::Serialise>::read(store, conn); @@ -177,28 +181,56 @@ BuildResult WorkerProto::Serialise::read(const StoreDirConfig & sto if (GET_PROTOCOL_MINOR(conn.version) >= 28) { auto builtOutputs = WorkerProto::Serialise::read(store, conn); for (auto && [output, realisation] : builtOutputs) - res.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + success.builtOutputs.insert_or_assign(std::move(output.outputName), std::move(realisation)); + } + + if (BuildResult::Success::statusIs(rawStatus)) { + success.status = static_cast(rawStatus); + res.inner = std::move(success); + } else { + failure.status = static_cast(rawStatus); + res.inner = std::move(failure); } + return res; } void WorkerProto::Serialise::write( const StoreDirConfig & store, WorkerProto::WriteConn conn, const BuildResult & res) { - conn.to << res.status << res.errorMsg; - if (GET_PROTOCOL_MINOR(conn.version) >= 29) { - conn.to << res.timesBuilt << res.isNonDeterministic << res.startTime << res.stopTime; - } - if (GET_PROTOCOL_MINOR(conn.version) >= 37) { - WorkerProto::write(store, conn, res.cpuUser); - WorkerProto::write(store, conn, res.cpuSystem); - } - if (GET_PROTOCOL_MINOR(conn.version) >= 28) { - DrvOutputs builtOutputs; - for (auto & [output, realisation] : res.builtOutputs) - builtOutputs.insert_or_assign(realisation.id, realisation); - WorkerProto::write(store, conn, builtOutputs); - } + /* The protocol predates the use of sum types (std::variant) to + separate the success or failure cases. As such, it transits some + success- or failure-only fields in both cases. This helper + function helps support this: in each case, we just pass the old + default value for the fields that don't exist in that case. */ + auto common = [&](std::string_view errorMsg, bool isNonDeterministic, const auto & builtOutputs) { + conn.to << errorMsg; + if (GET_PROTOCOL_MINOR(conn.version) >= 29) { + conn.to << res.timesBuilt << isNonDeterministic << res.startTime << res.stopTime; + } + if (GET_PROTOCOL_MINOR(conn.version) >= 37) { + WorkerProto::write(store, conn, res.cpuUser); + WorkerProto::write(store, conn, res.cpuSystem); + } + if (GET_PROTOCOL_MINOR(conn.version) >= 28) { + DrvOutputs builtOutputsFullKey; + for (auto & [output, realisation] : builtOutputs) + builtOutputsFullKey.insert_or_assign(realisation.id, realisation); + WorkerProto::write(store, conn, builtOutputsFullKey); + } + }; + std::visit( + overloaded{ + [&](const BuildResult::Failure & failure) { + conn.to << failure.status; + common(failure.errorMsg, failure.isNonDeterministic, decltype(BuildResult::Success::builtOutputs){}); + }, + [&](const BuildResult::Success & success) { + conn.to << success.status; + common(/*errorMsg=*/"", /*isNonDeterministic=*/false, success.builtOutputs); + }, + }, + res.inner); } ValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) @@ -219,11 +251,10 @@ void WorkerProto::Serialise::write( UnkeyedValidPathInfo WorkerProto::Serialise::read(const StoreDirConfig & store, ReadConn conn) { - auto deriver = readString(conn.from); + auto deriver = WorkerProto::Serialise>::read(store, conn); auto narHash = Hash::parseAny(readString(conn.from), HashAlgorithm::SHA256); UnkeyedValidPathInfo info(narHash); - if (deriver != "") - info.deriver = store.parseStorePath(deriver); + info.deriver = std::move(deriver); info.references = WorkerProto::Serialise::read(store, conn); conn.from >> info.registrationTime >> info.narSize; if (GET_PROTOCOL_MINOR(conn.version) >= 16) { @@ -237,8 +268,8 @@ UnkeyedValidPathInfo WorkerProto::Serialise::read(const St void WorkerProto::Serialise::write( const StoreDirConfig & store, WriteConn conn, const UnkeyedValidPathInfo & pathInfo) { - conn.to << (pathInfo.deriver ? store.printStorePath(*pathInfo.deriver) : "") - << pathInfo.narHash.to_string(HashFormat::Base16, false); + WorkerProto::write(store, conn, pathInfo.deriver); + conn.to << pathInfo.narHash.to_string(HashFormat::Base16, false); WorkerProto::write(store, conn, pathInfo.references); conn.to << pathInfo.registrationTime << pathInfo.narSize; if (GET_PROTOCOL_MINOR(conn.version) >= 16) { diff --git a/src/libutil-c/meson.build b/src/libutil-c/meson.build index 8131c517cd8..54fd53c74f0 100644 --- a/src/libutil-c/meson.build +++ b/src/libutil-c/meson.build @@ -32,6 +32,7 @@ config_priv_h = configure_file( ) subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'nix_api_util.cc', @@ -53,6 +54,7 @@ this_library = library( 'nixutilc', sources, config_priv_h, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, diff --git a/src/libutil-c/nix_api_util.cc b/src/libutil-c/nix_api_util.cc index 2254f18fa97..3903823aa9b 100644 --- a/src/libutil-c/nix_api_util.cc +++ b/src/libutil-c/nix_api_util.cc @@ -9,6 +9,8 @@ #include "nix_api_util_config.h" +extern "C" { + nix_c_context * nix_c_context_create() { return new nix_c_context(); @@ -156,3 +158,17 @@ nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callb callback(str.c_str(), str.size(), user_data); return NIX_OK; } + +nix_err nix_set_verbosity(nix_c_context * context, nix_verbosity level) +{ + if (context) + context->last_err_code = NIX_OK; + if (level > NIX_LVL_VOMIT || level < NIX_LVL_ERROR) + return nix_set_err_msg(context, NIX_ERR_UNKNOWN, "Invalid verbosity level"); + try { + nix::verbosity = static_cast(level); + } + NIXC_CATCH_ERRS +} + +} // extern "C" diff --git a/src/libutil-c/nix_api_util.h b/src/libutil-c/nix_api_util.h index 5f42641d426..4d7f394fa01 100644 --- a/src/libutil-c/nix_api_util.h +++ b/src/libutil-c/nix_api_util.h @@ -53,7 +53,7 @@ extern "C" { * - NIX_OK: No error occurred (0) * - NIX_ERR_UNKNOWN: An unknown error occurred (-1) * - NIX_ERR_OVERFLOW: An overflow error occurred (-2) - * - NIX_ERR_KEY: A key error occurred (-3) + * - NIX_ERR_KEY: A key/index access error occurred in C API functions (-3) * - NIX_ERR_NIX_ERROR: A generic Nix error occurred (-4) */ enum nix_err { @@ -83,10 +83,21 @@ enum nix_err { NIX_ERR_OVERFLOW = -2, /** - * @brief A key error occurred. + * @brief A key/index access error occurred in C API functions. * - * This error code is returned when a key error occurred during the function - * execution. + * This error code is returned when accessing a key, index, or identifier that + * does not exist in C API functions. Common scenarios include: + * - Setting keys that don't exist (nix_setting_get, nix_setting_set) + * - List indices that are out of bounds (nix_get_list_byidx*) + * - Attribute names that don't exist (nix_get_attr_byname*) + * - Attribute indices that are out of bounds (nix_get_attr_byidx*, nix_get_attr_name_byidx) + * + * This error typically indicates incorrect usage or assumptions about data structure + * contents, rather than internal Nix evaluation errors. + * + * @note This error code should ONLY be returned by C API functions themselves, + * not by underlying Nix evaluation. For example, evaluating `{}.foo` in Nix + * will throw a normal error (NIX_ERR_NIX_ERROR), not NIX_ERR_KEY. */ NIX_ERR_KEY = -3, @@ -102,6 +113,24 @@ enum nix_err { typedef enum nix_err nix_err; +/** + * @brief Verbosity level + * + * @note This should be kept in sync with the C++ implementation (nix::Verbosity) + */ +enum nix_verbosity { + NIX_LVL_ERROR = 0, + NIX_LVL_WARN, + NIX_LVL_NOTICE, + NIX_LVL_INFO, + NIX_LVL_TALKATIVE, + NIX_LVL_CHATTY, + NIX_LVL_DEBUG, + NIX_LVL_VOMIT, +}; + +typedef enum nix_verbosity nix_verbosity; + /** * @brief This object stores error state. * @struct nix_c_context @@ -316,6 +345,14 @@ nix_err nix_set_err_msg(nix_c_context * context, nix_err err, const char * msg); */ void nix_clear_err(nix_c_context * context); +/** + * @brief Sets the verbosity level + * + * @param[out] context Optional, additional error context. + * @param[in] level Verbosity level + */ +nix_err nix_set_verbosity(nix_c_context * context, nix_verbosity level); + /** * @} */ diff --git a/src/libutil-c/nix_api_util_internal.h b/src/libutil-c/nix_api_util_internal.h index 664cd6e239f..92bb9c1d298 100644 --- a/src/libutil-c/nix_api_util_internal.h +++ b/src/libutil-c/nix_api_util_internal.h @@ -7,6 +7,8 @@ #include "nix/util/error.hh" #include "nix_api_util.h" +extern "C" { + struct nix_c_context { nix_err last_err_code = NIX_OK; @@ -47,4 +49,6 @@ nix_err call_nix_get_string_callback(const std::string str, nix_get_string_callb } #define NIXC_CATCH_ERRS_NULL NIXC_CATCH_ERRS_RES(nullptr) +} // extern "C" + #endif // NIX_API_UTIL_INTERNAL_H diff --git a/src/libutil-c/package.nix b/src/libutil-c/package.nix index f26f57775d4..a1605bf5bb8 100644 --- a/src/libutil-c/package.nix +++ b/src/libutil-c/package.nix @@ -14,7 +14,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-util-c"; + pname = "determinate-nix-util-c"; inherit version; workDir = ./.; diff --git a/src/libutil-test-support/include/nix/util/tests/json-characterization.hh b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh new file mode 100644 index 00000000000..5a38b8e2c42 --- /dev/null +++ b/src/libutil-test-support/include/nix/util/tests/json-characterization.hh @@ -0,0 +1,54 @@ +#pragma once +///@file + +#include +#include + +#include "nix/util/types.hh" +#include "nix/util/file-system.hh" + +#include "nix/util/tests/characterization.hh" + +namespace nix { + +/** + * Mixin class for writing characterization tests for `nlohmann::json` + * conversions for a given type. + */ +template +struct JsonCharacterizationTest : virtual CharacterizationTest +{ + /** + * Golden test for reading + * + * @param test hook that takes the contents of the file and does the + * actual work + */ + void readJsonTest(PathView testStem, const T & expected) + { + using namespace nlohmann; + readTest(Path{testStem} + ".json", [&](const auto & encodedRaw) { + auto encoded = json::parse(encodedRaw); + T decoded = adl_serializer::from_json(encoded); + ASSERT_EQ(decoded, expected); + }); + } + + /** + * Golden test for writing + * + * @param test hook that produces contents of the file and does the + * actual work + */ + void writeJsonTest(PathView testStem, const T & value) + { + using namespace nlohmann; + writeTest( + Path{testStem} + ".json", + [&]() -> json { return static_cast(value); }, + [](const auto & file) { return json::parse(readFile(file)); }, + [](const auto & file, const auto & got) { return writeFile(file, got.dump(2) + "\n"); }); + } +}; + +} // namespace nix diff --git a/src/libutil-test-support/include/nix/util/tests/meson.build b/src/libutil-test-support/include/nix/util/tests/meson.build index ab143757c0e..3be085892c9 100644 --- a/src/libutil-test-support/include/nix/util/tests/meson.build +++ b/src/libutil-test-support/include/nix/util/tests/meson.build @@ -7,6 +7,7 @@ headers = files( 'gmock-matchers.hh', 'gtest-with-params.hh', 'hash.hh', + 'json-characterization.hh', 'nix_api_util.hh', 'string_callback.hh', ) diff --git a/src/libutil-test-support/include/nix/util/tests/nix_api_util.hh b/src/libutil-test-support/include/nix/util/tests/nix_api_util.hh index 57f7f1ecf39..cc1d244f5c8 100644 --- a/src/libutil-test-support/include/nix/util/tests/nix_api_util.hh +++ b/src/libutil-test-support/include/nix/util/tests/nix_api_util.hh @@ -54,4 +54,12 @@ protected: #define assert_ctx_err() assert_ctx_err(__FILE__, __LINE__) }; +static inline auto createOwnedNixContext() +{ + return std::unique_ptr(nix_c_context_create(), {}); +} + } // namespace nixC diff --git a/src/libutil-test-support/meson.build b/src/libutil-test-support/meson.build index 910f1d88164..1ca251ce8dc 100644 --- a/src/libutil-test-support/meson.build +++ b/src/libutil-test-support/meson.build @@ -27,6 +27,7 @@ rapidcheck = dependency('rapidcheck') deps_public += rapidcheck subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( 'hash.cc', @@ -41,6 +42,7 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nix-util-test-support', sources, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, # TODO: Remove `-lrapidcheck` when https://github.com/emil-e/rapidcheck/pull/326 diff --git a/src/libutil-test-support/package.nix b/src/libutil-test-support/package.nix index f8e92c27113..40ff65d6135 100644 --- a/src/libutil-test-support/package.nix +++ b/src/libutil-test-support/package.nix @@ -17,7 +17,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-util-test-support"; + pname = "determinate-nix-util-test-support"; inherit version; workDir = ./.; diff --git a/src/libutil-tests/archive.cc b/src/libutil-tests/archive.cc new file mode 100644 index 00000000000..386f7b857ba --- /dev/null +++ b/src/libutil-tests/archive.cc @@ -0,0 +1,47 @@ +#include "nix/util/archive.hh" +#include "nix/util/tests/characterization.hh" +#include "nix/util/tests/gmock-matchers.hh" + +#include + +namespace nix { + +namespace { + +class NarTest : public CharacterizationTest +{ + std::filesystem::path unitTestData = getUnitTestData() / "nars"; + +public: + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / (std::string(testStem) + ".nar"); + } +}; + +class InvalidNarTest : public NarTest, public ::testing::WithParamInterface> +{}; + +} // namespace + +TEST_P(InvalidNarTest, throwsErrorMessage) +{ + const auto & [name, message] = GetParam(); + readTest(name, [&](const std::string & narContents) { + ASSERT_THAT( + [&]() { + StringSource source{narContents}; + NullFileSystemObjectSink sink; + parseDump(sink, source); + }, + ::testing::ThrowsMessage(testing::HasSubstrIgnoreANSIMatcher(message))); + }); +} + +INSTANTIATE_TEST_SUITE_P( + NarTest, + InvalidNarTest, + ::testing::Values( + std::pair{"invalid-tag-instead-of-contents", "bad archive: expected tag 'contents', got 'AAAAAAAA'"})); + +} // namespace nix diff --git a/src/libutil-tests/config.cc b/src/libutil-tests/config.cc index 5fb2229b6b9..87c1e556b73 100644 --- a/src/libutil-tests/config.cc +++ b/src/libutil-tests/config.cc @@ -218,7 +218,7 @@ TEST(Config, toJSONOnNonEmptyConfigWithExperimentalSetting) "description", {}, true, - Xp::Flakes, + Xp::CaDerivations, }; setting.assign("value"); @@ -231,7 +231,7 @@ TEST(Config, toJSONOnNonEmptyConfigWithExperimentalSetting) "description": "description\n", "documentDefault": true, "value": "value", - "experimentalFeature": "flakes" + "experimentalFeature": "ca-derivations" } })#"_json); } diff --git a/src/libutil-tests/data/nars/invalid-tag-instead-of-contents.nar b/src/libutil-tests/data/nars/invalid-tag-instead-of-contents.nar new file mode 100644 index 00000000000..80dbf5a12ff Binary files /dev/null and b/src/libutil-tests/data/nars/invalid-tag-instead-of-contents.nar differ diff --git a/src/libutil-tests/hash.cc b/src/libutil-tests/hash.cc index f9d425d92c0..15e63918018 100644 --- a/src/libutil-tests/hash.cc +++ b/src/libutil-tests/hash.cc @@ -1,13 +1,17 @@ #include #include +#include #include "nix/util/hash.hh" +#include "nix/util/tests/characterization.hh" namespace nix { -class BLAKE3HashTest : public virtual ::testing::Test +class HashTest : public CharacterizationTest { + std::filesystem::path unitTestData = getUnitTestData() / "hash"; + public: /** @@ -16,8 +20,14 @@ class BLAKE3HashTest : public virtual ::testing::Test */ ExperimentalFeatureSettings mockXpSettings; -private: + std::filesystem::path goldenMaster(std::string_view testStem) const override + { + return unitTestData / testStem; + } +}; +class BLAKE3HashTest : public HashTest +{ void SetUp() override { mockXpSettings.set("experimental-features", "blake3-hashes"); @@ -137,6 +147,46 @@ TEST(hashString, testKnownSHA512Hashes2) "c7d329eeb6dd26545e96e55b874be909"); } +/* ---------------------------------------------------------------------------- + * parsing hashes + * --------------------------------------------------------------------------*/ + +TEST(hashParseExplicitFormatUnprefixed, testKnownSHA256Hashes1_correct) +{ + // values taken from: https://tools.ietf.org/html/rfc4634 + auto s = "abc"; + + auto hash = hashString(HashAlgorithm::SHA256, s); + ASSERT_EQ( + hash, + Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + HashAlgorithm::SHA256, + HashFormat::Base16)); +} + +TEST(hashParseExplicitFormatUnprefixed, testKnownSHA256Hashes1_wrongAlgo) +{ + // values taken from: https://tools.ietf.org/html/rfc4634 + ASSERT_THROW( + Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + HashAlgorithm::SHA1, + HashFormat::Base16), + BadHash); +} + +TEST(hashParseExplicitFormatUnprefixed, testKnownSHA256Hashes1_wrongBase) +{ + // values taken from: https://tools.ietf.org/html/rfc4634 + ASSERT_THROW( + Hash::parseExplicitFormatUnprefixed( + "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", + HashAlgorithm::SHA256, + HashFormat::Nix32), + BadHash); +} + /* ---------------------------------------------------------------------------- * parseHashFormat, parseHashFormatOpt, printHashFormat * --------------------------------------------------------------------------*/ diff --git a/src/libutil-tests/meson.build b/src/libutil-tests/meson.build index 0e2a2e4680c..83245a73ded 100644 --- a/src/libutil-tests/meson.build +++ b/src/libutil-tests/meson.build @@ -42,8 +42,10 @@ config_priv_h = configure_file( ) subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = files( + 'archive.cc', 'args.cc', 'base-n.cc', 'canon-path.cc', @@ -63,6 +65,7 @@ sources = files( 'lru-cache.cc', 'monitorfdhup.cc', 'nix_api_util.cc', + 'nix_api_util_internal.cc', 'pool.cc', 'position.cc', 'processes.cc', @@ -95,7 +98,7 @@ this_exe = executable( test( meson.project_name(), this_exe, - env : { + env : asan_test_options_env + { '_NIX_TEST_UNIT_DATA' : meson.current_source_dir() / 'data', }, protocol : 'gtest', diff --git a/src/libutil-tests/nix_api_util.cc b/src/libutil-tests/nix_api_util.cc index 9693ab3a530..48f85c403d6 100644 --- a/src/libutil-tests/nix_api_util.cc +++ b/src/libutil-tests/nix_api_util.cc @@ -1,7 +1,6 @@ #include "nix/util/config-global.hh" #include "nix/util/args.hh" #include "nix_api_util.h" -#include "nix_api_util_internal.h" #include "nix/util/tests/nix_api_util.hh" #include "nix/util/tests/string_callback.hh" @@ -13,41 +12,6 @@ namespace nixC { -TEST_F(nix_api_util_context, nix_context_error) -{ - std::string err_msg_ref; - try { - throw nix::Error("testing error"); - } catch (nix::Error & e) { - err_msg_ref = e.what(); - nix_context_error(ctx); - } - ASSERT_EQ(ctx->last_err_code, NIX_ERR_NIX_ERROR); - ASSERT_EQ(ctx->name, "nix::Error"); - ASSERT_EQ(*ctx->last_err, err_msg_ref); - ASSERT_EQ(ctx->info->msg.str(), "testing error"); - - try { - throw std::runtime_error("testing exception"); - } catch (std::exception & e) { - err_msg_ref = e.what(); - nix_context_error(ctx); - } - ASSERT_EQ(ctx->last_err_code, NIX_ERR_UNKNOWN); - ASSERT_EQ(*ctx->last_err, err_msg_ref); - - nix_clear_err(ctx); - ASSERT_EQ(ctx->last_err_code, NIX_OK); -} - -TEST_F(nix_api_util_context, nix_set_err_msg) -{ - ASSERT_EQ(ctx->last_err_code, NIX_OK); - nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "unknown test error"); - ASSERT_EQ(ctx->last_err_code, NIX_ERR_UNKNOWN); - ASSERT_EQ(*ctx->last_err, "unknown test error"); -} - TEST(nix_api_util, nix_version_get) { ASSERT_EQ(std::string(nix_version_get()), PACKAGE_VERSION); @@ -61,17 +25,9 @@ struct MySettings : nix::Config MySettings mySettings; static nix::GlobalConfig::Register rs(&mySettings); -static auto createOwnedNixContext() -{ - return std::unique_ptr(nix_c_context_create(), {}); -} - TEST_F(nix_api_util_context, nix_setting_get) { - ASSERT_EQ(ctx->last_err_code, NIX_OK); + ASSERT_EQ(nix_err_code(ctx), NIX_OK); std::string setting_value; nix_err result = nix_setting_get(ctx, "invalid-key", OBSERVE_STRING(setting_value)); ASSERT_EQ(result, NIX_ERR_KEY); @@ -114,40 +70,6 @@ TEST_F(nix_api_util_context, nix_err_msg) ASSERT_EQ(sz, err_msg.size()); } -TEST_F(nix_api_util_context, nix_err_info_msg) -{ - std::string err_info; - - // no error - EXPECT_THROW(nix_err_info_msg(NULL, ctx, OBSERVE_STRING(err_info)), nix::Error); - - try { - throw nix::Error("testing error"); - } catch (...) { - nix_context_error(ctx); - } - auto new_ctx = createOwnedNixContext(); - nix_err_info_msg(new_ctx.get(), ctx, OBSERVE_STRING(err_info)); - ASSERT_STREQ("testing error", err_info.c_str()); -} - -TEST_F(nix_api_util_context, nix_err_name) -{ - std::string err_name; - - // no error - EXPECT_THROW(nix_err_name(NULL, ctx, OBSERVE_STRING(err_name)), nix::Error); - - try { - throw nix::Error("testing error"); - } catch (...) { - nix_context_error(ctx); - } - auto new_ctx = createOwnedNixContext(); - nix_err_name(new_ctx.get(), ctx, OBSERVE_STRING(err_name)); - ASSERT_EQ(std::string(err_name), "nix::Error"); -} - TEST_F(nix_api_util_context, nix_err_code) { ASSERT_EQ(nix_err_code(ctx), NIX_OK); diff --git a/src/libutil-tests/nix_api_util_internal.cc b/src/libutil-tests/nix_api_util_internal.cc new file mode 100644 index 00000000000..6fb0a623f66 --- /dev/null +++ b/src/libutil-tests/nix_api_util_internal.cc @@ -0,0 +1,85 @@ +#include "nix/util/config-global.hh" +#include "nix/util/args.hh" +#include "nix_api_util.h" +#include "nix_api_util_internal.h" +#include "nix/util/tests/nix_api_util.hh" +#include "nix/util/tests/string_callback.hh" + +#include + +#include + +#include "util-tests-config.hh" + +namespace nixC { + +TEST_F(nix_api_util_context, nix_context_error) +{ + std::string err_msg_ref; + try { + throw nix::Error("testing error"); + } catch (nix::Error & e) { + err_msg_ref = e.what(); + nix_context_error(ctx); + } + ASSERT_EQ(nix_err_code(ctx), NIX_ERR_NIX_ERROR); + ASSERT_EQ(ctx->name, "nix::Error"); + ASSERT_EQ(*ctx->last_err, err_msg_ref); + ASSERT_EQ(ctx->info->msg.str(), "testing error"); + + try { + throw std::runtime_error("testing exception"); + } catch (std::exception & e) { + err_msg_ref = e.what(); + nix_context_error(ctx); + } + ASSERT_EQ(nix_err_code(ctx), NIX_ERR_UNKNOWN); + ASSERT_EQ(*ctx->last_err, err_msg_ref); + + nix_clear_err(ctx); + ASSERT_EQ(nix_err_code(ctx), NIX_OK); +} + +TEST_F(nix_api_util_context, nix_set_err_msg) +{ + ASSERT_EQ(nix_err_code(ctx), NIX_OK); + nix_set_err_msg(ctx, NIX_ERR_UNKNOWN, "unknown test error"); + ASSERT_EQ(nix_err_code(ctx), NIX_ERR_UNKNOWN); + ASSERT_EQ(*ctx->last_err, "unknown test error"); +} + +TEST_F(nix_api_util_context, nix_err_info_msg) +{ + std::string err_info; + + // no error + EXPECT_THROW(nix_err_info_msg(NULL, ctx, OBSERVE_STRING(err_info)), nix::Error); + + try { + throw nix::Error("testing error"); + } catch (...) { + nix_context_error(ctx); + } + auto new_ctx = createOwnedNixContext(); + nix_err_info_msg(new_ctx.get(), ctx, OBSERVE_STRING(err_info)); + ASSERT_STREQ("testing error", err_info.c_str()); +} + +TEST_F(nix_api_util_context, nix_err_name) +{ + std::string err_name; + + // no error + EXPECT_THROW(nix_err_name(NULL, ctx, OBSERVE_STRING(err_name)), nix::Error); + + try { + throw nix::Error("testing error"); + } catch (...) { + nix_context_error(ctx); + } + auto new_ctx = createOwnedNixContext(); + nix_err_name(new_ctx.get(), ctx, OBSERVE_STRING(err_name)); + ASSERT_EQ(std::string(err_name), "nix::Error"); +} + +} // namespace nixC diff --git a/src/libutil-tests/package.nix b/src/libutil-tests/package.nix index c06de6894af..077d36a4d82 100644 --- a/src/libutil-tests/package.nix +++ b/src/libutil-tests/package.nix @@ -61,6 +61,7 @@ mkMesonExecutable (finalAttrs: { mkdir -p "$HOME" '' + '' + export ASAN_OPTIONS=abort_on_error=1:print_summary=1:detect_leaks=0 export _NIX_TEST_UNIT_DATA=${./data} ${stdenv.hostPlatform.emulator buildPackages} ${lib.getExe finalAttrs.finalPackage} touch $out diff --git a/src/libutil-tests/sort.cc b/src/libutil-tests/sort.cc index 8eee961c8cd..11d8e5938a2 100644 --- a/src/libutil-tests/sort.cc +++ b/src/libutil-tests/sort.cc @@ -102,14 +102,14 @@ struct RandomPeekSort : public ::testing::TestWithParam< void SetUp() override { - auto [maxSize, min, max, iterations] = GetParam(); + const auto & [maxSize, min, max, iterations] = GetParam(); urng_ = std::mt19937(GTEST_FLAG_GET(random_seed)); distribution_ = std::uniform_int_distribution(min, max); } auto regenerate() { - auto [maxSize, min, max, iterations] = GetParam(); + const auto & [maxSize, min, max, iterations] = GetParam(); std::size_t dataSize = std::uniform_int_distribution(0, maxSize)(urng_); data_.resize(dataSize); std::generate(data_.begin(), data_.end(), [&]() { return distribution_(urng_); }); @@ -118,7 +118,7 @@ struct RandomPeekSort : public ::testing::TestWithParam< TEST_P(RandomPeekSort, defaultComparator) { - auto [maxSize, min, max, iterations] = GetParam(); + const auto & [maxSize, min, max, iterations] = GetParam(); for (std::size_t i = 0; i < iterations; ++i) { regenerate(); @@ -132,7 +132,7 @@ TEST_P(RandomPeekSort, defaultComparator) TEST_P(RandomPeekSort, greater) { - auto [maxSize, min, max, iterations] = GetParam(); + const auto & [maxSize, min, max, iterations] = GetParam(); for (std::size_t i = 0; i < iterations; ++i) { regenerate(); @@ -146,7 +146,7 @@ TEST_P(RandomPeekSort, greater) TEST_P(RandomPeekSort, brokenComparator) { - auto [maxSize, min, max, iterations] = GetParam(); + const auto & [maxSize, min, max, iterations] = GetParam(); /* This is a pretty nice way of modeling a worst-case scenario for a broken comparator. If the sorting algorithm doesn't break in such case, then surely all deterministic @@ -170,7 +170,7 @@ TEST_P(RandomPeekSort, brokenComparator) TEST_P(RandomPeekSort, stability) { - auto [maxSize, min, max, iterations] = GetParam(); + const auto & [maxSize, min, max, iterations] = GetParam(); for (std::size_t i = 0; i < iterations; ++i) { regenerate(); diff --git a/src/libutil-tests/strings.cc b/src/libutil-tests/strings.cc index bf1f66025eb..bd740ce0cf4 100644 --- a/src/libutil-tests/strings.cc +++ b/src/libutil-tests/strings.cc @@ -2,6 +2,7 @@ #include #include "nix/util/strings.hh" +#include "nix/util/strings-inline.hh" #include "nix/util/error.hh" namespace nix { @@ -271,113 +272,122 @@ TEST(tokenizeString, tokenizeSepEmpty) * splitString * --------------------------------------------------------------------------*/ -TEST(splitString, empty) +using SplitStringTestContainerTypes = ::testing:: + Types, std::vector, std::list, std::list>; + +template +class splitStringTest : public ::testing::Test +{}; + +TYPED_TEST_SUITE(splitStringTest, SplitStringTestContainerTypes); + +TYPED_TEST(splitStringTest, empty) { - Strings expected = {""}; + TypeParam expected = {""}; - ASSERT_EQ(splitString("", " \t\n\r"), expected); + EXPECT_EQ(splitString("", " \t\n\r"), expected); } -TEST(splitString, oneSep) +TYPED_TEST(splitStringTest, oneSep) { - Strings expected = {"", ""}; + TypeParam expected = {"", ""}; - ASSERT_EQ(splitString(" ", " \t\n\r"), expected); + EXPECT_EQ(splitString(" ", " \t\n\r"), expected); } -TEST(splitString, twoSep) +TYPED_TEST(splitStringTest, twoSep) { - Strings expected = {"", "", ""}; + TypeParam expected = {"", "", ""}; - ASSERT_EQ(splitString(" \n", " \t\n\r"), expected); + EXPECT_EQ(splitString(" \n", " \t\n\r"), expected); } -TEST(splitString, tokenizeSpacesWithSpaces) +TYPED_TEST(splitStringTest, tokenizeSpacesWithSpaces) { auto s = "foo bar baz"; - Strings expected = {"foo", "bar", "baz"}; + TypeParam expected = {"foo", "bar", "baz"}; - ASSERT_EQ(splitString(s, " \t\n\r"), expected); + EXPECT_EQ(splitString(s, " \t\n\r"), expected); } -TEST(splitString, tokenizeTabsWithDefaults) +TYPED_TEST(splitStringTest, tokenizeTabsWithDefaults) { auto s = "foo\tbar\tbaz"; // Using it like this is weird, but shows the difference with tokenizeString, which also has this test - Strings expected = {"foo", "bar", "baz"}; + TypeParam expected = {"foo", "bar", "baz"}; - ASSERT_EQ(splitString(s, " \t\n\r"), expected); + EXPECT_EQ(splitString(s, " \t\n\r"), expected); } -TEST(splitString, tokenizeTabsSpacesWithDefaults) +TYPED_TEST(splitStringTest, tokenizeTabsSpacesWithDefaults) { auto s = "foo\t bar\t baz"; // Using it like this is weird, but shows the difference with tokenizeString, which also has this test - Strings expected = {"foo", "", "bar", "", "baz"}; + TypeParam expected = {"foo", "", "bar", "", "baz"}; - ASSERT_EQ(splitString(s, " \t\n\r"), expected); + EXPECT_EQ(splitString(s, " \t\n\r"), expected); } -TEST(splitString, tokenizeTabsSpacesNewlineWithDefaults) +TYPED_TEST(splitStringTest, tokenizeTabsSpacesNewlineWithDefaults) { auto s = "foo\t\n bar\t\n baz"; // Using it like this is weird, but shows the difference with tokenizeString, which also has this test - Strings expected = {"foo", "", "", "bar", "", "", "baz"}; + TypeParam expected = {"foo", "", "", "bar", "", "", "baz"}; - ASSERT_EQ(splitString(s, " \t\n\r"), expected); + EXPECT_EQ(splitString(s, " \t\n\r"), expected); } -TEST(splitString, tokenizeTabsSpacesNewlineRetWithDefaults) +TYPED_TEST(splitStringTest, tokenizeTabsSpacesNewlineRetWithDefaults) { auto s = "foo\t\n\r bar\t\n\r baz"; // Using it like this is weird, but shows the difference with tokenizeString, which also has this test - Strings expected = {"foo", "", "", "", "bar", "", "", "", "baz"}; + TypeParam expected = {"foo", "", "", "", "bar", "", "", "", "baz"}; - ASSERT_EQ(splitString(s, " \t\n\r"), expected); + EXPECT_EQ(splitString(s, " \t\n\r"), expected); auto s2 = "foo \t\n\r bar \t\n\r baz"; - Strings expected2 = {"foo", "", "", "", "", "bar", "", "", "", "", "baz"}; + TypeParam expected2 = {"foo", "", "", "", "", "bar", "", "", "", "", "baz"}; - ASSERT_EQ(splitString(s2, " \t\n\r"), expected2); + EXPECT_EQ(splitString(s2, " \t\n\r"), expected2); } -TEST(splitString, tokenizeWithCustomSep) +TYPED_TEST(splitStringTest, tokenizeWithCustomSep) { auto s = "foo\n,bar\n,baz\n"; - Strings expected = {"foo\n", "bar\n", "baz\n"}; + TypeParam expected = {"foo\n", "bar\n", "baz\n"}; - ASSERT_EQ(splitString(s, ","), expected); + EXPECT_EQ(splitString(s, ","), expected); } -TEST(splitString, tokenizeSepAtStart) +TYPED_TEST(splitStringTest, tokenizeSepAtStart) { auto s = ",foo,bar,baz"; - Strings expected = {"", "foo", "bar", "baz"}; + TypeParam expected = {"", "foo", "bar", "baz"}; - ASSERT_EQ(splitString(s, ","), expected); + EXPECT_EQ(splitString(s, ","), expected); } -TEST(splitString, tokenizeSepAtEnd) +TYPED_TEST(splitStringTest, tokenizeSepAtEnd) { auto s = "foo,bar,baz,"; - Strings expected = {"foo", "bar", "baz", ""}; + TypeParam expected = {"foo", "bar", "baz", ""}; - ASSERT_EQ(splitString(s, ","), expected); + EXPECT_EQ(splitString(s, ","), expected); } -TEST(splitString, tokenizeSepEmpty) +TYPED_TEST(splitStringTest, tokenizeSepEmpty) { auto s = "foo,,baz"; - Strings expected = {"foo", "", "baz"}; + TypeParam expected = {"foo", "", "baz"}; - ASSERT_EQ(splitString(s, ","), expected); + EXPECT_EQ(splitString(s, ","), expected); } // concatStringsSep sep . splitString sep = id if sep is 1 char -RC_GTEST_PROP(splitString, recoveredByConcatStringsSep, (const std::string & s)) +RC_GTEST_TYPED_FIXTURE_PROP(splitStringTest, recoveredByConcatStringsSep, (const std::string & s)) { - RC_ASSERT(concatStringsSep("/", splitString(s, "/")) == s); - RC_ASSERT(concatStringsSep("a", splitString(s, "a")) == s); + RC_ASSERT(concatStringsSep("/", splitString(s, "/")) == s); + RC_ASSERT(concatStringsSep("a", splitString(s, "a")) == s); } /* ---------------------------------------------------------------------------- diff --git a/src/libutil-tests/url.cc b/src/libutil-tests/url.cc index ae383eb654a..cd681609670 100644 --- a/src/libutil-tests/url.cc +++ b/src/libutil-tests/url.cc @@ -3,6 +3,8 @@ #include #include +#include + namespace nix { /* ----------- tests for url.hh --------------------------------------------------*/ @@ -10,82 +12,330 @@ namespace nix { using Authority = ParsedURL::Authority; using HostType = Authority::HostType; -TEST(parseURL, parsesSimpleHttpUrl) -{ - auto s = "http://www.example.org/file.tar.gz"; - auto parsed = parseURL(s); - - ParsedURL expected{ - .scheme = "http", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = "/file.tar.gz", - .query = (StringMap) {}, - .fragment = "", - }; - - ASSERT_EQ(parsed, expected); - ASSERT_EQ(s, parsed.to_string()); +struct FixGitURLParam +{ + std::string input; + std::string expected; + ParsedURL parsed; +}; + +std::ostream & operator<<(std::ostream & os, const FixGitURLParam & param) +{ + return os << "Input: \"" << param.input << "\", Expected: \"" << param.expected << "\""; +} + +class FixGitURLTestSuite : public ::testing::TestWithParam +{}; + +INSTANTIATE_TEST_SUITE_P( + FixGitURLs, + FixGitURLTestSuite, + ::testing::Values( + // https://github.com/NixOS/nix/issues/5958 + // Already proper URL with git+ssh + FixGitURLParam{ + .input = "git+ssh://user@domain:1234/path", + .expected = "git+ssh://user@domain:1234/path", + .parsed = + ParsedURL{ + .scheme = "git+ssh", + .authority = + ParsedURL::Authority{ + .host = "domain", + .user = "user", + .port = 1234, + }, + .path = {"", "path"}, + }, + }, + // SCP-like URL (rewritten to ssh://) + FixGitURLParam{ + .input = "git@github.com:owner/repo.git", + .expected = "ssh://git@github.com/owner/repo.git", + .parsed = + ParsedURL{ + .scheme = "ssh", + .authority = + ParsedURL::Authority{ + .host = "github.com", + .user = "git", + }, + .path = {"", "owner", "repo.git"}, + }, + }, + // Absolute path (becomes file:) + FixGitURLParam{ + .input = "/home/me/repo", + .expected = "file:///home/me/repo", + .parsed = + ParsedURL{ + .scheme = "file", + .authority = ParsedURL::Authority{}, + .path = {"", "home", "me", "repo"}, + }, + }, + // Already file: scheme + // NOTE: Git/SCP treat this as a `:`, so we are + // failing to "fix up" this case. + FixGitURLParam{ + .input = "file:/var/repos/x", + .expected = "file:/var/repos/x", + .parsed = + ParsedURL{ + .scheme = "file", + .authority = std::nullopt, + .path = {"", "var", "repos", "x"}, + }, + }, + // IPV6 test case + FixGitURLParam{ + .input = "user@[2001:db8:1::2]:/home/file", + .expected = "ssh://user@[2001:db8:1::2]//home/file", + .parsed = + ParsedURL{ + .scheme = "ssh", + .authority = + ParsedURL::Authority{ + .hostType = HostType::IPv6, + .host = "2001:db8:1::2", + .user = "user", + }, + .path = {"", "", "home", "file"}, + }, + })); + +TEST_P(FixGitURLTestSuite, parsesVariedGitUrls) +{ + auto & p = GetParam(); + const auto actual = fixGitURL(p.input); + EXPECT_EQ(actual, p.parsed); + EXPECT_EQ(actual.to_string(), p.expected); +} + +TEST(FixGitURLTestSuite, scpLikeNoUserParsesPoorly) +{ + // SCP-like URL (no user) + + // Cannot "to_string" this because has illegal path not starting + // with `/`. + EXPECT_EQ( + fixGitURL("github.com:owner/repo.git"), + (ParsedURL{ + .scheme = "file", + .authority = ParsedURL::Authority{}, + .path = {"github.com:owner", "repo.git"}, + })); +} + +TEST(FixGitURLTestSuite, properlyRejectFileURLWithAuthority) +{ + /* From the underlying `parseURL` validations. */ + EXPECT_THAT( + []() { fixGitURL("file://var/repos/x"); }, + ::testing::ThrowsMessage( + testing::HasSubstrIgnoreANSIMatcher("file:// URL 'file://var/repos/x' has unexpected authority 'var'"))); +} + +TEST(FixGitURLTestSuite, scpLikePathLeadingSlashParsesPoorly) +{ + // SCP-like URL (no user) + + // Cannot "to_string" this because has illegal path not starting + // with `/`. + EXPECT_EQ( + fixGitURL("github.com:/owner/repo.git"), + (ParsedURL{ + .scheme = "file", + .authority = ParsedURL::Authority{}, + .path = {"github.com:", "owner", "repo.git"}, + })); +} + +TEST(FixGitURLTestSuite, relativePathParsesPoorly) +{ + // Relative path (becomes file:// absolute) + + // Cannot "to_string" this because has illegal path not starting + // with `/`. + EXPECT_EQ( + fixGitURL("relative/repo"), + (ParsedURL{ + .scheme = "file", + .authority = + ParsedURL::Authority{ + .hostType = ParsedURL::Authority::HostType::Name, + .host = "", + }, + .path = {"relative", "repo"}})); +} + +struct ParseURLSuccessCase +{ + std::string_view input; + ParsedURL expected; +}; + +class ParseURLSuccess : public ::testing::TestWithParam +{}; + +INSTANTIATE_TEST_SUITE_P( + ParseURLSuccessCases, + ParseURLSuccess, + ::testing::Values( + ParseURLSuccessCase{ + .input = "http://www.example.org/file.tar.gz", + .expected = + ParsedURL{ + .scheme = "http", + .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .path = {"", "file.tar.gz"}, + .query = (StringMap) {}, + .fragment = "", + }, + }, + ParseURLSuccessCase{ + .input = "https://www.example.org/file.tar.gz", + .expected = + ParsedURL{ + .scheme = "https", + .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .path = {"", "file.tar.gz"}, + .query = (StringMap) {}, + .fragment = "", + }, + }, + ParseURLSuccessCase{ + .input = "https://www.example.org/file.tar.gz?download=fast&when=now#hello", + .expected = + ParsedURL{ + .scheme = "https", + .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .path = {"", "file.tar.gz"}, + .query = (StringMap) {{"download", "fast"}, {"when", "now"}}, + .fragment = "hello", + }, + }, + ParseURLSuccessCase{ + .input = "file+https://www.example.org/video.mp4", + .expected = + ParsedURL{ + .scheme = "file+https", + .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .path = {"", "video.mp4"}, + .query = (StringMap) {}, + .fragment = "", + }, + }, + ParseURLSuccessCase{ + .input = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello", + .expected = + ParsedURL{ + .scheme = "http", + .authority = Authority{.hostType = HostType::IPv4, .host = "127.0.0.1", .port = 8080}, + .path = {"", "file.tar.gz"}, + .query = (StringMap) {{"download", "fast"}, {"when", "now"}}, + .fragment = "hello", + }, + }, + ParseURLSuccessCase{ + .input = "http://[fe80::818c:da4d:8975:415c\%25enp0s25]:8080", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c\%enp0s25", .port = 8080}, + .path = {""}, + .query = (StringMap) {}, + .fragment = "", + }, + + }, + ParseURLSuccessCase{ + .input = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080", + .expected = + ParsedURL{ + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::IPv6, + .host = "2a02:8071:8192:c100:311d:192d:81ac:11ea", + .port = 8080, + }, + .path = {""}, + .query = (StringMap) {}, + .fragment = "", + }, + })); + +TEST_P(ParseURLSuccess, parsesAsExpected) +{ + auto & p = GetParam(); + const auto parsed = parseURL(p.input); + EXPECT_EQ(parsed, p.expected); +} + +TEST_P(ParseURLSuccess, toStringRoundTrips) +{ + auto & p = GetParam(); + const auto parsed = parseURL(p.input); + EXPECT_EQ(p.input, parsed.to_string()); +} + +TEST_P(ParseURLSuccess, makeSureFixGitURLDoesNotModify) +{ + auto & p = GetParam(); + const auto parsed = fixGitURL(std::string{p.input}); + EXPECT_EQ(p.input, parsed.to_string()); } -TEST(parseURL, parsesSimpleHttpsUrl) +TEST(parseURL, parsesSimpleHttpUrlWithComplexFragment) { - auto s = "https://www.example.org/file.tar.gz"; + auto s = "http://www.example.org/file.tar.gz?field=value#?foo=bar%23"; auto parsed = parseURL(s); ParsedURL expected{ - .scheme = "https", + .scheme = "http", .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = "/file.tar.gz", - .query = (StringMap) {}, - .fragment = "", + .path = {"", "file.tar.gz"}, + .query = (StringMap) {{"field", "value"}}, + .fragment = "?foo=bar#", }; ASSERT_EQ(parsed, expected); - ASSERT_EQ(s, parsed.to_string()); } -TEST(parseURL, parsesSimpleHttpUrlWithQueryAndFragment) +TEST(parseURL, rejectsAuthorityInUrlsWithFileTransportation) { - auto s = "https://www.example.org/file.tar.gz?download=fast&when=now#hello"; - auto parsed = parseURL(s); - - ParsedURL expected{ - .scheme = "https", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = "/file.tar.gz", - .query = (StringMap) {{"download", "fast"}, {"when", "now"}}, - .fragment = "hello", - }; - - ASSERT_EQ(parsed, expected); - ASSERT_EQ(s, parsed.to_string()); + EXPECT_THAT( + []() { parseURL("file://www.example.org/video.mp4"); }, + ::testing::ThrowsMessage( + testing::HasSubstrIgnoreANSIMatcher("has unexpected authority 'www.example.org'"))); } -TEST(parseURL, parsesSimpleHttpUrlWithComplexFragment) +TEST(parseURL, parseEmptyQueryParams) { - auto s = "http://www.example.org/file.tar.gz?field=value#?foo=bar%23"; + auto s = "http://127.0.0.1:8080/file.tar.gz?&&&&&"; auto parsed = parseURL(s); - - ParsedURL expected{ - .scheme = "http", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = "/file.tar.gz", - .query = (StringMap) {{"field", "value"}}, - .fragment = "?foo=bar#", - }; - - ASSERT_EQ(parsed, expected); + ASSERT_EQ(parsed.query, (StringMap) {}); } -TEST(parseURL, parsesFilePlusHttpsUrl) +TEST(parseURL, parseUserPassword) { - auto s = "file+https://www.example.org/video.mp4"; + auto s = "http://user:pass@www.example.org:8080/file.tar.gz"; auto parsed = parseURL(s); ParsedURL expected{ - .scheme = "file+https", - .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = "/video.mp4", + .scheme = "http", + .authority = + Authority{ + .hostType = HostType::Name, + .host = "www.example.org", + .user = "user", + .password = "pass", + .port = 8080, + }, + .path = {"", "file.tar.gz"}, .query = (StringMap) {}, .fragment = "", }; @@ -94,112 +344,85 @@ TEST(parseURL, parsesFilePlusHttpsUrl) ASSERT_EQ(s, parsed.to_string()); } -TEST(parseURL, rejectsAuthorityInUrlsWithFileTransportation) -{ - auto s = "file://www.example.org/video.mp4"; - ASSERT_THROW(parseURL(s), Error); -} - -TEST(parseURL, parseIPv4Address) +TEST(parseURL, parseFileURLWithQueryAndFragment) { - auto s = "http://127.0.0.1:8080/file.tar.gz?download=fast&when=now#hello"; + auto s = "file:///none/of//your/business"; auto parsed = parseURL(s); ParsedURL expected{ - .scheme = "http", - .authority = Authority{.hostType = HostType::IPv4, .host = "127.0.0.1", .port = 8080}, - .path = "/file.tar.gz", - .query = (StringMap) {{"download", "fast"}, {"when", "now"}}, - .fragment = "hello", + .scheme = "file", + .authority = Authority{}, + .path = {"", "none", "of", "", "your", "business"}, + .query = (StringMap) {}, + .fragment = "", }; + ASSERT_EQ(parsed.renderPath(), "/none/of//your/business"); ASSERT_EQ(parsed, expected); ASSERT_EQ(s, parsed.to_string()); } -TEST(parseURL, parseScopedRFC6874IPv6Address) +TEST(parseURL, parseFileURL) { - auto s = "http://[fe80::818c:da4d:8975:415c\%25enp0s25]:8080"; + auto s = "file:/none/of/your/business/"; auto parsed = parseURL(s); ParsedURL expected{ - .scheme = "http", - .authority = Authority{.hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c\%enp0s25", .port = 8080}, - .path = "", - .query = (StringMap) {}, - .fragment = "", + .scheme = "file", + .authority = std::nullopt, + .path = {"", "none", "of", "your", "business", ""}, }; + ASSERT_EQ(parsed.renderPath(), "/none/of/your/business/"); ASSERT_EQ(parsed, expected); ASSERT_EQ(s, parsed.to_string()); } -TEST(parseURL, parseIPv6Address) +TEST(parseURL, parseFileURLWithAuthority) { - auto s = "http://[2a02:8071:8192:c100:311d:192d:81ac:11ea]:8080"; + auto s = "file://///of/your/business//"; auto parsed = parseURL(s); ParsedURL expected{ - .scheme = "http", - .authority = - Authority{ - .hostType = HostType::IPv6, - .host = "2a02:8071:8192:c100:311d:192d:81ac:11ea", - .port = 8080, - }, - .path = "", - .query = (StringMap) {}, - .fragment = "", + .scheme = "file", + .authority = Authority{.host = ""}, + .path = {"", "", "", "of", "your", "business", "", ""}, }; + ASSERT_EQ(parsed.path, expected.path); + ASSERT_EQ(parsed.renderPath(), "///of/your/business//"); ASSERT_EQ(parsed, expected); ASSERT_EQ(s, parsed.to_string()); } -TEST(parseURL, parseEmptyQueryParams) -{ - auto s = "http://127.0.0.1:8080/file.tar.gz?&&&&&"; - auto parsed = parseURL(s); - ASSERT_EQ(parsed.query, (StringMap) {}); -} - -TEST(parseURL, parseUserPassword) +TEST(parseURL, parseFileURLNoLeadingSlash) { - auto s = "http://user:pass@www.example.org:8080/file.tar.gz"; + auto s = "file:none/of/your/business/"; auto parsed = parseURL(s); ParsedURL expected{ - .scheme = "http", - .authority = - Authority{ - .hostType = HostType::Name, - .host = "www.example.org", - .user = "user", - .password = "pass", - .port = 8080, - }, - .path = "/file.tar.gz", - .query = (StringMap) {}, - .fragment = "", + .scheme = "file", + .authority = std::nullopt, + .path = {"none", "of", "your", "business", ""}, }; + ASSERT_EQ(parsed.renderPath(), "none/of/your/business/"); ASSERT_EQ(parsed, expected); - ASSERT_EQ(s, parsed.to_string()); + ASSERT_EQ("file:none/of/your/business/", parsed.to_string()); } -TEST(parseURL, parseFileURLWithQueryAndFragment) +TEST(parseURL, parseHttpTrailingSlash) { - auto s = "file:///none/of//your/business"; + auto s = "http://example.com/"; auto parsed = parseURL(s); ParsedURL expected{ - .scheme = "file", - .authority = Authority{}, - .path = "/none/of//your/business", - .query = (StringMap) {}, - .fragment = "", + .scheme = "http", + .authority = Authority{.host = "example.com"}, + .path = {"", ""}, }; + ASSERT_EQ(parsed.renderPath(), "/"); ASSERT_EQ(parsed, expected); ASSERT_EQ(s, parsed.to_string()); } @@ -245,7 +468,7 @@ TEST(parseURL, parseFTPUrl) ParsedURL expected{ .scheme = "ftp", .authority = Authority{.hostType = HostType::Name, .host = "ftp.nixos.org"}, - .path = "/downloads/nixos.iso", + .path = {"", "downloads", "nixos.iso"}, .query = (StringMap) {}, .fragment = "", }; @@ -281,7 +504,7 @@ TEST(parseURL, parsesHttpUrlWithEmptyPort) ParsedURL expected{ .scheme = "http", .authority = Authority{.hostType = HostType::Name, .host = "www.example.org"}, - .path = "/file.tar.gz", + .path = {"", "file.tar.gz"}, .query = (StringMap) {{"foo", "bar"}}, .fragment = "", }; @@ -290,6 +513,208 @@ TEST(parseURL, parsesHttpUrlWithEmptyPort) ASSERT_EQ("http://www.example.org/file.tar.gz?foo=bar", parsed.to_string()); } +/* ---------------------------------------------------------------------------- + * parseURLRelative + * --------------------------------------------------------------------------*/ + +TEST(parseURLRelative, resolvesRelativePath) +{ + ParsedURL base = parseURL("http://example.org/dir/page.html"); + auto parsed = parseURLRelative("subdir/file.txt", base); + ParsedURL expected{ + .scheme = "http", + .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"}, + .path = {"", "dir", "subdir", "file.txt"}, + .query = {}, + .fragment = "", + }; + ASSERT_EQ(parsed, expected); +} + +TEST(parseURLRelative, baseUrlIpv6AddressWithoutZoneId) +{ + ParsedURL base = parseURL("http://[fe80::818c:da4d:8975:415c]/dir/page.html"); + auto parsed = parseURLRelative("subdir/file.txt", base); + ParsedURL expected{ + .scheme = "http", + .authority = ParsedURL::Authority{.hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c"}, + .path = {"", "dir", "subdir", "file.txt"}, + .query = {}, + .fragment = "", + }; + ASSERT_EQ(parsed, expected); +} + +TEST(parseURLRelative, resolvesRelativePathIpv6AddressWithZoneId) +{ + ParsedURL base = parseURL("http://[fe80::818c:da4d:8975:415c\%25enp0s25]:8080/dir/page.html"); + auto parsed = parseURLRelative("subdir/file2.txt", base); + ParsedURL expected{ + .scheme = "http", + .authority = Authority{.hostType = HostType::IPv6, .host = "fe80::818c:da4d:8975:415c\%enp0s25", .port = 8080}, + .path = {"", "dir", "subdir", "file2.txt"}, + .query = {}, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); +} + +TEST(parseURLRelative, resolvesRelativePathWithDot) +{ + ParsedURL base = parseURL("http://example.org/dir/page.html"); + auto parsed = parseURLRelative("./subdir/file.txt", base); + ParsedURL expected{ + .scheme = "http", + .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"}, + .path = {"", "dir", "subdir", "file.txt"}, + .query = {}, + .fragment = "", + }; + ASSERT_EQ(parsed, expected); +} + +TEST(parseURLRelative, resolvesParentDirectory) +{ + ParsedURL base = parseURL("http://example.org:234/dir/page.html"); + auto parsed = parseURLRelative("../up.txt", base); + ParsedURL expected{ + .scheme = "http", + .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org", .port = 234}, + .path = {"", "up.txt"}, + .query = {}, + .fragment = "", + }; + ASSERT_EQ(parsed, expected); +} + +TEST(parseURLRelative, resolvesParentDirectoryNotTrickedByEscapedSlash) +{ + ParsedURL base = parseURL("http://example.org:234/dir\%2Ffirst-trick/another-dir\%2Fsecond-trick/page.html"); + auto parsed = parseURLRelative("../up.txt", base); + ParsedURL expected{ + .scheme = "http", + .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org", .port = 234}, + .path = {"", "dir/first-trick", "up.txt"}, + .query = {}, + .fragment = "", + }; + ASSERT_EQ(parsed, expected); +} + +TEST(parseURLRelative, replacesPathWithAbsoluteRelative) +{ + ParsedURL base = parseURL("http://example.org/dir/page.html"); + auto parsed = parseURLRelative("/rooted.txt", base); + ParsedURL expected{ + .scheme = "http", + .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "example.org"}, + .path = {"", "rooted.txt"}, + .query = {}, + .fragment = "", + }; + ASSERT_EQ(parsed, expected); +} + +TEST(parseURLRelative, keepsQueryAndFragmentFromRelative) +{ + // But discard query params on base URL + ParsedURL base = parseURL("https://www.example.org/path/index.html?z=3"); + auto parsed = parseURLRelative("other.html?x=1&y=2#frag", base); + ParsedURL expected{ + .scheme = "https", + .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .path = {"", "path", "other.html"}, + .query = {{"x", "1"}, {"y", "2"}}, + .fragment = "frag", + }; + ASSERT_EQ(parsed, expected); +} + +TEST(parseURLRelative, absOverride) +{ + ParsedURL base = parseURL("http://example.org/path/page.html"); + std::string_view abs = "https://127.0.0.1.org/secure"; + auto parsed = parseURLRelative(abs, base); + auto parsedAbs = parseURL(abs); + ASSERT_EQ(parsed, parsedAbs); +} + +TEST(parseURLRelative, absOverrideWithZoneId) +{ + ParsedURL base = parseURL("http://example.org/path/page.html"); + std::string_view abs = "https://[fe80::818c:da4d:8975:415c\%25enp0s25]/secure?foo=bar"; + auto parsed = parseURLRelative(abs, base); + auto parsedAbs = parseURL(abs); + ASSERT_EQ(parsed, parsedAbs); +} + +TEST(parseURLRelative, bothWithoutAuthority) +{ + ParsedURL base = parseURL("mailto:mail-base@bar.baz?bcc=alice@asdf.com"); + std::string_view over = "mailto:mail-override@foo.bar?subject=url-testing"; + auto parsed = parseURLRelative(over, base); + auto parsedOverride = parseURL(over); + ASSERT_EQ(parsed, parsedOverride); +} + +TEST(parseURLRelative, emptyRelative) +{ + ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag"); + auto parsed = parseURLRelative("", base); + ParsedURL expected{ + .scheme = "https", + .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .path = {"", "path", "index.html"}, + .query = {{"a b", "5 6"}, {"x y", "34"}}, + .fragment = "", + }; + EXPECT_EQ(base.fragment, "frag"); + EXPECT_EQ(parsed, expected); +} + +TEST(parseURLRelative, fragmentRelative) +{ + ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag"); + auto parsed = parseURLRelative("#frag2", base); + ParsedURL expected{ + .scheme = "https", + .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .path = {"", "path", "index.html"}, + .query = {{"a b", "5 6"}, {"x y", "34"}}, + .fragment = "frag2", + }; + EXPECT_EQ(parsed, expected); +} + +TEST(parseURLRelative, queryRelative) +{ + ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag"); + auto parsed = parseURLRelative("?asdf\%20qwer=1\%202\%203", base); + ParsedURL expected{ + .scheme = "https", + .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .path = {"", "path", "index.html"}, + .query = {{"asdf qwer", "1 2 3"}}, + .fragment = "", + }; + EXPECT_EQ(parsed, expected); +} + +TEST(parseURLRelative, queryFragmentRelative) +{ + ParsedURL base = parseURL("https://www.example.org/path/index.html?a\%20b=5\%206&x\%20y=34#frag"); + auto parsed = parseURLRelative("?asdf\%20qwer=1\%202\%203#frag2", base); + ParsedURL expected{ + .scheme = "https", + .authority = ParsedURL::Authority{.hostType = HostType::Name, .host = "www.example.org"}, + .path = {"", "path", "index.html"}, + .query = {{"asdf qwer", "1 2 3"}}, + .fragment = "frag2", + }; + EXPECT_EQ(parsed, expected); +} + /* ---------------------------------------------------------------------------- * decodeQuery * --------------------------------------------------------------------------*/ @@ -399,7 +824,134 @@ TEST(percentEncode, yen) ASSERT_EQ(percentDecode(e), s); } +TEST(parseURL, gitlabNamespacedProjectUrls) +{ + // Test GitLab URL patterns with namespaced projects + // These should preserve %2F encoding in the path + auto s = "https://gitlab.example.com/api/v4/projects/group%2Fsubgroup%2Fproject/repository/archive.tar.gz"; + auto parsed = parseURL(s); + + ParsedURL expected{ + .scheme = "https", + .authority = Authority{.hostType = HostType::Name, .host = "gitlab.example.com"}, + .path = {"", "api", "v4", "projects", "group/subgroup/project", "repository", "archive.tar.gz"}, + .query = {}, + .fragment = "", + }; + + ASSERT_EQ(parsed, expected); + ASSERT_EQ(s, parsed.to_string()); +} + +/* ---------------------------------------------------------------------------- + * pathSegments + * --------------------------------------------------------------------------*/ + +struct ParsedURLPathSegmentsTestCase +{ + std::string url; + std::vector segments; + std::string path; + bool skipEmpty; + std::string description; +}; + +class ParsedURLPathSegmentsTest : public ::testing::TestWithParam +{}; + +TEST_P(ParsedURLPathSegmentsTest, segmentsAreCorrect) +{ + const auto & testCase = GetParam(); + auto segments = parseURL(testCase.url).pathSegments(/*skipEmpty=*/testCase.skipEmpty) + | std::ranges::to(); + EXPECT_EQ(segments, testCase.segments); + EXPECT_EQ(encodeUrlPath(segments), testCase.path); +} + +TEST_P(ParsedURLPathSegmentsTest, to_string) +{ + const auto & testCase = GetParam(); + EXPECT_EQ(testCase.url, parseURL(testCase.url).to_string()); +} + +INSTANTIATE_TEST_SUITE_P( + ParsedURL, + ParsedURLPathSegmentsTest, + ::testing::Values( + ParsedURLPathSegmentsTestCase{ + .url = "scheme:", + .segments = {""}, + .path = "", + .skipEmpty = false, + .description = "no_authority_empty_path", + }, + ParsedURLPathSegmentsTestCase{ + .url = "scheme://", + .segments = {""}, + .path = "", + .skipEmpty = false, + .description = "empty_authority_empty_path", + }, + ParsedURLPathSegmentsTestCase{ + .url = "path:/", + .segments = {"", ""}, + .path = "/", + .skipEmpty = false, + .description = "empty_authority_root_path", + }, + ParsedURLPathSegmentsTestCase{ + .url = "scheme:///", + .segments = {"", ""}, + .path = "/", + .skipEmpty = false, + .description = "empty_authority_empty_path_trailing", + }, + ParsedURLPathSegmentsTestCase{ + .url = "scheme://example.com/", + .segments = {"", ""}, + .path = "/", + .skipEmpty = false, + .description = "non_empty_authority_empty_path", + }, + ParsedURLPathSegmentsTestCase{ + .url = "scheme://example.com//", + .segments = {"", "", ""}, + .path = "//", + .skipEmpty = false, + .description = "non_empty_authority_non_empty_path", + }, + ParsedURLPathSegmentsTestCase{ + .url = "scheme://example.com///path///with//strange/empty///segments////", + .segments = {"path", "with", "strange", "empty", "segments"}, + .path = "path/with/strange/empty/segments", + .skipEmpty = true, + .description = "skip_all_empty_segments_with_authority", + }, + ParsedURLPathSegmentsTestCase{ + .url = "scheme://example.com///lots///empty///", + .segments = {"", "", "", "lots", "", "", "empty", "", "", ""}, + .path = "///lots///empty///", + .skipEmpty = false, + .description = "empty_segments_with_authority", + }, + ParsedURLPathSegmentsTestCase{ + .url = "scheme:/path///with//strange/empty///segments////", + .segments = {"path", "with", "strange", "empty", "segments"}, + .path = "path/with/strange/empty/segments", + .skipEmpty = true, + .description = "skip_all_empty_segments_no_authority_starts_with_slash", + }, + ParsedURLPathSegmentsTestCase{ + .url = "scheme:path///with//strange/empty///segments////", + .segments = {"path", "with", "strange", "empty", "segments"}, + .path = "path/with/strange/empty/segments", + .skipEmpty = true, + .description = "skip_all_empty_segments_no_authority_doesnt_start_with_slash", + }), + [](const auto & info) { return info.param.description; }); + TEST(nix, isValidSchemeName) + { ASSERT_TRUE(isValidSchemeName("http")); ASSERT_TRUE(isValidSchemeName("https")); diff --git a/src/libutil/archive.cc b/src/libutil/archive.cc index b978ac4dbff..b8fef9ef3d7 100644 --- a/src/libutil/archive.cc +++ b/src/libutil/archive.cc @@ -132,6 +132,11 @@ static void parseContents(CreateRegularFileSink & sink, Source & source) sink.preallocateContents(size); + if (sink.skipContents) { + source.skip(size + (size % 8 ? 8 - (size % 8) : 0)); + return; + } + uint64_t left = size; std::array buf; @@ -166,7 +171,7 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath auto expectTag = [&](std::string_view expected) { auto tag = getString(); if (tag != expected) - throw badArchive("expected tag '%s', got '%s'", expected, tag); + throw badArchive("expected tag '%s', got '%s'", expected, tag.substr(0, 1024)); }; expectTag("("); @@ -187,8 +192,10 @@ static void parse(FileSystemObjectSink & sink, Source & source, const CanonPath tag = getString(); } - if (tag == "contents") - parseContents(crf, source); + if (tag != "contents") + throw badArchive("expected tag 'contents', got '%s'", tag); + + parseContents(crf, source); expectTag(")"); }); diff --git a/src/libutil/args.cc b/src/libutil/args.cc index f4309473b20..5faf9dd43ea 100644 --- a/src/libutil/args.cc +++ b/src/libutil/args.cc @@ -318,6 +318,7 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) } catch (SystemError &) { } } + for (auto pos = cmdline.begin(); pos != cmdline.end();) { auto arg = *pos; @@ -354,6 +355,9 @@ void RootArgs::parseCmdline(const Strings & _cmdline, bool allowShebang) processArgs(pendingArgs, true); + if (!completions) + checkArgs(); + initialFlagsProcessed(); /* Now that we are done parsing, make sure that any experimental @@ -384,7 +388,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) auto & rootArgs = getRoot(); - auto process = [&](const std::string & name, const Flag & flag) -> bool { + auto process = [&](const std::string & name, Flag & flag) -> bool { ++pos; if (auto & f = flag.experimentalFeature) @@ -413,6 +417,7 @@ bool Args::processFlag(Strings::iterator & pos, Strings::iterator end) } if (!anyCompleted) flag.handler.fun(std::move(args)); + flag.timesUsed++; return true; }; @@ -504,6 +509,14 @@ bool Args::processArgs(const Strings & args, bool finish) return res; } +void Args::checkArgs() +{ + for (auto & [name, flag] : longFlags) { + if (flag->required && flag->timesUsed == 0) + throw UsageError("required argument '%s' is missing", "--" + name); + } +} + nlohmann::json Args::toJSON() { auto flags = nlohmann::json::object(); @@ -594,7 +607,7 @@ Strings argvToStrings(int argc, char ** argv) std::optional Command::experimentalFeature() { - return {Xp::NixCommand}; + return {}; } MultiCommand::MultiCommand(std::string_view commandName, const Commands & commands_) @@ -643,6 +656,13 @@ bool MultiCommand::processArgs(const Strings & args, bool finish) return Args::processArgs(args, finish); } +void MultiCommand::checkArgs() +{ + Args::checkArgs(); + if (command) + command->second->checkArgs(); +} + nlohmann::json MultiCommand::toJSON() { auto cmds = nlohmann::json::object(); diff --git a/src/libutil/configuration.cc b/src/libutil/configuration.cc index dc9d91f63b9..ca3c08cd9b3 100644 --- a/src/libutil/configuration.cc +++ b/src/libutil/configuration.cc @@ -375,11 +375,11 @@ std::set BaseSetting>::parse( { std::set res; for (auto & s : tokenizeString(str)) { - if (auto thisXpFeature = parseExperimentalFeature(s); thisXpFeature) { + if (auto thisXpFeature = parseExperimentalFeature(s)) res.insert(thisXpFeature.value()); - if (thisXpFeature.value() == Xp::Flakes) - res.insert(Xp::FetchTree); - } else + else if (stabilizedFeatures.count(s)) + debug("experimental feature '%s' is now stable", s); + else warn("unknown experimental feature '%s'", s); } return res; diff --git a/src/libutil/error.cc b/src/libutil/error.cc index b50b1f3be68..35e42823ce6 100644 --- a/src/libutil/error.cc +++ b/src/libutil/error.cc @@ -6,6 +6,7 @@ #include "nix/util/terminal.hh" #include "nix/util/position.hh" +#include #include #include #include "nix/util/serialise.hh" @@ -436,13 +437,19 @@ void panic(std::string_view msg) writeErr("\n\n" ANSI_RED "terminating due to unexpected unrecoverable internal error: " ANSI_NORMAL); writeErr(msg); writeErr("\n"); - abort(); + std::terminate(); } -void panic(const char * file, int line, const char * func) +void unreachable(std::source_location loc) { char buf[512]; - int n = snprintf(buf, sizeof(buf), "Unexpected condition in %s at %s:%d", func, file, line); + int n = snprintf( + buf, + sizeof(buf), + "Unexpected condition in %s at %s:%" PRIuLEAST32, + loc.function_name(), + loc.file_name(), + loc.line()); if (n < 0) panic("Unexpected condition and could not format error message"); panic(std::string_view(buf, std::min(static_cast(sizeof(buf)), n))); diff --git a/src/libutil/experimental-features.cc b/src/libutil/experimental-features.cc index 60d6bf74de0..b9034821733 100644 --- a/src/libutil/experimental-features.cc +++ b/src/libutil/experimental-features.cc @@ -16,7 +16,7 @@ struct ExperimentalFeatureDetails /** * If two different PRs both add an experimental feature, and we just - * used a number for this, we *woudln't* get merge conflict and the + * used a number for this, we *wouldn't* get merge conflict and the * counter will be incremented once instead of twice, causing a build * failure. * @@ -24,7 +24,7 @@ struct ExperimentalFeatureDetails * feature, we either have no issue at all if few features are not added * at the end of the list, or a proper merge conflict if they are. */ -constexpr size_t numXpFeatures = 1 + static_cast(Xp::BLAKE3Hashes); +constexpr size_t numXpFeatures = 1 + static_cast(Xp::ParallelEval); constexpr std::array xpFeatureDetails = {{ { @@ -70,19 +70,12 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/42", }, - { - .tag = Xp::Flakes, - .name = "flakes", - .description = R"( - Enable flakes. See the manual entry for [`nix - flake`](@docroot@/command-ref/new-cli/nix3-flake.md) for details. - )", - .trackingUrl = "https://github.com/NixOS/nix/milestone/27", - }, { .tag = Xp::FetchTree, .name = "fetch-tree", .description = R"( + *Enabled for Determinate Nix Installer users since 2.24* + Enable the use of the [`fetchTree`](@docroot@/language/builtins.md#builtins-fetchTree) built-in function in the Nix language. `fetchTree` exposes a generic interface for fetching remote file system trees from different types of remote sources. @@ -93,15 +86,6 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/31", }, - { - .tag = Xp::NixCommand, - .name = "nix-command", - .description = R"( - Enable the new `nix` subcommands. See the manual on - [`nix`](@docroot@/command-ref/new-cli/nix.md) for details. - )", - .trackingUrl = "https://github.com/NixOS/nix/milestone/28", - }, { .tag = Xp::GitHashing, .name = "git-hashing", @@ -170,7 +154,7 @@ constexpr std::array xpFeatureDetails "http://foo" ``` - But enabling this experimental feature will cause the Nix parser to + But enabling this experimental feature causes the Nix parser to throw an error when encountering a URL literal: ``` @@ -304,6 +288,14 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "https://github.com/NixOS/nix/milestone/55", }, + { + .tag = Xp::ExternalBuilders, + .name = "external-builders", + .description = R"( + Enables support for external builders / sandbox providers. + )", + .trackingUrl = "", + }, { .tag = Xp::BLAKE3Hashes, .name = "blake3-hashes", @@ -312,6 +304,22 @@ constexpr std::array xpFeatureDetails )", .trackingUrl = "", }, + { + .tag = Xp::BuildTimeFetchTree, + .name = "build-time-fetch-tree", + .description = R"( + Enable the built-in derivation `builtin:fetch-tree`, as well as the flake input attribute `buildTime`. + )", + .trackingUrl = "", + }, + { + .tag = Xp::ParallelEval, + .name = "parallel-eval", + .description = R"( + Enable built-in functions for parallel evaluation. + )", + .trackingUrl = "", + }, }}; static_assert( @@ -323,6 +331,12 @@ static_assert( }(), "array order does not match enum tag order"); +/** + * A set of previously experimental features that are now considered + * stable. We don't warn if users have these in `experimental-features`. + */ +std::set stabilizedFeatures{"flakes", "nix-command"}; + const std::optional parseExperimentalFeature(const std::string_view & name) { using ReverseXpMap = std::map; diff --git a/src/libutil/fs-sink.cc b/src/libutil/fs-sink.cc index 6efd5e0c7e2..45ef57a9f5b 100644 --- a/src/libutil/fs-sink.cc +++ b/src/libutil/fs-sink.cc @@ -196,6 +196,8 @@ void NullFileSystemObjectSink::createRegularFile( void isExecutable() override {} } crf; + crf.skipContents = true; + // Even though `NullFileSystemObjectSink` doesn't do anything, it's important // that we call the function, to e.g. advance the parser using this // sink. diff --git a/src/libutil/hash.cc b/src/libutil/hash.cc index e469957a0d5..d306360ef79 100644 --- a/src/libutil/hash.cc +++ b/src/libutil/hash.cc @@ -18,6 +18,8 @@ #include #include +#include + #include namespace nix { @@ -99,22 +101,38 @@ struct DecodeNamePair } // namespace +static DecodeNamePair baseExplicit(HashFormat format) +{ + switch (format) { + case HashFormat::Base16: + return {base16::decode, "base16"}; + case HashFormat::Nix32: + return {BaseNix32::decode, "nix32"}; + case HashFormat::Base64: + return {base64::decode, "Base64"}; + case HashFormat::SRI: + break; + } + unreachable(); +} + /** * Given the expected size of the message once decoded it, figure out * which encoding we are using by looking at the size of the encoded * message. */ -static DecodeNamePair baseFromSize(std::string_view rest, HashAlgorithm algo) +static HashFormat baseFromSize(std::string_view rest, HashAlgorithm algo) { auto hashSize = regularHashSize(algo); + if (rest.size() == base16::encodedLength(hashSize)) - return {base16::decode, "base16"}; + return HashFormat::Base16; if (rest.size() == BaseNix32::encodedLength(hashSize)) - return {BaseNix32::decode, "nix32"}; + return HashFormat::Nix32; if (rest.size() == base64::encodedLength(hashSize)) - return {base64::decode, "Base64"}; + return HashFormat::Base64; throw BadHash("hash '%s' has wrong length for hash algorithm '%s'", rest, printHashAlgo(algo)); } @@ -135,7 +153,8 @@ static Hash parseLowLevel(std::string_view rest, HashAlgorithm algo, DecodeNameP e.addTrace({}, "While decoding hash '%s'", rest); } if (d.size() != res.hashSize) - throw BadHash("invalid %s hash '%s' %d %d", pair.encodingName, rest); + throw BadHash( + "invalid %s hash '%s', length %d != expected length %d", pair.encodingName, rest, d.size(), res.hashSize); assert(res.hashSize); memcpy(res.hash, d.data(), res.hashSize); @@ -189,7 +208,7 @@ static Hash parseAnyHelper(std::string_view rest, auto resolveAlgo) } else { /* Otherwise, decide via the length of the hash (for the given algorithm) what base encoding it is. */ - return baseFromSize(rest, algo); + return baseExplicit(baseFromSize(rest, algo)); } }(); @@ -224,7 +243,12 @@ Hash Hash::parseAny(std::string_view original, std::optional optA Hash Hash::parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo) { - return parseLowLevel(s, algo, baseFromSize(s, algo)); + return parseExplicitFormatUnprefixed(s, algo, baseFromSize(s, algo)); +} + +Hash Hash::parseExplicitFormatUnprefixed(std::string_view s, HashAlgorithm algo, HashFormat format) +{ + return parseLowLevel(s, algo, baseExplicit(format)); } Hash Hash::random(HashAlgorithm algo) @@ -468,4 +492,12 @@ std::string_view printHashAlgo(HashAlgorithm ha) } } +void to_json(nlohmann::json & json, const Hash & hash) +{ + json = nlohmann::json::object({ + {"algo", printHashAlgo(hash.algo)}, + {"base16", hash.to_string(HashFormat::Base16, false)}, + }); +} + } // namespace nix diff --git a/src/libutil/include/nix/util/args.hh b/src/libutil/include/nix/util/args.hh index 443db445f2a..99f6e23e8e9 100644 --- a/src/libutil/include/nix/util/args.hh +++ b/src/libutil/include/nix/util/args.hh @@ -202,8 +202,12 @@ public: Strings labels; Handler handler; CompleterClosure completer; + bool required = false; std::optional experimentalFeature; + + // FIXME: this should be private, but that breaks designated initializers. + size_t timesUsed = 0; }; protected: @@ -283,6 +287,8 @@ protected: StringSet hiddenCategories; + virtual void checkArgs(); + /** * Called after all command line flags before the first non-flag * argument (if any) have been processed. @@ -428,6 +434,8 @@ public: protected: std::string commandName = ""; bool aliasUsed = false; + + void checkArgs() override; }; Strings argvToStrings(int argc, char ** argv); diff --git a/src/libutil/include/nix/util/canon-path.hh b/src/libutil/include/nix/util/canon-path.hh index cb8b4325d0b..dd07929b4f4 100644 --- a/src/libutil/include/nix/util/canon-path.hh +++ b/src/libutil/include/nix/util/canon-path.hh @@ -8,6 +8,8 @@ #include #include +#include + namespace nix { /** @@ -258,18 +260,26 @@ public: */ std::string makeRelative(const CanonPath & path) const; - friend class std::hash; + friend std::size_t hash_value(const CanonPath &); }; std::ostream & operator<<(std::ostream & stream, const CanonPath & path); +inline std::size_t hash_value(const CanonPath & path) +{ + boost::hash hasher; + return hasher(path.path); +} + } // namespace nix template<> struct std::hash { - std::size_t operator()(const nix::CanonPath & s) const noexcept + using is_avalanching = std::true_type; + + std::size_t operator()(const nix::CanonPath & path) const noexcept { - return std::hash{}(s.path); + return nix::hash_value(path); } }; diff --git a/src/libutil/include/nix/util/configuration.hh b/src/libutil/include/nix/util/configuration.hh index 65391721c86..73e3fb81a52 100644 --- a/src/libutil/include/nix/util/configuration.hh +++ b/src/libutil/include/nix/util/configuration.hh @@ -444,7 +444,7 @@ struct ExperimentalFeatureSettings : Config Example: ``` - experimental-features = nix-command flakes + experimental-features = ca-derivations ``` The following experimental features are available: diff --git a/src/libutil/include/nix/util/current-process.hh b/src/libutil/include/nix/util/current-process.hh index 36449313797..c4a95258174 100644 --- a/src/libutil/include/nix/util/current-process.hh +++ b/src/libutil/include/nix/util/current-process.hh @@ -2,6 +2,7 @@ ///@file #include +#include #ifndef _WIN32 # include @@ -11,6 +12,11 @@ namespace nix { +/** + * Get the current process's user space CPU time. + */ +std::chrono::microseconds getCpuUserTime(); + /** * If cgroups are active, attempt to calculate the number of CPUs available. * If cgroups are unavailable or if cpu.max is set to "max", return 0. diff --git a/src/libutil/include/nix/util/error.hh b/src/libutil/include/nix/util/error.hh index bd21e02d3ce..e564ca5b9cc 100644 --- a/src/libutil/include/nix/util/error.hh +++ b/src/libutil/include/nix/util/error.hh @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -299,23 +300,16 @@ using NativeSysError = void throwExceptionSelfCheck(); /** - * Print a message and abort(). + * Print a message and std::terminate(). */ [[noreturn]] void panic(std::string_view msg); /** - * Print a basic error message with source position and abort(). - * Use the unreachable() macro to call this. - */ -[[noreturn]] -void panic(const char * file, int line, const char * func); - -/** - * Print a basic error message with source position and abort(). + * Print a basic error message with source position and std::terminate(). * * @note: This assumes that the logger is operational */ -#define unreachable() (::nix::panic(__FILE__, __LINE__, __func__)) +[[gnu::noinline, gnu::cold, noreturn]] void unreachable(std::source_location loc = std::source_location::current()); } // namespace nix diff --git a/src/libutil/include/nix/util/experimental-features.hh b/src/libutil/include/nix/util/experimental-features.hh index 1eabc34619b..7b0592b8ccb 100644 --- a/src/libutil/include/nix/util/experimental-features.hh +++ b/src/libutil/include/nix/util/experimental-features.hh @@ -3,6 +3,7 @@ #include "nix/util/error.hh" #include "nix/util/types.hh" +#include "nix/util/json-non-null.hh" #include @@ -18,9 +19,7 @@ namespace nix { enum struct ExperimentalFeature { CaDerivations, ImpureDerivations, - Flakes, FetchTree, - NixCommand, GitHashing, RecursiveNix, NoUrlLiterals, @@ -36,9 +35,14 @@ enum struct ExperimentalFeature { MountedSSHStore, VerifiedFetches, PipeOperators, + ExternalBuilders, BLAKE3Hashes, + BuildTimeFetchTree, + ParallelEval, }; +extern std::set stabilizedFeatures; + /** * Just because writing `ExperimentalFeature::CaDerivations` is way too long */ @@ -89,6 +93,13 @@ public: MissingExperimentalFeature(ExperimentalFeature missingFeature); }; +/** + * `ExperimentalFeature` is always rendered as a string. + */ +template<> +struct json_avoids_null : std::true_type +{}; + /** * Semi-magic conversion to and from json. * See the nlohmann/json readme for more details. diff --git a/src/libutil/include/nix/util/forwarding-source-accessor.hh b/src/libutil/include/nix/util/forwarding-source-accessor.hh new file mode 100644 index 00000000000..02474a3a7f3 --- /dev/null +++ b/src/libutil/include/nix/util/forwarding-source-accessor.hh @@ -0,0 +1,57 @@ +#pragma once + +#include "source-accessor.hh" + +namespace nix { + +/** + * A source accessor that just forwards every operation to another + * accessor. This is not useful in itself but can be used as a + * superclass for accessors that do change some operations. + */ +struct ForwardingSourceAccessor : SourceAccessor +{ + ref next; + + ForwardingSourceAccessor(ref next) + : next(next) + { + } + + std::string readFile(const CanonPath & path) override + { + return next->readFile(path); + } + + void readFile(const CanonPath & path, Sink & sink, std::function sizeCallback) override + { + next->readFile(path, sink, sizeCallback); + } + + std::optional maybeLstat(const CanonPath & path) override + { + return next->maybeLstat(path); + } + + DirEntries readDirectory(const CanonPath & path) override + { + return next->readDirectory(path); + } + + std::string readLink(const CanonPath & path) override + { + return next->readLink(path); + } + + std::string showPath(const CanonPath & path) override + { + return next->showPath(path); + } + + std::optional getPhysicalPath(const CanonPath & path) override + { + return next->getPhysicalPath(path); + } +}; + +} // namespace nix diff --git a/src/libutil/include/nix/util/fs-sink.hh b/src/libutil/include/nix/util/fs-sink.hh index f96fe3ef954..bd2db7f53e6 100644 --- a/src/libutil/include/nix/util/fs-sink.hh +++ b/src/libutil/include/nix/util/fs-sink.hh @@ -14,6 +14,14 @@ namespace nix { */ struct CreateRegularFileSink : Sink { + /** + * If set to true, the sink will not be called with the contents + * of the file. `preallocateContents()` will still be called to + * convey the file size. Useful for sinks that want to efficiently + * discard the contents of the file. + */ + bool skipContents = false; + virtual void isExecutable() = 0; /** diff --git a/src/libutil/include/nix/util/hash.hh b/src/libutil/include/nix/util/hash.hh index f4d137bd0ce..81e622b2217 100644 --- a/src/libutil/include/nix/util/hash.hh +++ b/src/libutil/include/nix/util/hash.hh @@ -6,6 +6,8 @@ #include "nix/util/serialise.hh" #include "nix/util/file-system.hh" +#include + namespace nix { MakeError(BadHash, Error); @@ -90,6 +92,15 @@ struct Hash */ static Hash parseNonSRIUnprefixed(std::string_view s, HashAlgorithm algo); + /** + * Like `parseNonSRIUnprefixed`, but the hash format has been + * explicitly given. + * + * @param explicitFormat cannot be SRI, but must be one of the + * "bases". + */ + static Hash parseExplicitFormatUnprefixed(std::string_view s, HashAlgorithm algo, HashFormat explicitFormat); + static Hash parseSRI(std::string_view original); public: @@ -191,6 +202,11 @@ std::optional parseHashAlgoOpt(std::string_view s); */ std::string_view printHashAlgo(HashAlgorithm ha); +/** + * Write a JSON serialisation of the format `{"algo":"","base16":""}`. + */ +void to_json(nlohmann::json & json, const Hash & hash); + struct AbstractHashSink : virtual Sink { virtual HashResult finish() = 0; diff --git a/src/libutil/include/nix/util/json-impls.hh b/src/libutil/include/nix/util/json-impls.hh index 8a619831327..751fc410f56 100644 --- a/src/libutil/include/nix/util/json-impls.hh +++ b/src/libutil/include/nix/util/json-impls.hh @@ -4,13 +4,13 @@ #include // Following https://github.com/nlohmann/json#how-can-i-use-get-for-non-default-constructiblenon-copyable-types -#define JSON_IMPL(TYPE) \ - namespace nlohmann { \ - using namespace nix; \ - template<> \ - struct adl_serializer \ - { \ - static TYPE from_json(const json & json); \ - static void to_json(json & json, TYPE t); \ - }; \ +#define JSON_IMPL(TYPE) \ + namespace nlohmann { \ + using namespace nix; \ + template<> \ + struct adl_serializer \ + { \ + static TYPE from_json(const json & json); \ + static void to_json(json & json, const TYPE & t); \ + }; \ } diff --git a/src/libutil/include/nix/util/json-non-null.hh b/src/libutil/include/nix/util/json-non-null.hh new file mode 100644 index 00000000000..6bacce58fa5 --- /dev/null +++ b/src/libutil/include/nix/util/json-non-null.hh @@ -0,0 +1,55 @@ +#pragma once +///@file + +#include +#include +#include +#include +#include + +namespace nix { + +/** + * For `adl_serializer>` below, we need to track what + * types are not already using `null`. Only for them can we use `null` + * to represent `std::nullopt`. + */ +template +struct json_avoids_null; + +/** + * Handle numbers in default impl + */ +template +struct json_avoids_null : std::bool_constant::value> +{}; + +template<> +struct json_avoids_null : std::false_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template<> +struct json_avoids_null : std::true_type +{}; + +template +struct json_avoids_null> : std::true_type +{}; + +template +struct json_avoids_null> : std::true_type +{}; + +template +struct json_avoids_null> : std::true_type +{}; + +template +struct json_avoids_null> : std::true_type +{}; + +} // namespace nix diff --git a/src/libutil/include/nix/util/json-utils.hh b/src/libutil/include/nix/util/json-utils.hh index 20c50f9579a..4b5fb4b21be 100644 --- a/src/libutil/include/nix/util/json-utils.hh +++ b/src/libutil/include/nix/util/json-utils.hh @@ -6,6 +6,7 @@ #include "nix/util/error.hh" #include "nix/util/types.hh" +#include "nix/util/json-non-null.hh" namespace nix { @@ -59,56 +60,6 @@ Strings getStringList(const nlohmann::json & value); StringMap getStringMap(const nlohmann::json & value); StringSet getStringSet(const nlohmann::json & value); -/** - * For `adl_serializer>` below, we need to track what - * types are not already using `null`. Only for them can we use `null` - * to represent `std::nullopt`. - */ -template -struct json_avoids_null; - -/** - * Handle numbers in default impl - */ -template -struct json_avoids_null : std::bool_constant::value> -{}; - -template<> -struct json_avoids_null : std::false_type -{}; - -template<> -struct json_avoids_null : std::true_type -{}; - -template<> -struct json_avoids_null : std::true_type -{}; - -template -struct json_avoids_null> : std::true_type -{}; - -template -struct json_avoids_null> : std::true_type -{}; - -template -struct json_avoids_null> : std::true_type -{}; - -template -struct json_avoids_null> : std::true_type -{}; - -/** - * `ExperimentalFeature` is always rendered as a string. - */ -template<> -struct json_avoids_null : std::true_type -{}; - } // namespace nix namespace nlohmann { diff --git a/src/libutil/include/nix/util/logging.hh b/src/libutil/include/nix/util/logging.hh index 500d443e6e2..5e211703daa 100644 --- a/src/libutil/include/nix/util/logging.hh +++ b/src/libutil/include/nix/util/logging.hh @@ -39,6 +39,8 @@ typedef enum { resSetExpected = 106, resPostBuildLogLine = 107, resFetchStatus = 108, + resHashMismatch = 109, + resBuildResult = 110, } ResultType; typedef uint64_t ActivityId; @@ -59,7 +61,7 @@ struct LoggerSettings : Config "", "json-log-path", R"( - A file or unix socket to which JSON records of Nix's log output are + A file or Unix domain socket to which JSON records of Nix's log output are written, in the same format as `--log-format internal-json` (without the `@nix ` prefixes on each line). Concurrent writes to the same file by multiple Nix processes are not supported and @@ -158,6 +160,8 @@ public: virtual void result(ActivityId act, ResultType type, const Fields & fields) {}; + virtual void result(ActivityId act, ResultType type, const nlohmann::json & json) {}; + virtual void writeToStdout(std::string_view s); template @@ -222,6 +226,11 @@ struct Activity result(resSetExpected, type2, expected); } + void result(ResultType type, const nlohmann::json & json) const + { + logger.result(id, type, json); + } + template void result(ResultType type, const Args &... args) const { diff --git a/src/libutil/include/nix/util/memory-source-accessor.hh b/src/libutil/include/nix/util/memory-source-accessor.hh index a04d1d347b2..eba282fe1c1 100644 --- a/src/libutil/include/nix/util/memory-source-accessor.hh +++ b/src/libutil/include/nix/util/memory-source-accessor.hh @@ -1,3 +1,6 @@ +#pragma once +///@file + #include "nix/util/source-path.hh" #include "nix/util/fs-sink.hh" #include "nix/util/variant-wrapper.hh" @@ -55,7 +58,7 @@ struct MemorySourceAccessor : virtual SourceAccessor Stat lstat() const; }; - File root{File::Directory{}}; + std::optional root; bool operator==(const MemorySourceAccessor &) const noexcept = default; diff --git a/src/libutil/include/nix/util/meson.build b/src/libutil/include/nix/util/meson.build index bdf1142590c..eebbf0f6afe 100644 --- a/src/libutil/include/nix/util/meson.build +++ b/src/libutil/include/nix/util/meson.build @@ -37,15 +37,18 @@ headers = files( 'file-system.hh', 'finally.hh', 'fmt.hh', + 'forwarding-source-accessor.hh', 'fs-sink.hh', 'git.hh', 'hash.hh', 'hilite.hh', 'json-impls.hh', + 'json-non-null.hh', 'json-utils.hh', 'logging.hh', 'lru-cache.hh', 'memory-source-accessor.hh', + 'mounted-source-accessor.hh', 'muxable-pipe.hh', 'os-string.hh', 'pool.hh', diff --git a/src/libutil/include/nix/util/mounted-source-accessor.hh b/src/libutil/include/nix/util/mounted-source-accessor.hh new file mode 100644 index 00000000000..518ae4f0959 --- /dev/null +++ b/src/libutil/include/nix/util/mounted-source-accessor.hh @@ -0,0 +1,20 @@ +#pragma once + +#include "source-accessor.hh" + +namespace nix { + +struct MountedSourceAccessor : SourceAccessor +{ + virtual void mount(CanonPath mountPoint, ref accessor) = 0; + + /** + * Return the accessor mounted on `mountPoint`, or `nullptr` if + * there is no such mount point. + */ + virtual std::shared_ptr getMount(CanonPath mountPoint) = 0; +}; + +ref makeMountedSourceAccessor(std::map> mounts); + +} // namespace nix diff --git a/src/libutil/include/nix/util/pool.hh b/src/libutil/include/nix/util/pool.hh index a9091c2dee2..5afadc72c75 100644 --- a/src/libutil/include/nix/util/pool.hh +++ b/src/libutil/include/nix/util/pool.hh @@ -211,6 +211,12 @@ public: left.push_back(p); std::swap(state_->idle, left); } + + std::vector> clear() + { + auto state_(state.lock()); + return std::move(state_->idle); + } }; } // namespace nix diff --git a/src/libutil/include/nix/util/pos-idx.hh b/src/libutil/include/nix/util/pos-idx.hh index 8e668176c61..7b7d16ca3a4 100644 --- a/src/libutil/include/nix/util/pos-idx.hh +++ b/src/libutil/include/nix/util/pos-idx.hh @@ -15,12 +15,12 @@ class PosIdx private: uint32_t id; +public: explicit PosIdx(uint32_t id) : id(id) { } -public: PosIdx() : id(0) { @@ -45,6 +45,11 @@ public: { return std::hash{}(id); } + + uint32_t get() const + { + return id; + } }; inline PosIdx noPos = {}; diff --git a/src/libutil/include/nix/util/pos-table.hh b/src/libutil/include/nix/util/pos-table.hh index d944b135317..4ef4b9af4cb 100644 --- a/src/libutil/include/nix/util/pos-table.hh +++ b/src/libutil/include/nix/util/pos-table.hh @@ -49,20 +49,29 @@ private: */ using LinesCache = LRUCache; - std::map origins; - mutable Sync linesCache; + // FIXME: this could be made lock-free (at least for access) if we + // have a data structure where pointers to existing positions are + // never invalidated. + struct State + { + std::map origins; + }; + + SharedSync state_; + const Origin * resolve(PosIdx p) const { if (p.id == 0) return nullptr; + auto state(state_.readLock()); const auto idx = p.id - 1; - /* we want the last key <= idx, so we'll take prev(first key > idx). - this is guaranteed to never rewind origin.begin because the first - key is always 0. */ - const auto pastOrigin = origins.upper_bound(idx); + /* We want the last key <= idx, so we'll take prev(first key > + idx). This is guaranteed to never rewind origin.begin + because the first key is always 0. */ + const auto pastOrigin = state->origins.upper_bound(idx); return &std::prev(pastOrigin)->second; } @@ -74,15 +83,16 @@ public: Origin addOrigin(Pos::Origin origin, size_t size) { + auto state(state_.lock()); uint32_t offset = 0; - if (auto it = origins.rbegin(); it != origins.rend()) + if (auto it = state->origins.rbegin(); it != state->origins.rend()) offset = it->first + it->second.size; // +1 because all PosIdx are offset by 1 to begin with, and // another +1 to ensure that all origins can point to EOF, eg // on (invalid) empty inputs. if (2 + offset + size < offset) return Origin{origin, offset, 0}; - return origins.emplace(offset, Origin{origin, offset, size}).first->second; + return state->origins.emplace(offset, Origin{origin, offset, size}).first->second; } PosIdx add(const Origin & origin, size_t offset) diff --git a/src/libutil/include/nix/util/ref.hh b/src/libutil/include/nix/util/ref.hh index fb27949c006..7cf5ef25ebc 100644 --- a/src/libutil/include/nix/util/ref.hh +++ b/src/libutil/include/nix/util/ref.hh @@ -18,6 +18,9 @@ private: std::shared_ptr p; public: + + using element_type = T; + explicit ref(const std::shared_ptr & p) : p(p) { diff --git a/src/libutil/include/nix/util/serialise.hh b/src/libutil/include/nix/util/serialise.hh index 16e0d0fa568..d6845a494dc 100644 --- a/src/libutil/include/nix/util/serialise.hh +++ b/src/libutil/include/nix/util/serialise.hh @@ -97,6 +97,8 @@ struct Source void drainInto(Sink & sink); std::string drain(); + + virtual void skip(size_t len); }; /** @@ -177,6 +179,7 @@ struct FdSource : BufferedSource Descriptor fd; size_t read = 0; BackedStringView endOfFileError{"unexpected end-of-file"}; + bool isSeekable = true; FdSource() : fd(INVALID_DESCRIPTOR) @@ -200,6 +203,8 @@ struct FdSource : BufferedSource */ bool hasData(); + void skip(size_t len) override; + protected: size_t readUnbuffered(char * data, size_t len) override; private: @@ -250,6 +255,8 @@ struct StringSource : Source } size_t read(char * data, size_t len) override; + + void skip(size_t len) override; }; /** diff --git a/src/libutil/include/nix/util/source-accessor.hh b/src/libutil/include/nix/util/source-accessor.hh index aa937da487c..671444e6f37 100644 --- a/src/libutil/include/nix/util/source-accessor.hh +++ b/src/libutil/include/nix/util/source-accessor.hh @@ -121,7 +121,7 @@ struct SourceAccessor : std::enable_shared_from_this std::string typeString(); }; - Stat lstat(const CanonPath & path); + virtual Stat lstat(const CanonPath & path); virtual std::optional maybeLstat(const CanonPath & path) = 0; @@ -180,6 +180,27 @@ struct SourceAccessor : std::enable_shared_from_this */ std::optional fingerprint; + /** + * Return the fingerprint for `path`. This is usually the + * fingerprint of the current accessor, but for composite + * accessors (like `MountedSourceAccessor`), we want to return the + * fingerprint of the "inner" accessor if the current one lacks a + * fingerprint. + * + * So this method is intended to return the most-outer accessor + * that has a fingerprint for `path`. It also returns the path that `path` + * corresponds to in that accessor. + * + * For example: in a `MountedSourceAccessor` that has + * `/nix/store/foo` mounted, + * `getFingerprint("/nix/store/foo/bar")` will return the path + * `/bar` and the fingerprint of the `/nix/store/foo` accessor. + */ + virtual std::pair> getFingerprint(const CanonPath & path) + { + return {path, fingerprint}; + } + /** * Return the maximum last-modified time of the files in this * tree, if available. @@ -214,8 +235,6 @@ ref getFSSourceAccessor(); */ ref makeFSSourceAccessor(std::filesystem::path root); -ref makeMountedSourceAccessor(std::map> mounts); - /** * Construct an accessor that presents a "union" view of a vector of * underlying accessors. Earlier accessors take precedence over later. diff --git a/src/libutil/include/nix/util/source-path.hh b/src/libutil/include/nix/util/source-path.hh index f7cfc8ef72b..08f9fe580b0 100644 --- a/src/libutil/include/nix/util/source-path.hh +++ b/src/libutil/include/nix/util/source-path.hh @@ -119,15 +119,23 @@ struct SourcePath std::ostream & operator<<(std::ostream & str, const SourcePath & path); +inline std::size_t hash_value(const SourcePath & path) +{ + std::size_t hash = 0; + boost::hash_combine(hash, path.accessor->number); + boost::hash_combine(hash, path.path); + return hash; +} + } // namespace nix template<> struct std::hash { + using is_avalanching = std::true_type; + std::size_t operator()(const nix::SourcePath & s) const noexcept { - std::size_t hash = 0; - hash_combine(hash, s.accessor->number, s.path); - return hash; + return nix::hash_value(s); } }; diff --git a/src/libutil/include/nix/util/strings-inline.hh b/src/libutil/include/nix/util/strings-inline.hh index d99b686fc13..61bddfedadf 100644 --- a/src/libutil/include/nix/util/strings-inline.hh +++ b/src/libutil/include/nix/util/strings-inline.hh @@ -26,18 +26,29 @@ C tokenizeString(std::string_view s, std::string_view separators) } template -C basicSplitString(std::basic_string_view s, std::basic_string_view separators) +void basicSplitStringInto(C & accum, std::basic_string_view s, std::basic_string_view separators) { - C result; size_t pos = 0; while (pos <= s.size()) { auto end = s.find_first_of(separators, pos); if (end == s.npos) end = s.size(); - result.insert(result.end(), std::basic_string(s, pos, end - pos)); + accum.insert(accum.end(), typename C::value_type{s.substr(pos, end - pos)}); pos = end + 1; } +} +template +void splitStringInto(C & accum, std::string_view s, std::string_view separators) +{ + basicSplitStringInto(accum, s, separators); +} + +template +C basicSplitString(std::basic_string_view s, std::basic_string_view separators) +{ + C result; + basicSplitStringInto(result, s, separators); return result; } diff --git a/src/libutil/include/nix/util/strings.hh b/src/libutil/include/nix/util/strings.hh index b4ef66bfeb3..ba37ce79f63 100644 --- a/src/libutil/include/nix/util/strings.hh +++ b/src/libutil/include/nix/util/strings.hh @@ -12,11 +12,6 @@ namespace nix { -/* - * workaround for unavailable view() method (C++20) of std::ostringstream under MacOS with clang-16 - */ -std::string_view toView(const std::ostringstream & os); - /** * String tokenizer. * diff --git a/src/libutil/include/nix/util/sync.hh b/src/libutil/include/nix/util/sync.hh index 262fc328b57..3a41d1bd808 100644 --- a/src/libutil/include/nix/util/sync.hh +++ b/src/libutil/include/nix/util/sync.hh @@ -36,6 +36,8 @@ private: public: + using element_type = T; + SyncBase() {} SyncBase(const T & data) diff --git a/src/libutil/include/nix/util/terminal.hh b/src/libutil/include/nix/util/terminal.hh index f19de268c8a..fa71e074e6c 100644 --- a/src/libutil/include/nix/util/terminal.hh +++ b/src/libutil/include/nix/util/terminal.hh @@ -36,4 +36,12 @@ void updateWindowSize(); */ std::pair getWindowSize(); +/** + * Get the slave name of a pseudoterminal in a thread-safe manner. + * + * @param fd The file descriptor of the pseudoterminal master + * @return The slave device name as a string + */ +std::string getPtsName(int fd); + } // namespace nix diff --git a/src/libutil/include/nix/util/thread-pool.hh b/src/libutil/include/nix/util/thread-pool.hh index 811c03d889f..ce34516ef4b 100644 --- a/src/libutil/include/nix/util/thread-pool.hh +++ b/src/libutil/include/nix/util/thread-pool.hh @@ -85,21 +85,24 @@ template void processGraph( const std::set & nodes, std::function(const T &)> getEdges, - std::function processNode) + std::function processNode, + bool discoverNodes = false, + size_t maxThreads = 0) { struct Graph { + std::set known; std::set left; std::map> refs, rrefs; }; - Sync graph_(Graph{nodes, {}, {}}); + Sync graph_(Graph{nodes, nodes, {}, {}}); std::function worker; - /* Create pool last to ensure threads are stopped before other destructors - * run */ - ThreadPool pool; + /* Create pool last to ensure threads are stopped before other + destructors run. */ + ThreadPool pool(maxThreads); worker = [&](const T & node) { { @@ -116,11 +119,19 @@ void processGraph( { auto graph(graph_.lock()); - for (auto & ref : refs) + for (auto & ref : refs) { + if (discoverNodes) { + auto [i, inserted] = graph->known.insert(ref); + if (inserted) { + pool.enqueue(std::bind(worker, std::ref(*i))); + graph->left.insert(ref); + } + } if (graph->left.count(ref)) { graph->refs[node].insert(ref); graph->rrefs[ref].insert(node); } + } if (graph->refs[node].empty()) goto doWork; } diff --git a/src/libutil/include/nix/util/url.hh b/src/libutil/include/nix/util/url.hh index 3262b44b719..4ed80feb3a2 100644 --- a/src/libutil/include/nix/util/url.hh +++ b/src/libutil/include/nix/util/url.hh @@ -1,7 +1,14 @@ #pragma once ///@file +#include +#include + #include "nix/util/error.hh" +#include "nix/util/canon-path.hh" +#include "nix/util/split.hh" +#include "nix/util/util.hh" +#include "nix/util/variant-wrapper.hh" namespace nix { @@ -65,6 +72,7 @@ struct ParsedURL }; std::string scheme; + /** * Optional parsed authority component of the URL. * @@ -75,18 +83,171 @@ struct ParsedURL * part of the URL. */ std::optional authority; - std::string path; + + /** + * @note Unlike Unix paths, URLs provide a way to escape path + * separators, in the form of the `%2F` encoding of `/`. That means + * that if one percent-decodes the path into a single string, that + * decoding will be *lossy*, because `/` and `%2F` both become `/`. + * The right thing to do is instead split up the path on `/`, and + * then percent decode each part. + * + * For an example, the path + * ``` + * foo/bar%2Fbaz/quux + * ``` + * is parsed as + * ``` + * {"foo, "bar/baz", "quux"} + * ``` + * + * We're doing splitting and joining that assumes the separator (`/` in this case) only goes *between* elements. + * + * That means the parsed representation will begin with an empty + * element to make an initial `/`, and will end with an ementy + * element to make a trailing `/`. That means that elements of this + * vector mostly, but *not always*, correspond to segments of the + * path. + * + * Examples: + * + * - ``` + * https://foo.com/bar + * ``` + * has path + * ``` + * {"", "bar"} + * ``` + * + * - ``` + * https://foo.com/bar/ + * ``` + * has path + * ``` + * {"", "bar", ""} + * ``` + * + * - ``` + * https://foo.com//bar/// + * ``` + * has path + * ``` + * {"", "", "bar", "", "", ""} + * ``` + * + * - ``` + * https://foo.com + * ``` + * has path + * ``` + * {""} + * ``` + * + * - ``` + * https://foo.com/ + * ``` + * has path + * ``` + * {"", ""} + * ``` + * + * - ``` + * tel:01234 + * ``` + * has path `{"01234"}` (and no authority) + * + * - ``` + * foo:/01234 + * ``` + * has path `{"", "01234"}` (and no authority) + * + * Note that both trailing and leading slashes are, in general, + * semantically significant. + * + * For trailing slashes, the main example affecting many schemes is + * that `../baz` resolves against a base URL different depending on + * the presence/absence of a trailing slash: + * + * - `https://foo.com/bar` is `https://foo.com/baz` + * + * - `https://foo.com/bar/` is `https://foo.com/bar/baz` + * + * See `parseURLRelative` for more details. + * + * For leading slashes, there are some requirements to be aware of. + * + * - When there is an authority, the path *must* start with a leading + * slash. Otherwise the path will not be separated from the + * authority, and will not round trip though the parser: + * + * ``` + * {.scheme="https", .authority.host = "foo", .path={"bad"}} + * ``` + * will render to `https://foobar`. but that would parse back as as + * ``` + * {.scheme="https", .authority.host = "foobar", .path={}} + * ``` + * + * - When there is no authority, the path must *not* begin with two + * slashes. Otherwise, there will be another parser round trip + * issue: + * + * ``` + * {.scheme="https", .path={"", "", "bad"}} + * ``` + * will render to `https://bad`. but that would parse back as as + * ``` + * {.scheme="https", .authority.host = "bad", .path={}} + * ``` + * + * These invariants will be checked in `to_string` and + * `renderAuthorityAndPath`. + */ + std::vector path; + StringMap query; + std::string fragment; + /** + * Render just the middle part of a URL, without the `//` which + * indicates whether the authority is present. + * + * @note This is kind of an ad-hoc + * operation, but it ends up coming up with some frequency, probably + * due to the current design of `StoreReference` in `nix-store`. + */ + std::string renderAuthorityAndPath() const; + std::string to_string() const; + /** + * Render the path to a string. + * + * @param encode Whether to percent encode path segments. + */ + std::string renderPath(bool encode = false) const; + auto operator<=>(const ParsedURL & other) const noexcept = default; /** - * Remove `.` and `..` path elements. + * Remove `.` and `..` path segments. */ ParsedURL canonicalise(); + + /** + * Get a range of path segments (the substrings separated by '/' characters). + * + * @param skipEmpty Skip all empty path segments + */ + auto pathSegments(bool skipEmpty) const & + { + return std::views::filter(path, [skipEmpty](std::string_view segment) { + if (skipEmpty) + return !segment.empty(); + return true; + }); + } }; std::ostream & operator<<(std::ostream & os, const ParsedURL & url); @@ -96,6 +257,22 @@ MakeError(BadURL, Error); std::string percentDecode(std::string_view in); std::string percentEncode(std::string_view s, std::string_view keep = ""); +/** + * Get the path part of the URL as an absolute or relative Path. + * + * @throws if any path component contains an slash (which would have + * been escaped `%2F` in the rendered URL). This is because OS file + * paths have no escape sequences --- file names cannot contain a + * `/`. + */ +Path renderUrlPathEnsureLegal(const std::vector & urlPath); + +/** + * Percent encode path. `%2F` for "interior slashes" is the most + * important. + */ +std::string encodeUrlPath(std::span urlPath); + /** * @param lenient @see parseURL */ @@ -114,9 +291,29 @@ std::string encodeQuery(const StringMap & query); * @note IPv6 ZoneId literals (RFC4007) are represented in URIs according to RFC6874. * * @throws BadURL + * + * The WHATWG specification of the URL constructor in Java Script is + * also a useful reference: + * https://url.spec.whatwg.org/#concept-basic-url-parser. Note, however, + * that it includes various scheme-specific normalizations / extra steps + * that we do not implement. */ ParsedURL parseURL(std::string_view url, bool lenient = false); +/** + * Like `parseURL`, but also accepts relative URLs, which are resolved + * against the given base URL. + * + * This is specified in [IETF RFC 3986, section 5](https://datatracker.ietf.org/doc/html/rfc3986#section-5) + * + * @throws BadURL + * + * Behavior should also match the `new URL(url, base)` JavaScript + * constructor, except for extra steps specific to the HTTP scheme. See + * `parseURL` for link to the relevant WHATWG standard. + */ +ParsedURL parseURLRelative(std::string_view url, const ParsedURL & base); + /** * Although that’s not really standardized anywhere, an number of tools * use a scheme of the form 'x+y' in urls, where y is the “transport layer” @@ -136,7 +333,7 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme); /* Detects scp-style uris (e.g. git@github.com:NixOS/nix) and fixes them by removing the `:` and assuming a scheme of `ssh://`. Also changes absolute paths into file:// URLs. */ -std::string fixGitURL(const std::string & url); +ParsedURL fixGitURL(const std::string & url); /** * Whether a string is valid as RFC 3986 scheme name. @@ -147,4 +344,72 @@ std::string fixGitURL(const std::string & url); */ bool isValidSchemeName(std::string_view scheme); +/** + * Either a ParsedURL or a verbatim string. This is necessary because in certain cases URI must be passed + * verbatim (e.g. in builtin fetchers), since those are specified by the user. + * In those cases normalizations performed by the ParsedURL might be surprising + * and undesirable, since Nix must be a universal client that has to work with + * various broken services that might interpret URLs in quirky and non-standard ways. + * + * One of those examples is space-as-plus encoding that is very widespread, but it's + * not strictly RFC3986 compliant. We must preserve that information verbatim. + * + * Though we perform parsing and validation for internal needs. + */ +struct VerbatimURL +{ + using Raw = std::variant; + Raw raw; + + VerbatimURL(std::string_view s) + : raw(std::string{s}) + { + } + + VerbatimURL(std::string s) + : raw(std::move(s)) + { + } + + VerbatimURL(ParsedURL url) + : raw(std::move(url)) + { + } + + /** + * Get the encoded URL (if specified) verbatim or encode the parsed URL. + */ + std::string to_string() const + { + return std::visit( + overloaded{ + [](const std::string & str) { return str; }, [](const ParsedURL & url) { return url.to_string(); }}, + raw); + } + + const ParsedURL parsed() const + { + return std::visit( + overloaded{ + [](const std::string & str) { return parseURL(str); }, [](const ParsedURL & url) { return url; }}, + raw); + } + + std::string_view scheme() const & + { + return std::visit( + overloaded{ + [](std::string_view str) { + auto scheme = splitPrefixTo(str, ':'); + if (!scheme) + throw BadURL("URL '%s' doesn't have a scheme", str); + return *scheme; + }, + [](const ParsedURL & url) -> std::string_view { return url.scheme; }}, + raw); + } +}; + +std::ostream & operator<<(std::ostream & os, const VerbatimURL & url); + } // namespace nix diff --git a/src/libutil/include/nix/util/util.hh b/src/libutil/include/nix/util/util.hh index 561550c4144..0bf9efcb5a5 100644 --- a/src/libutil/include/nix/util/util.hh +++ b/src/libutil/include/nix/util/util.hh @@ -197,7 +197,7 @@ std::pair getLine(std::string_view s); * Get a value for the specified key from an associate container. */ template -const typename T::mapped_type * get(const T & map, K & key) +const typename T::mapped_type * get(const T & map, const K & key) { auto i = map.find(key); if (i == map.end()) @@ -206,7 +206,7 @@ const typename T::mapped_type * get(const T & map, K & key) } template -typename T::mapped_type * get(T & map, K & key) +typename T::mapped_type * get(T & map, const K & key) { auto i = map.find(key); if (i == map.end()) @@ -214,15 +214,34 @@ typename T::mapped_type * get(T & map, K & key) return &i->second; } -/** Deleted because this is use-after-free liability. Just don't pass temporaries to this overload set. */ +/** + * Deleted because this is use-after-free liability. Just don't pass temporaries to this overload set. + */ +template +typename T::mapped_type * get(T && map, const K & key) = delete; + template -typename T::mapped_type * get(T && map, const typename T::key_type & key) = delete; +std::optional getOptional(const T & map, const typename T::key_type & key) +{ + auto i = map.find(key); + if (i == map.end()) + return std::nullopt; + return {i->second}; +} + +template +std::optional getConcurrent(const T & map, const typename T::key_type & key) +{ + std::optional res; + map.cvisit(key, [&](auto & x) { res = x.second; }); + return res; +} /** * Get a value for the specified key from an associate container, or a default value if the key isn't present. */ template -const typename T::mapped_type & getOr(T & map, K & key, const typename T::mapped_type & defaultValue) +const typename T::mapped_type & getOr(T & map, const K & key, const typename T::mapped_type & defaultValue) { auto i = map.find(key); if (i == map.end()) @@ -230,10 +249,11 @@ const typename T::mapped_type & getOr(T & map, K & key, const typename T::mapped return i->second; } -/** Deleted because this is use-after-free liability. Just don't pass temporaries to this overload set. */ -template -const typename T::mapped_type & -getOr(T && map, const typename T::key_type & key, const typename T::mapped_type & defaultValue) = delete; +/** + * Deleted because this is use-after-free liability. Just don't pass temporaries to this overload set. + */ +template +const typename T::mapped_type & getOr(T && map, const K & key, const typename T::mapped_type & defaultValue) = delete; /** * Remove and return the first item from a container. diff --git a/src/libutil/linux/cgroup.cc b/src/libutil/linux/cgroup.cc index 20d19ae7dea..9e78ac6d2ae 100644 --- a/src/libutil/linux/cgroup.cc +++ b/src/libutil/linux/cgroup.cc @@ -4,10 +4,10 @@ #include "nix/util/file-system.hh" #include "nix/util/finally.hh" +#include #include #include #include -#include #include #include @@ -76,7 +76,7 @@ static CgroupStats destroyCgroup(const std::filesystem::path & cgroup, bool retu int round = 1; - std::unordered_set pidsShown; + boost::unordered_flat_set pidsShown; while (true) { auto pids = tokenizeString>(readFile(procsFile)); diff --git a/src/libutil/logging.cc b/src/libutil/logging.cc index 997110617b3..b3e69405eb6 100644 --- a/src/libutil/logging.cc +++ b/src/libutil/logging.cc @@ -121,7 +121,7 @@ class SimpleLogger : public Logger std::ostringstream oss; showErrorInfo(oss, ei, loggerSettings.showTrace.get()); - log(ei.level, toView(oss)); + log(ei.level, oss.view()); } void startActivity( @@ -336,6 +336,16 @@ struct JSONLogger : Logger addFields(json, fields); write(json); } + + void result(ActivityId act, ResultType type, const nlohmann::json & j) override + { + nlohmann::json json; + json["action"] = "result"; + json["id"] = act; + json["type"] = type; + json["payload"] = j; + write(json); + } }; std::unique_ptr makeJSONLogger(Descriptor fd, bool includeNixPrefix) diff --git a/src/libutil/memory-source-accessor.cc b/src/libutil/memory-source-accessor.cc index 363f52a54e9..a9ffb77469c 100644 --- a/src/libutil/memory-source-accessor.cc +++ b/src/libutil/memory-source-accessor.cc @@ -4,7 +4,22 @@ namespace nix { MemorySourceAccessor::File * MemorySourceAccessor::open(const CanonPath & path, std::optional create) { - File * cur = &root; + bool hasRoot = root.has_value(); + + // Special handling of root directory. + if (path.isRoot() && !hasRoot) { + if (create) { + root = std::move(*create); + return &root.value(); + } + return nullptr; + } + + // Root does not exist. + if (!hasRoot) + return nullptr; + + File * cur = &root.value(); bool newF = false; @@ -112,6 +127,10 @@ std::string MemorySourceAccessor::readLink(const CanonPath & path) SourcePath MemorySourceAccessor::addFile(CanonPath path, std::string && contents) { + // Create root directory automatically if necessary as a convenience. + if (!root && !path.isRoot()) + open(CanonPath::root, File::Directory{}); + auto * f = open(path, File{File::Regular{}}); if (!f) throw Error("file '%s' cannot be made because some parent file is not a directory", path); @@ -189,11 +208,16 @@ void MemorySink::createSymlink(const CanonPath & path, const std::string & targe ref makeEmptySourceAccessor() { - static auto empty = make_ref().cast(); - /* Don't forget to clear the display prefix, as the default constructed - SourceAccessor has the «unknown» prefix. Since this accessor is supposed - to mimic an empty root directory the prefix needs to be empty. */ - empty->setPathDisplay(""); + static auto empty = []() { + auto empty = make_ref(); + MemorySink sink{*empty}; + sink.createDirectory(CanonPath::root); + /* Don't forget to clear the display prefix, as the default constructed + SourceAccessor has the «unknown» prefix. Since this accessor is supposed + to mimic an empty root directory the prefix needs to be empty. */ + empty->setPathDisplay(""); + return empty.cast(); + }(); return empty; } diff --git a/src/libutil/meson.build b/src/libutil/meson.build index c294f895ace..8c9e1f1eb24 100644 --- a/src/libutil/meson.build +++ b/src/libutil/meson.build @@ -57,7 +57,12 @@ deps_private += blake3 boost = dependency( 'boost', - modules : [ 'context', 'coroutine', 'iostreams', 'url' ], + modules : [ + 'context', + 'coroutine', + 'iostreams', + 'url', + ], include_type : 'system', version : '>=1.82.0', ) @@ -113,6 +118,7 @@ config_priv_h = configure_file( ) subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') sources = [ config_priv_h ] + files( 'archive.cc', @@ -192,6 +198,7 @@ subdir('nix-meson-build-support/windows-version') this_library = library( 'nixutil', sources, + soversion : nix_soversion, dependencies : deps_public + deps_private + deps_other, include_directories : include_dirs, link_args : linker_export_flags, diff --git a/src/libutil/mounted-source-accessor.cc b/src/libutil/mounted-source-accessor.cc index 4c32147f961..d9398045cc5 100644 --- a/src/libutil/mounted-source-accessor.cc +++ b/src/libutil/mounted-source-accessor.cc @@ -1,18 +1,22 @@ -#include "nix/util/source-accessor.hh" +#include "nix/util/mounted-source-accessor.hh" + +#include namespace nix { -struct MountedSourceAccessor : SourceAccessor +struct MountedSourceAccessorImpl : MountedSourceAccessor { - std::map> mounts; + boost::concurrent_flat_map> mounts; - MountedSourceAccessor(std::map> _mounts) - : mounts(std::move(_mounts)) + MountedSourceAccessorImpl(std::map> _mounts) { displayPrefix.clear(); // Currently we require a root filesystem. This could be relaxed. - assert(mounts.contains(CanonPath::root)); + assert(_mounts.contains(CanonPath::root)); + + for (auto & [path, accessor] : _mounts) + mount(path, accessor); // FIXME: return dummy parent directories automatically? } @@ -23,6 +27,12 @@ struct MountedSourceAccessor : SourceAccessor return accessor->readFile(subpath); } + Stat lstat(const CanonPath & path) override + { + auto [accessor, subpath] = resolve(path); + return accessor->lstat(subpath); + } + std::optional maybeLstat(const CanonPath & path) override { auto [accessor, subpath] = resolve(path); @@ -52,10 +62,9 @@ struct MountedSourceAccessor : SourceAccessor // Find the nearest parent of `path` that is a mount point. std::vector subpath; while (true) { - auto i = mounts.find(path); - if (i != mounts.end()) { + if (auto mount = getMount(path)) { std::reverse(subpath.begin(), subpath.end()); - return {i->second, CanonPath(subpath)}; + return {ref(mount), CanonPath(subpath)}; } assert(!path.isRoot()); @@ -69,11 +78,32 @@ struct MountedSourceAccessor : SourceAccessor auto [accessor, subpath] = resolve(path); return accessor->getPhysicalPath(subpath); } + + void mount(CanonPath mountPoint, ref accessor) override + { + mounts.emplace(std::move(mountPoint), std::move(accessor)); + } + + std::shared_ptr getMount(CanonPath mountPoint) override + { + if (auto res = getConcurrent(mounts, mountPoint)) + return *res; + else + return nullptr; + } + + std::pair> getFingerprint(const CanonPath & path) override + { + if (fingerprint) + return {path, fingerprint}; + auto [accessor, subpath] = resolve(path); + return accessor->getFingerprint(subpath); + } }; -ref makeMountedSourceAccessor(std::map> mounts) +ref makeMountedSourceAccessor(std::map> mounts) { - return make_ref(std::move(mounts)); + return make_ref(std::move(mounts)); } } // namespace nix diff --git a/src/libutil/package.nix b/src/libutil/package.nix index 3deb7ba3ae3..287e6c6a113 100644 --- a/src/libutil/package.nix +++ b/src/libutil/package.nix @@ -22,7 +22,7 @@ let in mkMesonLibrary (finalAttrs: { - pname = "nix-util"; + pname = "determinate-nix-util"; inherit version; workDir = ./.; diff --git a/src/libutil/posix-source-accessor.cc b/src/libutil/posix-source-accessor.cc index b932f6ab5e5..fe3bcb1c1c7 100644 --- a/src/libutil/posix-source-accessor.cc +++ b/src/libutil/posix-source-accessor.cc @@ -95,16 +95,14 @@ std::optional PosixSourceAccessor::cachedLstat(const CanonPath & pa // former is not hashable on libc++. Path absPath = makeAbsPath(path).string(); - std::optional res; - cache.cvisit(absPath, [&](auto & x) { res.emplace(x.second); }); - if (res) + if (auto res = getConcurrent(cache, absPath)) return *res; auto st = nix::maybeLstat(absPath.c_str()); if (cache.size() >= 16384) cache.clear(); - cache.emplace(absPath, st); + cache.emplace(std::move(absPath), st); return st; } @@ -116,6 +114,8 @@ std::optional PosixSourceAccessor::maybeLstat(const CanonP auto st = cachedLstat(path); if (!st) return std::nullopt; + // This makes the accessor thread-unsafe, but we only seem to use the actual value in a single threaded context in + // `src/libfetchers/path.cc`. mtime = std::max(mtime, st->st_mtime); return Stat{ .type = S_ISREG(st->st_mode) ? tRegular diff --git a/src/libutil/serialise.cc b/src/libutil/serialise.cc index 15629935e12..ba153625ee9 100644 --- a/src/libutil/serialise.cc +++ b/src/libutil/serialise.cc @@ -94,9 +94,8 @@ void Source::drainInto(Sink & sink) { std::array buf; while (true) { - size_t n; try { - n = read(buf.data(), buf.size()); + auto n = read(buf.data(), buf.size()); sink({buf.data(), n}); } catch (EndOfFile &) { break; @@ -111,6 +110,16 @@ std::string Source::drain() return std::move(s.s); } +void Source::skip(size_t len) +{ + std::array buf; + while (len) { + auto n = read(buf.data(), std::min(len, buf.size())); + assert(n <= len); + len -= n; + } +} + size_t BufferedSource::read(char * data, size_t len) { if (!buffer) @@ -120,7 +129,7 @@ size_t BufferedSource::read(char * data, size_t len) bufPosIn = readUnbuffered(buffer.get(), bufSize); /* Copy out the data in the buffer. */ - size_t n = len > bufPosIn - bufPosOut ? bufPosIn - bufPosOut : len; + auto n = std::min(len, bufPosIn - bufPosOut); memcpy(data, buffer.get() + bufPosOut, n); bufPosOut += n; if (bufPosIn == bufPosOut) @@ -191,6 +200,39 @@ bool FdSource::hasData() } } +void FdSource::skip(size_t len) +{ + /* Discard data in the buffer. */ + if (len && buffer && bufPosIn - bufPosOut) { + if (len >= bufPosIn - bufPosOut) { + len -= bufPosIn - bufPosOut; + bufPosIn = bufPosOut = 0; + } else { + bufPosOut += len; + len = 0; + } + } + +#ifndef _WIN32 + /* If we can, seek forward in the file to skip the rest. */ + if (isSeekable && len) { + if (lseek(fd, len, SEEK_CUR) == -1) { + if (errno == ESPIPE) + isSeekable = false; + else + throw SysError("seeking forward in file"); + } else { + read += len; + return; + } + } +#endif + + /* Otherwise, skip by reading. */ + if (len) + BufferedSource::skip(len); +} + size_t StringSource::read(char * data, size_t len) { if (pos == s.size()) @@ -200,6 +242,16 @@ size_t StringSource::read(char * data, size_t len) return n; } +void StringSource::skip(size_t len) +{ + const size_t remain = s.size() - pos; + if (len > remain) { + pos = s.size(); + throw EndOfFile("end of string reached"); + } + pos += len; +} + std::unique_ptr sourceToSink(std::function fun) { struct SourceToSink : FinishSink diff --git a/src/libutil/strings.cc b/src/libutil/strings.cc index a953900891e..a87567cefaf 100644 --- a/src/libutil/strings.cc +++ b/src/libutil/strings.cc @@ -8,23 +8,6 @@ namespace nix { -struct view_stringbuf : public std::stringbuf -{ - inline std::string_view toView() - { - auto begin = pbase(); - return {begin, begin + pubseekoff(0, std::ios_base::cur, std::ios_base::out)}; - } -}; - -__attribute__((no_sanitize("undefined"))) std::string_view toView(const std::ostringstream & os) -{ - /* Downcasting like this is very much undefined behavior, so we disable - UBSAN for this function. */ - auto buf = static_cast(os.rdbuf()); - return buf->toView(); -} - template std::list tokenizeString(std::string_view s, std::string_view separators); template StringSet tokenizeString(std::string_view s, std::string_view separators); template std::vector tokenizeString(std::string_view s, std::string_view separators); diff --git a/src/libutil/tee-logger.cc b/src/libutil/tee-logger.cc index 8433168a5a8..889b82ca02b 100644 --- a/src/libutil/tee-logger.cc +++ b/src/libutil/tee-logger.cc @@ -65,6 +65,12 @@ struct TeeLogger : Logger logger->result(act, type, fields); } + void result(ActivityId act, ResultType type, const nlohmann::json & json) override + { + for (auto & logger : loggers) + logger->result(act, type, json); + } + void writeToStdout(std::string_view s) override { for (auto & logger : loggers) { diff --git a/src/libutil/terminal.cc b/src/libutil/terminal.cc index b5765487c25..fe22146abb0 100644 --- a/src/libutil/terminal.cc +++ b/src/libutil/terminal.cc @@ -1,6 +1,7 @@ #include "nix/util/terminal.hh" #include "nix/util/environment-variables.hh" #include "nix/util/sync.hh" +#include "nix/util/error.hh" #ifdef _WIN32 # include @@ -12,6 +13,8 @@ #endif #include #include +#include +#include // for ptsname and ptsname_r namespace { @@ -176,4 +179,31 @@ std::pair getWindowSize() return *windowSize.lock(); } +#ifndef _WIN32 +std::string getPtsName(int fd) +{ +# ifdef __APPLE__ + static std::mutex ptsnameMutex; + // macOS doesn't have ptsname_r, use mutex-protected ptsname + std::lock_guard lock(ptsnameMutex); + const char * name = ptsname(fd); + if (!name) { + throw SysError("getting pseudoterminal slave name"); + } + return name; +# else + // Use thread-safe ptsname_r on platforms that support it + // PTY names are typically short: + // - Linux: /dev/pts/N (where N is usually < 1000) + // - FreeBSD: /dev/pts/N + // 64 bytes is more than sufficient for any Unix PTY name + char buf[64]; + if (ptsname_r(fd, buf, sizeof(buf)) != 0) { + throw SysError("getting pseudoterminal slave name"); + } + return buf; +# endif +} +#endif + } // namespace nix diff --git a/src/libutil/union-source-accessor.cc b/src/libutil/union-source-accessor.cc index 96b6a643a22..e3b39f14ed2 100644 --- a/src/libutil/union-source-accessor.cc +++ b/src/libutil/union-source-accessor.cc @@ -72,6 +72,18 @@ struct UnionSourceAccessor : SourceAccessor } return std::nullopt; } + + std::pair> getFingerprint(const CanonPath & path) override + { + if (fingerprint) + return {path, fingerprint}; + for (auto & accessor : accessors) { + auto [subpath, fingerprint] = accessor->getFingerprint(path); + if (fingerprint) + return {subpath, fingerprint}; + } + return {path, std::nullopt}; + } }; ref makeUnionSourceAccessor(std::vector> && accessors) diff --git a/src/libutil/unix/current-process.cc b/src/libutil/unix/current-process.cc new file mode 100644 index 00000000000..eaa2424abcd --- /dev/null +++ b/src/libutil/unix/current-process.cc @@ -0,0 +1,23 @@ +#include "nix/util/current-process.hh" +#include "nix/util/error.hh" +#include + +#include + +namespace nix { + +std::chrono::microseconds getCpuUserTime() +{ + struct rusage buf; + + if (getrusage(RUSAGE_SELF, &buf) != 0) { + throw SysError("failed to get CPU time"); + } + + std::chrono::seconds seconds(buf.ru_utime.tv_sec); + std::chrono::microseconds microseconds(buf.ru_utime.tv_usec); + + return seconds + microseconds; +} + +} // namespace nix diff --git a/src/libutil/unix/include/nix/util/monitor-fd.hh b/src/libutil/unix/include/nix/util/monitor-fd.hh index 5c1e5f1957e..b87bf5ca4f7 100644 --- a/src/libutil/unix/include/nix/util/monitor-fd.hh +++ b/src/libutil/unix/include/nix/util/monitor-fd.hh @@ -2,15 +2,18 @@ ///@file #include -#include +#include -#include #include -#include -#include -#include +#include + +#ifdef __APPLE__ +# include +# include +#endif #include "nix/util/signals.hh" +#include "nix/util/file-descriptor.hh" namespace nix { @@ -20,111 +23,113 @@ private: std::thread thread; Pipe notifyPipe; + void runThread(int watchFd, int notifyFd); + public: - MonitorFdHup(int fd) - { - notifyPipe.create(); - thread = std::thread([this, fd]() { - while (true) { - // There is a POSIX violation on macOS: you have to listen for - // at least POLLHUP to receive HUP events for a FD. POSIX says - // this is not so, and you should just receive them regardless. - // However, as of our testing on macOS 14.5, the events do not - // get delivered if in the all-bits-unset case, but do get - // delivered if `POLLHUP` is set. - // - // This bug filed as rdar://37537852 - // (https://openradar.appspot.com/37537852). - // - // macOS's own man page - // (https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/poll.2.html) - // additionally says that `POLLHUP` is ignored as an input. It - // seems the likely order of events here was - // - // 1. macOS did not follow the POSIX spec - // - // 2. Somebody ninja-fixed this other spec violation to make - // sure `POLLHUP` was not forgotten about, even though they - // "fixed" this issue in a spec-non-compliant way. Whatever, - // we'll use the fix. - // - // Relevant code, current version, which shows the : - // https://github.com/apple-oss-distributions/xnu/blob/94d3b452840153a99b38a3a9659680b2a006908e/bsd/kern/sys_generic.c#L1751-L1758 - // - // The `POLLHUP` detection was added in - // https://github.com/apple-oss-distributions/xnu/commit/e13b1fa57645afc8a7b2e7d868fe9845c6b08c40#diff-a5aa0b0e7f4d866ca417f60702689fc797e9cdfe33b601b05ccf43086c35d395R1468 - // That means added in 2007 or earlier. Should be good enough - // for us. - short hangup_events = -#ifdef __APPLE__ - POLLHUP -#else - 0 -#endif - ; - - /* Wait indefinitely until a POLLHUP occurs. */ - constexpr size_t num_fds = 2; - struct pollfd fds[num_fds] = { - { - .fd = fd, - .events = hangup_events, - }, - { - .fd = notifyPipe.readSide.get(), - .events = hangup_events, - }, - }; - - auto count = poll(fds, num_fds, -1); - if (count == -1) { - if (errno == EINTR || errno == EAGAIN) - continue; - throw SysError("failed to poll() in MonitorFdHup"); - } - /* This shouldn't happen, but can on macOS due to a bug. - See rdar://37550628. - - This may eventually need a delay or further - coordination with the main thread if spinning proves - too harmful. - */ - if (count == 0) - continue; - if (fds[0].revents & POLLHUP) { - unix::triggerInterrupt(); - break; - } - if (fds[1].revents & POLLHUP) { - break; - } - // On macOS, (jade thinks that) it is possible (although not - // observed on macOS 14.5) that in some limited cases on buggy - // kernel versions, all the non-POLLHUP events for the socket - // get delivered. - // - // We could sleep to avoid pointlessly spinning a thread on - // those, but this opens up a different problem, which is that - // if do sleep, it will be longer before the daemon fork for a - // client exits. Imagine a sequential shell script, running Nix - // commands, each of which talk to the daemon. If the previous - // command registered a temp root, exits, and then the next - // command issues a delete request before the temp root is - // cleaned up, that delete request might fail. - // - // Not sleeping doesn't actually fix the race condition --- we - // would need to block on the old connections' tempt roots being - // cleaned up in in the new connection --- but it does make it - // much less likely. - } - }); - }; + MonitorFdHup(int fd); ~MonitorFdHup() { + // Close the write side to signal termination via POLLHUP notifyPipe.writeSide.close(); thread.join(); } }; +#ifdef __APPLE__ +/* This custom kqueue usage exists because Apple's poll implementation is + * broken and loses event subscriptions if EVFILT_READ fires without matching + * the requested `events` in the pollfd. + * + * We use EVFILT_READ, which causes some spurious wakeups (at most one per write + * from the client, in addition to the socket lifecycle events), because the + * alternate API, EVFILT_SOCK, doesn't work on pipes, which this is also used + * to monitor in certain situations. + * + * See (EVFILT_SOCK): + * https://github.com/netty/netty/blob/64bd2f4eb62c2fb906bc443a2aabf894c8b7dce9/transport-classes-kqueue/src/main/java/io/netty/channel/kqueue/AbstractKQueueChannel.java#L434 + * + * See: https://git.lix.systems/lix-project/lix/issues/729 + * Apple bug in poll(2): FB17447257, available at https://openradar.appspot.com/FB17447257 + */ +inline void MonitorFdHup::runThread(int watchFd, int notifyFd) +{ + int kqResult = kqueue(); + if (kqResult < 0) { + throw SysError("MonitorFdHup kqueue"); + } + AutoCloseFD kq{kqResult}; + + std::array kevs; + + // kj uses EVFILT_WRITE for this, but it seems that it causes more spurious + // wakeups in our case of doing blocking IO from another thread compared to + // EVFILT_READ. + // + // EVFILT_WRITE and EVFILT_READ (for sockets at least, where I am familiar + // with the internals) both go through a common filter which catches EOFs + // and generates spurious wakeups for either readable/writable events. + EV_SET(&kevs[0], watchFd, EVFILT_READ, EV_ADD | EV_ENABLE | EV_CLEAR, 0, 0, nullptr); + EV_SET(&kevs[1], notifyFd, EVFILT_READ, EV_ADD | EV_ENABLE | EV_CLEAR, 0, 0, nullptr); + + int result = kevent(kq.get(), kevs.data(), kevs.size(), nullptr, 0, nullptr); + if (result < 0) { + throw SysError("MonitorFdHup kevent add"); + } + + while (true) { + struct kevent event; + int numEvents = kevent(kq.get(), nullptr, 0, &event, 1, nullptr); + if (numEvents < 0) { + throw SysError("MonitorFdHup kevent watch"); + } + + if (numEvents > 0 && (event.flags & EV_EOF)) { + if (event.ident == uintptr_t(watchFd)) { + unix::triggerInterrupt(); + } + // Either watched fd or notify fd closed, exit + return; + } + } +} +#else +inline void MonitorFdHup::runThread(int watchFd, int notifyFd) +{ + while (true) { + struct pollfd fds[2]; + fds[0].fd = watchFd; + fds[0].events = 0; // POSIX: POLLHUP is always reported + fds[1].fd = notifyFd; + fds[1].events = 0; + + auto count = poll(fds, 2, -1); + if (count == -1) { + if (errno == EINTR || errno == EAGAIN) { + continue; + } else { + throw SysError("in MonitorFdHup poll()"); + } + } + + if (fds[0].revents & POLLHUP) { + unix::triggerInterrupt(); + break; + } + + if (fds[1].revents & POLLHUP) { + // Notify pipe closed, exit thread + break; + } + } +} +#endif + +inline MonitorFdHup::MonitorFdHup(int fd) +{ + notifyPipe.create(); + int notifyFd = notifyPipe.readSide.get(); + thread = std::thread([this, fd, notifyFd]() { this->runThread(fd, notifyFd); }); +}; + } // namespace nix diff --git a/src/libutil/unix/include/nix/util/signals-impl.hh b/src/libutil/unix/include/nix/util/signals-impl.hh index 1bcc90cdf67..2456119beba 100644 --- a/src/libutil/unix/include/nix/util/signals-impl.hh +++ b/src/libutil/unix/include/nix/util/signals-impl.hh @@ -42,13 +42,6 @@ extern thread_local std::function interruptCheck; void _interrupted(); -/** - * Sets the signal mask. Like saveSignalMask() but for a signal set that doesn't - * necessarily match the current thread's mask. - * See saveSignalMask() to set the saved mask to the current mask. - */ -void setChildSignalMask(sigset_t * sigs); - /** * Start a thread that handles various signals. Also block those signals * on the current thread (and thus any threads created by it). @@ -60,8 +53,6 @@ void startSignalHandlerThread(); /** * Saves the signal mask, which is the signal mask that nix will restore * before creating child processes. - * See setChildSignalMask() to set an arbitrary signal mask instead of the - * current mask. */ void saveSignalMask(); diff --git a/src/libutil/unix/meson.build b/src/libutil/unix/meson.build index 13bb380b4f3..8f89b65ab65 100644 --- a/src/libutil/unix/meson.build +++ b/src/libutil/unix/meson.build @@ -49,6 +49,7 @@ config_unix_priv_h = configure_file( sources += config_unix_priv_h sources += files( + 'current-process.cc', 'environment-variables.cc', 'file-descriptor.cc', 'file-path.cc', diff --git a/src/libutil/unix/signals.cc b/src/libutil/unix/signals.cc index 8a94cc2b150..d6efd6aa7b1 100644 --- a/src/libutil/unix/signals.cc +++ b/src/libutil/unix/signals.cc @@ -99,26 +99,6 @@ void unix::triggerInterrupt() static sigset_t savedSignalMask; static bool savedSignalMaskIsSet = false; -void unix::setChildSignalMask(sigset_t * sigs) -{ - assert(sigs); // C style function, but think of sigs as a reference - -#if (defined(_POSIX_C_SOURCE) && _POSIX_C_SOURCE >= 1) || (defined(_XOPEN_SOURCE) && _XOPEN_SOURCE) \ - || (defined(_POSIX_SOURCE) && _POSIX_SOURCE) - sigemptyset(&savedSignalMask); - // There's no "assign" or "copy" function, so we rely on (math) idempotence - // of the or operator: a or a = a. - sigorset(&savedSignalMask, sigs, sigs); -#else - // Without sigorset, our best bet is to assume that sigset_t is a type that - // can be assigned directly, such as is the case for a sigset_t defined as - // an integer type. - savedSignalMask = *sigs; -#endif - - savedSignalMaskIsSet = true; -} - void unix::saveSignalMask() { if (sigprocmask(SIG_BLOCK, nullptr, &savedSignalMask)) diff --git a/src/libutil/url.cc b/src/libutil/url.cc index 73e8cc1811a..7410e4062bd 100644 --- a/src/libutil/url.cc +++ b/src/libutil/url.cc @@ -3,6 +3,7 @@ #include "nix/util/util.hh" #include "nix/util/split.hh" #include "nix/util/canon-path.hh" +#include "nix/util/strings-inline.hh" #include @@ -108,6 +109,8 @@ static std::string percentEncodeCharSet(std::string_view s, auto charSet) return res; } +static ParsedURL fromBoostUrlView(boost::urls::url_view url, bool lenient); + ParsedURL parseURL(std::string_view url, bool lenient) try { /* Account for several non-standard properties of nix urls (for back-compat): @@ -149,10 +152,15 @@ try { }(); } - auto urlView = boost::urls::url_view(lenient ? fixedEncodedUrl : url); + return fromBoostUrlView(boost::urls::url_view(lenient ? fixedEncodedUrl : url), lenient); +} catch (boost::system::system_error & e) { + throw BadURL("'%s' is not a valid URL: %s", url, e.code().message()); +} +static ParsedURL fromBoostUrlView(boost::urls::url_view urlView, bool lenient) +{ if (!urlView.has_scheme()) - throw BadURL("'%s' doesn't have a scheme", url); + throw BadURL("'%s' doesn't have a scheme", urlView.buffer()); auto scheme = urlView.scheme(); auto authority = [&]() -> std::optional { @@ -170,13 +178,16 @@ try { * scheme considers a missing authority or empty host invalid. */ auto transportIsFile = parseUrlScheme(scheme).transport == "file"; if (authority && authority->host.size() && transportIsFile) - throw BadURL("file:// URL '%s' has unexpected authority '%s'", url, *authority); + throw BadURL("file:// URL '%s' has unexpected authority '%s'", urlView.buffer(), *authority); - auto path = urlView.path(); /* Does pct-decoding */ auto fragment = urlView.fragment(); /* Does pct-decoding */ - if (transportIsFile && path.empty()) - path = "/"; + boost::core::string_view encodedPath = urlView.encoded_path(); + if (transportIsFile && encodedPath.empty()) + encodedPath = "/"; + + auto path = std::views::transform(splitString>(encodedPath, "/"), percentDecode) + | std::ranges::to>(); /* Get the raw query. Store URI supports smuggling doubly nested queries, where the inner &/? are pct-encoded. */ @@ -185,12 +196,62 @@ try { return ParsedURL{ .scheme = scheme, .authority = authority, - .path = path, + .path = std::move(path), .query = decodeQuery(query, lenient), .fragment = fragment, }; -} catch (boost::system::system_error & e) { - throw BadURL("'%s' is not a valid URL: %s", url, e.code().message()); +} + +ParsedURL parseURLRelative(std::string_view urlS, const ParsedURL & base) +try { + + boost::urls::url resolved; + + try { + resolved.set_scheme(base.scheme); + if (base.authority) { + auto & authority = *base.authority; + resolved.set_host_address(authority.host); + if (authority.user) + resolved.set_user(*authority.user); + if (authority.password) + resolved.set_password(*authority.password); + if (authority.port) + resolved.set_port_number(*authority.port); + } + resolved.set_encoded_path(encodeUrlPath(base.path)); + resolved.set_encoded_query(encodeQuery(base.query)); + resolved.set_fragment(base.fragment); + } catch (boost::system::system_error & e) { + throw BadURL("'%s' is not a valid URL: %s", base.to_string(), e.code().message()); + } + + boost::urls::url_view url; + try { + url = urlS; + resolved.resolve(url).value(); + } catch (boost::system::system_error & e) { + throw BadURL("'%s' is not a valid URL: %s", urlS, e.code().message()); + } + + auto ret = fromBoostUrlView(resolved, /*lenient=*/false); + + /* Hack: Boost `url_view` supports Zone IDs, but `url` does not. + Just manually take the authority from the original URL to work + around it. See https://github.com/boostorg/url/issues/919 for + details. */ + if (!url.has_authority()) { + ret.authority = base.authority; + } + + /* Hack, work around fragment of base URL improperly being preserved + https://github.com/boostorg/url/issues/920 */ + ret.fragment = url.has_fragment() ? std::string{url.fragment()} : ""; + + return ret; +} catch (BadURL & e) { + e.addTrace({}, "while resolving possibly-relative url '%s' against base URL '%s'", urlS, base); + throw; } std::string percentDecode(std::string_view in) @@ -234,7 +295,15 @@ try { } const static std::string allowedInQuery = ":@/?"; -const static std::string allowedInPath = ":@/"; +const static std::string allowedInPath = ":@"; + +std::string encodeUrlPath(std::span urlPath) +{ + std::vector encodedPath; + for (auto & p : urlPath) + encodedPath.push_back(percentEncode(p, allowedInPath)); + return concatStringsSep("/", encodedPath); +} std::string encodeQuery(const StringMap & ss) { @@ -251,10 +320,62 @@ std::string encodeQuery(const StringMap & ss) return res; } +Path renderUrlPathEnsureLegal(const std::vector & urlPath) +{ + for (const auto & comp : urlPath) { + /* This is only really valid for UNIX. Windows has more restrictions. */ + if (comp.contains('/')) + throw BadURL("URL path component '%s' contains '/', which is not allowed in file names", comp); + if (comp.contains(char(0))) + throw BadURL("URL path component '%s' contains NUL byte which is not allowed", comp); + } + + return concatStringsSep("/", urlPath); +} + +std::string ParsedURL::renderPath(bool encode) const +{ + if (encode) + return encodeUrlPath(path); + return concatStringsSep("/", path); +} + +std::string ParsedURL::renderAuthorityAndPath() const +{ + std::string res; + /* The following assertions correspond to 3.3. Path [rfc3986]. URL parser + will never violate these properties, but hand-constructed ParsedURLs might. */ + if (authority.has_value()) { + /* If a URI contains an authority component, then the path component + must either be empty or begin with a slash ("/") character. */ + assert(path.empty() || path.front().empty()); + res += authority->to_string(); + } else if (std::ranges::equal(std::views::take(path, 3), std::views::repeat("", 3))) { + /* If a URI does not contain an authority component, then the path cannot begin + with two slash characters ("//") */ + unreachable(); + } + res += encodeUrlPath(path); + return res; +} + std::string ParsedURL::to_string() const { - return scheme + ":" + (authority ? "//" + authority->to_string() : "") + percentEncode(path, allowedInPath) - + (query.empty() ? "" : "?" + encodeQuery(query)) + (fragment.empty() ? "" : "#" + percentEncode(fragment)); + std::string res; + res += scheme; + res += ":"; + if (authority.has_value()) + res += "//"; + res += renderAuthorityAndPath(); + if (!query.empty()) { + res += "?"; + res += encodeQuery(query); + } + if (!fragment.empty()) { + res += "#"; + res += percentEncode(fragment); + } + return res; } std::ostream & operator<<(std::ostream & os, const ParsedURL & url) @@ -266,7 +387,7 @@ std::ostream & operator<<(std::ostream & os, const ParsedURL & url) ParsedURL ParsedURL::canonicalise() { ParsedURL res(*this); - res.path = CanonPath(res.path).abs(); + res.path = splitString>(CanonPath(renderPath()).abs(), "/"); return res; } @@ -287,17 +408,21 @@ ParsedUrlScheme parseUrlScheme(std::string_view scheme) }; } -std::string fixGitURL(const std::string & url) +ParsedURL fixGitURL(const std::string & url) { std::regex scpRegex("([^/]*)@(.*):(.*)"); if (!hasPrefix(url, "/") && std::regex_match(url, scpRegex)) - return std::regex_replace(url, scpRegex, "ssh://$1@$2/$3"); + return parseURL(std::regex_replace(url, scpRegex, "ssh://$1@$2/$3")); if (hasPrefix(url, "file:")) - return url; + return parseURL(url); if (url.find("://") == std::string::npos) { - return (ParsedURL{.scheme = "file", .authority = ParsedURL::Authority{}, .path = url}).to_string(); + return ParsedURL{ + .scheme = "file", + .authority = ParsedURL::Authority{}, + .path = splitString>(url, "/"), + }; } - return url; + return parseURL(url); } // https://www.rfc-editor.org/rfc/rfc3986#section-3.1 @@ -309,4 +434,10 @@ bool isValidSchemeName(std::string_view s) return std::regex_match(s.begin(), s.end(), regex, std::regex_constants::match_default); } +std::ostream & operator<<(std::ostream & os, const VerbatimURL & url) +{ + os << url.to_string(); + return os; +} + } // namespace nix diff --git a/src/libutil/windows/current-process.cc b/src/libutil/windows/current-process.cc new file mode 100644 index 00000000000..4bc866bb3ef --- /dev/null +++ b/src/libutil/windows/current-process.cc @@ -0,0 +1,35 @@ +#include "nix/util/current-process.hh" +#include "nix/util/windows-error.hh" +#include + +#ifdef _WIN32 +# define WIN32_LEAN_AND_MEAN +# include + +namespace nix { + +std::chrono::microseconds getCpuUserTime() +{ + FILETIME creationTime; + FILETIME exitTime; + FILETIME kernelTime; + FILETIME userTime; + + if (!GetProcessTimes(GetCurrentProcess(), &creationTime, &exitTime, &kernelTime, &userTime)) { + auto lastError = GetLastError(); + throw windows::WinError(lastError, "failed to get CPU time"); + } + + ULARGE_INTEGER uLargeInt; + uLargeInt.LowPart = userTime.dwLowDateTime; + uLargeInt.HighPart = userTime.dwHighDateTime; + + // FILETIME stores units of 100 nanoseconds. + // Dividing by 10 gives microseconds. + std::chrono::microseconds microseconds(uLargeInt.QuadPart / 10); + + return microseconds; +} + +} // namespace nix +#endif // ifdef _WIN32 diff --git a/src/libutil/windows/meson.build b/src/libutil/windows/meson.build index 0c1cec49cac..fb4de2017d7 100644 --- a/src/libutil/windows/meson.build +++ b/src/libutil/windows/meson.build @@ -1,4 +1,5 @@ sources += files( + 'current-process.cc', 'environment-variables.cc', 'file-descriptor.cc', 'file-path.cc', diff --git a/src/nix/app.cc b/src/nix/app.cc index 412b53817b0..9d3ea6b352e 100644 --- a/src/nix/app.cc +++ b/src/nix/app.cc @@ -74,6 +74,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) std::visit( overloaded{ [&](const NixStringContextElem::DrvDeep & d) -> DerivedPath { + state.waitForPath(d.drvPath); /* We want all outputs of the drv */ return DerivedPath::Built{ .drvPath = makeConstantStorePathRef(d.drvPath), @@ -81,6 +82,7 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) }; }, [&](const NixStringContextElem::Built & b) -> DerivedPath { + state.waitForPath(*b.drvPath); return DerivedPath::Built{ .drvPath = b.drvPath, .outputs = OutputsSpec::Names{b.output}, @@ -91,6 +93,9 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) .path = o.path, }; }, + [&](const NixStringContextElem::Path & p) -> DerivedPath { + throw Error("'program' attribute of an 'app' output cannot have no context"); + }, }, c.raw)); } @@ -103,11 +108,11 @@ UnresolvedApp InstallableValue::toApp(EvalState & state) else if (type == "derivation") { auto drvPath = cursor->forceDerivation(); - auto outPath = cursor->getAttr(state.sOutPath)->getString(); - auto outputName = cursor->getAttr(state.sOutputName)->getString(); - auto name = cursor->getAttr(state.sName)->getString(); + auto outPath = cursor->getAttr(state.s.outPath)->getString(); + auto outputName = cursor->getAttr(state.s.outputName)->getString(); + auto name = cursor->getAttr(state.s.name)->getString(); auto aPname = cursor->maybeGetAttr("pname"); - auto aMeta = cursor->maybeGetAttr(state.sMeta); + auto aMeta = cursor->maybeGetAttr(state.s.meta); auto aMainProgram = aMeta ? aMeta->maybeGetAttr("mainProgram") : nullptr; auto mainProgram = aMainProgram ? aMainProgram->getString() : aPname ? aPname->getString() : DrvName(name).name; auto program = outPath + "/bin/" + mainProgram; diff --git a/src/nix/asan-options.cc b/src/nix/asan-options.cc new file mode 100644 index 00000000000..256f34cbed1 --- /dev/null +++ b/src/nix/asan-options.cc @@ -0,0 +1,6 @@ +extern "C" [[gnu::retain]] const char * __asan_default_options() +{ + // We leak a bunch of memory knowingly on purpose. It's not worthwhile to + // diagnose that memory being leaked for now. + return "abort_on_error=1:print_summary=1:detect_leaks=0"; +} diff --git a/src/nix/build-remote/build-remote.cc b/src/nix/build-remote/build-remote.cc index 11df8cc5eeb..ffb77ddf1c8 100644 --- a/src/nix/build-remote/build-remote.cc +++ b/src/nix/build-remote/build-remote.cc @@ -324,7 +324,7 @@ static int main_build_remote(int argc, char ** argv) drv.inputSrcs = store->parseStorePathSet(inputs); optResult = sshStore->buildDerivation(*drvPath, (const BasicDerivation &) drv); auto & result = *optResult; - if (!result.success()) { + if (auto * failureP = result.tryGetFailure()) { if (settings.keepFailed) { warn( "The failed build directory was kept on the remote builder due to `--keep-failed`.%s", @@ -333,7 +333,7 @@ static int main_build_remote(int argc, char ** argv) : ""); } throw Error( - "build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, result.errorMsg); + "build of '%s' on '%s' failed: %s", store->printStorePath(*drvPath), storeUri, failureP->errorMsg); } } else { copyClosure(*store, *sshStore, StorePathSet{*drvPath}, NoRepair, NoCheckSigs, substitute); @@ -357,11 +357,14 @@ static int main_build_remote(int argc, char ** argv) debug("missing output %s", outputName); assert(optResult); auto & result = *optResult; - auto i = result.builtOutputs.find(outputName); - assert(i != result.builtOutputs.end()); - auto & newRealisation = i->second; - missingRealisations.insert(newRealisation); - missingPaths.insert(newRealisation.outPath); + if (auto * successP = result.tryGetSuccess()) { + auto & success = *successP; + auto i = success.builtOutputs.find(outputName); + assert(i != success.builtOutputs.end()); + auto & newRealisation = i->second; + missingRealisations.insert(newRealisation); + missingPaths.insert(newRealisation.outPath); + } } } } else { diff --git a/src/nix/build.cc b/src/nix/build.cc index eb47c31337a..2d4f426a495 100644 --- a/src/nix/build.cc +++ b/src/nix/build.cc @@ -8,11 +8,79 @@ using namespace nix; +/* This serialization code is diferent from the canonical (single) + derived path serialization because: + + - It looks up output paths where possible + + - It includes the store dir in store paths + + We might want to replace it with the canonical format at some point, + but that would be a breaking change (to a still-experimental but + widely-used command, so that isn't being done at this time just yet. + */ + +static nlohmann::json toJSON(Store & store, const SingleDerivedPath::Opaque & o) +{ + return store.printStorePath(o.path); +} + +static nlohmann::json toJSON(Store & store, const SingleDerivedPath & sdp); +static nlohmann::json toJSON(Store & store, const DerivedPath & dp); + +static nlohmann::json toJSON(Store & store, const SingleDerivedPath::Built & sdpb) +{ + nlohmann::json res; + res["drvPath"] = toJSON(store, *sdpb.drvPath); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *sdpb.drvPath)); + res["output"] = sdpb.output; + auto outputPathIter = outputMap.find(sdpb.output); + if (outputPathIter == outputMap.end()) + res["outputPath"] = nullptr; + else if (std::optional p = outputPathIter->second) + res["outputPath"] = store.printStorePath(*p); + else + res["outputPath"] = nullptr; + return res; +} + +static nlohmann::json toJSON(Store & store, const DerivedPath::Built & dpb) +{ + nlohmann::json res; + res["drvPath"] = toJSON(store, *dpb.drvPath); + // Fallback for the input-addressed derivation case: We expect to always be + // able to print the output paths, so let’s do it + // FIXME try-resolve on drvPath + const auto outputMap = store.queryPartialDerivationOutputMap(resolveDerivedPath(store, *dpb.drvPath)); + for (const auto & [output, outputPathOpt] : outputMap) { + if (!dpb.outputs.contains(output)) + continue; + if (outputPathOpt) + res["outputs"][output] = store.printStorePath(*outputPathOpt); + else + res["outputs"][output] = nullptr; + } + return res; +} + +static nlohmann::json toJSON(Store & store, const SingleDerivedPath & sdp) +{ + return std::visit([&](const auto & buildable) { return toJSON(store, buildable); }, sdp.raw()); +} + +static nlohmann::json toJSON(Store & store, const DerivedPath & dp) +{ + return std::visit([&](const auto & buildable) { return toJSON(store, buildable); }, dp.raw()); +} + static nlohmann::json derivedPathsToJSON(const DerivedPaths & paths, Store & store) { auto res = nlohmann::json::array(); for (auto & t : paths) { - std::visit([&](const auto & t) { res.push_back(t.toJSON(store)); }, t.raw()); + res.push_back(toJSON(store, t)); } return res; } @@ -22,22 +90,18 @@ builtPathsWithResultToJSON(const std::vector & buildables, { auto res = nlohmann::json::array(); for (auto & b : buildables) { - std::visit( - [&](const auto & t) { - auto j = t.toJSON(store); - if (b.result) { - if (b.result->startTime) - j["startTime"] = b.result->startTime; - if (b.result->stopTime) - j["stopTime"] = b.result->stopTime; - if (b.result->cpuUser) - j["cpuUser"] = ((double) b.result->cpuUser->count()) / 1000000; - if (b.result->cpuSystem) - j["cpuSystem"] = ((double) b.result->cpuSystem->count()) / 1000000; - } - res.push_back(j); - }, - b.path.raw()); + auto j = b.path.toJSON(store); + if (b.result) { + if (b.result->startTime) + j["startTime"] = b.result->startTime; + if (b.result->stopTime) + j["stopTime"] = b.result->stopTime; + if (b.result->cpuUser) + j["cpuUser"] = ((double) b.result->cpuUser->count()) / 1000000; + if (b.result->cpuSystem) + j["cpuSystem"] = ((double) b.result->cpuSystem->count()) / 1000000; + } + res.push_back(j); } return res; } diff --git a/src/nix/bundle.cc b/src/nix/bundle.cc index 29960c281d4..e11f37b847e 100644 --- a/src/nix/bundle.cc +++ b/src/nix/bundle.cc @@ -100,7 +100,7 @@ struct CmdBundle : InstallableValueCommand if (!evalState->isDerivation(*vRes)) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); - auto attr1 = vRes->attrs()->get(evalState->sDrvPath); + auto attr1 = vRes->attrs()->get(evalState->s.drvPath); if (!attr1) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); @@ -109,7 +109,7 @@ struct CmdBundle : InstallableValueCommand drvPath.requireDerivation(); - auto attr2 = vRes->attrs()->get(evalState->sOutPath); + auto attr2 = vRes->attrs()->get(evalState->s.outPath); if (!attr2) throw Error("the bundler '%s' does not produce a derivation", bundler.what()); @@ -123,7 +123,7 @@ struct CmdBundle : InstallableValueCommand }); if (!outLink) { - auto * attr = vRes->attrs()->get(evalState->sName); + auto * attr = vRes->attrs()->get(evalState->s.name); if (!attr) throw Error("attribute 'name' missing"); outLink = evalState->forceStringNoCtx(*attr->value, attr->pos, ""); diff --git a/src/nix/cat.cc b/src/nix/cat.cc index 276e01f5d59..145336723f1 100644 --- a/src/nix/cat.cc +++ b/src/nix/cat.cc @@ -41,7 +41,10 @@ struct CmdCatStore : StoreCommand, MixCat void run(ref store) override { auto [storePath, rest] = store->toStorePath(path); - cat(store->getFSAccessor(), CanonPath{storePath.to_string()} / CanonPath{rest}); + auto accessor = store->getFSAccessor(storePath); + if (!accessor) + throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); + cat(ref{std::move(accessor)}, CanonPath{rest}); } }; diff --git a/src/nix/config-check.cc b/src/nix/config-check.cc index c04943eab5c..e1efb40ebec 100644 --- a/src/nix/config-check.cc +++ b/src/nix/config-check.cc @@ -100,7 +100,7 @@ struct CmdConfigCheck : StoreCommand ss << "Multiple versions of nix found in PATH:\n"; for (auto & dir : dirs) ss << " " << dir << "\n"; - return checkFail(toView(ss)); + return checkFail(ss.view()); } return checkPass("PATH contains only one nix version."); @@ -143,7 +143,7 @@ struct CmdConfigCheck : StoreCommand for (auto & dir : dirs) ss << " " << dir << "\n"; ss << "\n"; - return checkFail(toView(ss)); + return checkFail(ss.view()); } return checkPass("All profiles are gcroots."); @@ -162,7 +162,7 @@ struct CmdConfigCheck : StoreCommand << "sync with the daemon.\n\n" << "Client protocol: " << formatProtocol(clientProto) << "\n" << "Store protocol: " << formatProtocol(storeProto) << "\n\n"; - return checkFail(toView(ss)); + return checkFail(ss.view()); } return checkPass("Client protocol matches store protocol."); diff --git a/src/nix/copy.cc b/src/nix/copy.cc index 62e8b64f513..706edc6c9c5 100644 --- a/src/nix/copy.cc +++ b/src/nix/copy.cc @@ -5,10 +5,9 @@ using namespace nix; -struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile +struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile, MixNoCheckSigs { std::optional outLink; - CheckSigsFlag checkSigs = CheckSigs; SubstituteFlag substitute = NoSubstitute; @@ -24,13 +23,6 @@ struct CmdCopy : virtual CopyCommand, virtual BuiltPathsCommand, MixProfile .handler = {&outLink}, .completer = completePath, }); - - addFlag({ - .longName = "no-check-sigs", - .description = "Do not require that paths are signed by trusted keys.", - .handler = {&checkSigs, NoCheckSigs}, - }); - addFlag({ .longName = "substitute-on-destination", .shortName = 's', diff --git a/src/nix/derivation-add.cc b/src/nix/derivation-add.cc index 0f797bb206d..2d13aba52c9 100644 --- a/src/nix/derivation-add.cc +++ b/src/nix/derivation-add.cc @@ -33,7 +33,7 @@ struct CmdAddDerivation : MixDryRun, StoreCommand { auto json = nlohmann::json::parse(drainFD(STDIN_FILENO)); - auto drv = Derivation::fromJSON(*store, json); + auto drv = Derivation::fromJSON(json); auto drvPath = writeDerivation(*store, drv, NoRepair, /* read only */ dryRun); diff --git a/src/nix/derivation-show.cc b/src/nix/derivation-show.cc index 1a61ccd5cba..20e54bba76b 100644 --- a/src/nix/derivation-show.cc +++ b/src/nix/derivation-show.cc @@ -58,7 +58,7 @@ struct CmdShowDerivation : InstallablesCommand, MixPrintJSON if (!drvPath.isDerivation()) continue; - jsonRoot[store->printStorePath(drvPath)] = store->readDerivation(drvPath).toJSON(*store); + jsonRoot[drvPath.to_string()] = store->readDerivation(drvPath).toJSON(); } printJSON(jsonRoot); } diff --git a/src/nix/develop.cc b/src/nix/develop.cc index ed25e655d8f..e914d5f6cfb 100644 --- a/src/nix/develop.cc +++ b/src/nix/develop.cc @@ -1,5 +1,6 @@ #include "nix/util/config-global.hh" #include "nix/expr/eval.hh" +#include "nix/fetchers/fetch-settings.hh" #include "nix/cmd/installable-flake.hh" #include "nix/cmd/command-installable-value.hh" #include "nix/main/common-args.hh" @@ -227,11 +228,13 @@ const static std::string getEnvSh = #include "get-env.sh.gen.hh" ; -/* Given an existing derivation, return the shell environment as - initialised by stdenv's setup script. We do this by building a - modified derivation with the same dependencies and nearly the same - initial environment variables, that just writes the resulting - environment to a file and exits. */ +/** + * Given an existing derivation, return the shell environment as + * initialised by stdenv's setup script. We do this by building a + * modified derivation with the same dependencies and nearly the same + * initial environment variables, that just writes the resulting + * environment to a file and exits. + */ static StorePath getDerivationEnvironment(ref store, ref evalStore, const StorePath & drvPath) { auto drv = evalStore->derivationFromPath(drvPath); @@ -297,12 +300,13 @@ static StorePath getDerivationEnvironment(ref store, ref evalStore bmNormal, evalStore); + // `get-env.sh` will write its JSON output to an arbitrary output + // path, so return the first non-empty output path. for (auto & [_0, optPath] : evalStore->queryPartialDerivationOutputMap(shellDrvPath)) { assert(optPath); auto & outPath = *optPath; - assert(store->isValidPath(outPath)); - auto outPathS = store->toRealPath(outPath); - if (lstat(outPathS).st_size) + auto st = store->getFSAccessor()->lstat(CanonPath(outPath.to_string())); + if (st.fileSize.value_or(0)) return outPath; } @@ -492,17 +496,15 @@ struct Common : InstallableCommand, MixProfile } } - std::pair getBuildEnvironment(ref store, ref installable) + std::pair getBuildEnvironment(ref store, ref installable) { auto shellOutPath = getShellOutPath(store, installable); - auto strPath = store->printStorePath(shellOutPath); - updateProfile(shellOutPath); - debug("reading environment file '%s'", strPath); + debug("reading environment file '%s'", store->printStorePath(shellOutPath)); - return {BuildEnvironment::parseJSON(readFile(store->toRealPath(shellOutPath))), strPath}; + return {BuildEnvironment::parseJSON(store->getFSAccessor()->readFile(shellOutPath.to_string())), shellOutPath}; } }; @@ -627,13 +629,12 @@ struct CmdDevelop : Common, MixEnvironment fmt("[ -n \"$PS1\" ] && PS1+=%s;\n", escapeShellArgAlways(developSettings.bashPromptSuffix.get())); } - writeFull(rcFileFd.get(), script); - setEnviron(); // prevent garbage collection until shell exits - setEnv("NIX_GCROOT", gcroot.c_str()); + setEnv("NIX_GCROOT", store->printStorePath(gcroot).c_str()); Path shell = "bash"; + bool foundInteractive = false; try { auto state = getEvalState(); @@ -656,19 +657,17 @@ struct CmdDevelop : Common, MixEnvironment Strings{"legacyPackages." + settings.thisSystem.get() + "."}, nixpkgsLockFlags); - bool found = false; - for (auto & path : Installable::toStorePathSet( getEvalStore(), store, Realise::Outputs, OperateOn::Output, {bashInstallable})) { auto s = store->printStorePath(path) + "/bin/bash"; if (pathExists(s)) { shell = s; - found = true; + foundInteractive = true; break; } } - if (!found) + if (!foundInteractive) throw Error("package 'nixpkgs#bashInteractive' does not provide a 'bin/bash'"); } catch (Error &) { @@ -678,6 +677,11 @@ struct CmdDevelop : Common, MixEnvironment // Override SHELL with the one chosen for this environment. // This is to make sure the system shell doesn't leak into the build environment. setEnv("SHELL", shell.c_str()); + // https://github.com/NixOS/nix/issues/5873 + script += fmt("SHELL=\"%s\"\n", shell); + if (foundInteractive) + script += fmt("PATH=\"%s${PATH:+:$PATH}\"\n", std::filesystem::path(shell).parent_path()); + writeFull(rcFileFd.get(), script); #ifdef _WIN32 // TODO re-enable on Windows throw UnimplementedError("Cannot yet spawn processes on Windows"); diff --git a/src/nix/diff-closures.cc b/src/nix/diff-closures.cc index cbf842e5cce..0ce83628d33 100644 --- a/src/nix/diff-closures.cc +++ b/src/nix/diff-closures.cc @@ -54,10 +54,10 @@ GroupedPaths getClosureInfo(ref store, const StorePath & toplevel) std::string showVersions(const StringSet & versions) { if (versions.empty()) - return "∅"; + return "(absent)"; StringSet versions2; for (auto & version : versions) - versions2.insert(version.empty() ? "ε" : version); + versions2.insert(version.empty() ? "(no version)" : version); return concatStringsSep(", ", versions2); } @@ -104,8 +104,13 @@ void printClosureDiff( if (showDelta || !removed.empty() || !added.empty()) { std::vector items; - if (!removed.empty() || !added.empty()) + if (!removed.empty() && !added.empty()) { items.push_back(fmt("%s → %s", showVersions(removed), showVersions(added))); + } else if (!removed.empty()) { + items.push_back(fmt("%s removed", showVersions(removed))); + } else if (!added.empty()) { + items.push_back(fmt("%s added", showVersions(added))); + } if (showDelta) items.push_back( fmt("%s%+.1f KiB" ANSI_NORMAL, sizeDelta > 0 ? ANSI_RED : ANSI_GREEN, sizeDelta / 1024.0)); diff --git a/src/nix/diff-closures.md b/src/nix/diff-closures.md index 0294c0d8def..6b07af28f95 100644 --- a/src/nix/diff-closures.md +++ b/src/nix/diff-closures.md @@ -11,8 +11,8 @@ R""( baloo-widgets: 20.08.1 → 20.08.2 bluez-qt: +12.6 KiB dolphin: 20.08.1 → 20.08.2, +13.9 KiB - kdeconnect: 20.08.2 → ∅, -6597.8 KiB - kdeconnect-kde: ∅ → 20.08.2, +6599.7 KiB + kdeconnect: 20.08.2 removed, -6597.8 KiB + kdeconnect-kde: 20.08.2 added, +6599.7 KiB … ``` @@ -34,9 +34,9 @@ dolphin: 20.08.1 → 20.08.2, +13.9 KiB No size change is shown if it's below the threshold. If the package does not exist in either the *before* or *after* closures, it is -represented using `∅` (empty set) on the appropriate side of the -arrow. If a package has an empty version string, the version is -rendered as `ε` (epsilon). +represented using `added` or `removed`. +If a package has an empty version string, the version is +rendered as `(no version)`. There may be multiple versions of a package in each closure. In that case, only the changed versions are shown. Thus, diff --git a/src/nix/dump-path.cc b/src/nix/dump-path.cc index 8475655e927..f375b0ac8e4 100644 --- a/src/nix/dump-path.cc +++ b/src/nix/dump-path.cc @@ -4,6 +4,14 @@ using namespace nix; +static FdSink getNarSink() +{ + auto fd = getStandardOutput(); + if (isatty(fd)) + throw UsageError("refusing to write NAR to a terminal"); + return FdSink(std::move(fd)); +} + struct CmdDumpPath : StorePathCommand { std::string description() override @@ -20,7 +28,7 @@ struct CmdDumpPath : StorePathCommand void run(ref store, const StorePath & storePath) override { - FdSink sink(getStandardOutput()); + auto sink = getNarSink(); store->narFromPath(storePath, sink); sink.flush(); } @@ -51,7 +59,7 @@ struct CmdDumpPath2 : Command void run() override { - FdSink sink(getStandardOutput()); + auto sink = getNarSink(); dumpPath(path, sink); sink.flush(); } diff --git a/src/nix/env.cc b/src/nix/env.cc index d91ee72d738..a80bcda6707 100644 --- a/src/nix/env.cc +++ b/src/nix/env.cc @@ -1,12 +1,14 @@ -#include #include +#include + #include "nix/cmd/command.hh" #include "nix/expr/eval.hh" #include "run.hh" #include "nix/util/strings.hh" #include "nix/util/executable-path.hh" #include "nix/util/environment-variables.hh" +#include "nix/util/mounted-source-accessor.hh" using namespace nix; @@ -71,7 +73,7 @@ struct CmdShell : InstallablesCommand, MixEnvironment auto outPaths = Installable::toStorePaths(getEvalStore(), store, Realise::Outputs, OperateOn::Output, installables); - std::unordered_set done; + boost::unordered_flat_set> done; std::queue todo; for (auto & path : outPaths) todo.push(path); diff --git a/src/nix/eval.cc b/src/nix/eval.cc index 10d0a184187..33c091a3511 100644 --- a/src/nix/eval.cc +++ b/src/nix/eval.cc @@ -120,11 +120,14 @@ struct CmdEval : MixJSON, InstallableValueCommand, MixReadOnlyOption logger->stop(); writeFull( getStandardOutput(), - *state->coerceToString(noPos, *v, context, "while generating the eval command output")); + state->devirtualize( + *state->coerceToString(noPos, *v, context, "while generating the eval command output"), context)); } else if (json) { - printJSON(printValueAsJSON(*state, true, *v, pos, context, false)); + // FIXME: use printJSON + auto j = printValueAsJSON(*state, true, *v, pos, context, false); + logger->cout("%s", state->devirtualize(outputPretty ? j.dump(2) : j.dump(), context)); } else { diff --git a/src/nix/flake-check.md b/src/nix/flake-check.md index c8307f8d85b..007640c27c9 100644 --- a/src/nix/flake-check.md +++ b/src/nix/flake-check.md @@ -31,39 +31,49 @@ at the first error. The following flake output attributes must be derivations: * `checks.`*system*`.`*name* -* `defaultPackage.`*system* -* `devShell.`*system* +* `devShells.`*system*`.default` * `devShells.`*system*`.`*name* * `nixosConfigurations.`*name*`.config.system.build.toplevel` +* `packages.`*system*`.default` * `packages.`*system*`.`*name* The following flake output attributes must be [app definitions](./nix3-run.md): +* `apps.`*system*`.default` * `apps.`*system*`.`*name* -* `defaultApp.`*system* The following flake output attributes must be [template definitions](./nix3-flake-init.md): -* `defaultTemplate` +* `templates.default` * `templates.`*name* The following flake output attributes must be *Nixpkgs overlays*: -* `overlay` +* `overlays.default` * `overlays.`*name* The following flake output attributes must be *NixOS modules*: -* `nixosModule` +* `nixosModules.default` * `nixosModules.`*name* The following flake output attributes must be [bundlers](./nix3-bundle.md): +* `bundlers.default` * `bundlers.`*name* -* `defaultBundler` + +Old default attributes are renamed, they will work but will emit a warning: + +* `defaultPackage.` → `packages.`*system*`.default` +* `defaultApps.` → `apps.`*system*`.default` +* `defaultTemplate` → `templates.default` +* `defaultBundler.` → `bundlers.`*system*`.default` +* `overlay` → `overlays.default` +* `devShell.` → `devShells.`*system*`.default` +* `nixosModule` → `nixosModules.default` In addition, the `hydraJobs` output is evaluated in the same way as Hydra's `hydra-eval-jobs` (i.e. as a arbitrarily deeply nested diff --git a/src/nix/flake-prefetch-inputs.cc b/src/nix/flake-prefetch-inputs.cc index 096eaf5392b..9ee4b546e70 100644 --- a/src/nix/flake-prefetch-inputs.cc +++ b/src/nix/flake-prefetch-inputs.cc @@ -47,8 +47,9 @@ struct CmdFlakePrefetchInputs : FlakeCommand try { Activity act(*logger, lvlInfo, actUnknown, fmt("fetching '%s'", lockedNode->lockedRef)); auto accessor = lockedNode->lockedRef.input.getAccessor(store).first; - fetchToStore( - fetchSettings, *store, accessor, FetchMode::Copy, lockedNode->lockedRef.input.getName()); + if (!evalSettings.lazyTrees) + fetchToStore( + fetchSettings, *store, accessor, FetchMode::Copy, lockedNode->lockedRef.input.getName()); } catch (Error & e) { printError("%s", e.what()); nrFailed++; diff --git a/src/nix/flake.cc b/src/nix/flake.cc index c04eab2919d..f92c3249994 100644 --- a/src/nix/flake.cc +++ b/src/nix/flake.cc @@ -17,6 +17,7 @@ #include "nix/fetchers/fetch-to-store.hh" #include "nix/store/local-fs-store.hh" #include "nix/store/globals.hh" +#include "nix/expr/parallel-eval.hh" #include #include @@ -132,6 +133,7 @@ struct CmdFlakeUpdate : FlakeCommand lockFlags.recreateLockFile = updateAll; lockFlags.writeLockFile = true; lockFlags.applyNixConfig = true; + lockFlags.requireLockable = false; lockFlake(); } @@ -164,6 +166,7 @@ struct CmdFlakeLock : FlakeCommand lockFlags.writeLockFile = true; lockFlags.failOnUnlocked = true; lockFlags.applyNixConfig = true; + lockFlags.requireLockable = false; lockFlake(); } @@ -212,11 +215,17 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON void run(nix::ref store) override { + lockFlags.requireLockable = false; auto lockedFlake = lockFlake(); auto & flake = lockedFlake.flake; - // Currently, all flakes are in the Nix store via the rootFS accessor. - auto storePath = store->printStorePath(store->toStorePath(flake.path.path.abs()).first); + /* Hack to show the store path if available. */ + std::optional storePath; + if (store->isInStore(flake.path.path.abs())) { + auto path = store->toStorePath(flake.path.path.abs()).first; + if (store->isValidPath(path)) + storePath = path; + } if (json) { nlohmann::json j; @@ -238,7 +247,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON j["revCount"] = *revCount; if (auto lastModified = flake.lockedRef.input.getLastModified()) j["lastModified"] = *lastModified; - j["path"] = storePath; + if (storePath) + j["path"] = store->printStorePath(*storePath); j["locks"] = lockedFlake.lockFile.toJSON().first; if (auto fingerprint = lockedFlake.getFingerprint(store, fetchSettings)) j["fingerprint"] = fingerprint->to_string(HashFormat::Base16, false); @@ -249,7 +259,8 @@ struct CmdFlakeMetadata : FlakeCommand, MixJSON logger->cout(ANSI_BOLD "Locked URL:" ANSI_NORMAL " %s", flake.lockedRef.to_string()); if (flake.description) logger->cout(ANSI_BOLD "Description:" ANSI_NORMAL " %s", *flake.description); - logger->cout(ANSI_BOLD "Path:" ANSI_NORMAL " %s", storePath); + if (storePath) + logger->cout(ANSI_BOLD "Path:" ANSI_NORMAL " %s", store->printStorePath(*storePath)); if (auto rev = flake.lockedRef.input.getRev()) logger->cout(ANSI_BOLD "Revision:" ANSI_NORMAL " %s", rev->to_string(HashFormat::Base16, false)); if (auto dirtyRev = fetchers::maybeGetStrAttr(flake.lockedRef.toAttrs(), "dirtyRev")) @@ -359,7 +370,7 @@ struct CmdFlakeCheck : FlakeCommand auto flake = lockFlake(); auto localSystem = std::string(settings.thisSystem.get()); - bool hasErrors = false; + std::atomic_bool hasErrors = false; auto reportError = [&](const Error & e) { try { throw e; @@ -374,7 +385,7 @@ struct CmdFlakeCheck : FlakeCommand } }; - StringSet omittedSystems; + Sync omittedSystems; // FIXME: rewrite to use EvalCache. @@ -393,7 +404,7 @@ struct CmdFlakeCheck : FlakeCommand auto checkSystemType = [&](std::string_view system, const PosIdx pos) { if (!checkAllSystems && system != localSystem) { - omittedSystems.insert(std::string(system)); + omittedSystems.lock()->insert(std::string(system)); return false; } else { return true; @@ -425,6 +436,8 @@ struct CmdFlakeCheck : FlakeCommand std::vector drvPaths; + FutureVector futures(*state->executor); + auto checkApp = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking app '%s'", attrPath)); @@ -493,9 +506,9 @@ struct CmdFlakeCheck : FlakeCommand } }; - std::function checkHydraJobs; + std::function checkHydraJobs; - checkHydraJobs = [&](std::string_view attrPath, Value & v, const PosIdx pos) { + checkHydraJobs = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking Hydra job '%s'", attrPath)); state->forceAttrs(v, pos, ""); @@ -503,15 +516,16 @@ struct CmdFlakeCheck : FlakeCommand if (state->isDerivation(v)) throw Error("jobset should not be a derivation at top-level"); - for (auto & attr : *v.attrs()) { - state->forceAttrs(*attr.value, attr.pos, ""); - auto attrPath2 = concatStrings(attrPath, ".", state->symbols[attr.name]); - if (state->isDerivation(*attr.value)) { - Activity act(*logger, lvlInfo, actUnknown, fmt("checking Hydra job '%s'", attrPath2)); - checkDerivation(attrPath2, *attr.value, attr.pos); - } else - checkHydraJobs(attrPath2, *attr.value, attr.pos); - } + for (auto & attr : *v.attrs()) + futures.spawn(1, [&, attrPath]() { + state->forceAttrs(*attr.value, attr.pos, ""); + auto attrPath2 = concatStrings(attrPath, ".", state->symbols[attr.name]); + if (state->isDerivation(*attr.value)) { + Activity act(*logger, lvlInfo, actUnknown, fmt("checking Hydra job '%s'", attrPath2)); + checkDerivation(attrPath2, *attr.value, attr.pos); + } else + checkHydraJobs(attrPath2, *attr.value, attr.pos); + }); } catch (Error & e) { e.addTrace(resolve(pos), HintFmt("while checking the Hydra jobset '%s'", attrPath)); @@ -522,7 +536,7 @@ struct CmdFlakeCheck : FlakeCommand auto checkNixOSConfiguration = [&](const std::string & attrPath, Value & v, const PosIdx pos) { try { Activity act(*logger, lvlInfo, actUnknown, fmt("checking NixOS configuration '%s'", attrPath)); - Bindings & bindings(*state->allocBindings(0)); + Bindings & bindings = Bindings::emptyBindings; auto vToplevel = findAlongAttrPath(*state, "config.system.build.toplevel", bindings, v).first; state->forceValue(*vToplevel, pos); if (!state->isDerivation(*vToplevel)) @@ -579,225 +593,259 @@ struct CmdFlakeCheck : FlakeCommand } }; - { + auto checkFlake = [&]() { Activity act(*logger, lvlInfo, actUnknown, "evaluating flake"); auto vFlake = state->allocValue(); flake::callFlake(*state, flake, *vFlake); enumerateOutputs(*state, *vFlake, [&](std::string_view name, Value & vOutput, const PosIdx pos) { - Activity act(*logger, lvlInfo, actUnknown, fmt("checking flake output '%s'", name)); - - try { - evalSettings.enableImportFromDerivation.setDefault(name != "hydraJobs"); - - state->forceValue(vOutput, pos); - - std::string_view replacement = name == "defaultPackage" ? "packages..default" - : name == "defaultApp" ? "apps..default" - : name == "defaultTemplate" ? "templates.default" - : name == "defaultBundler" ? "bundlers..default" - : name == "overlay" ? "overlays.default" - : name == "devShell" ? "devShells..default" - : name == "nixosModule" ? "nixosModules.default" - : ""; - if (replacement != "") - warn("flake output attribute '%s' is deprecated; use '%s' instead", name, replacement); - - if (name == "checks") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - std::string_view attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs()) { - auto drvPath = checkDerivation( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, - attr2.pos); - if (drvPath && attr_name == settings.thisSystem.get()) { - auto path = DerivedPath::Built{ - .drvPath = makeConstantStorePathRef(*drvPath), - .outputs = OutputsSpec::All{}, - }; - drvPaths.push_back(std::move(path)); + futures.spawn(2, [&, name, pos]() { + Activity act(*logger, lvlInfo, actUnknown, fmt("checking flake output '%s'", name)); + + try { + evalSettings.enableImportFromDerivation.setDefault(name != "hydraJobs"); + + state->forceValue(vOutput, pos); + + std::string_view replacement = name == "defaultPackage" ? "packages..default" + : name == "defaultApp" ? "apps..default" + : name == "defaultTemplate" ? "templates.default" + : name == "defaultBundler" ? "bundlers..default" + : name == "overlay" ? "overlays.default" + : name == "devShell" ? "devShells..default" + : name == "nixosModule" ? "nixosModules.default" + : ""; + if (replacement != "") + warn("flake output attribute '%s' is deprecated; use '%s' instead", name, replacement); + + if (name == "checks") { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) + futures.spawn(3, [&, name]() { + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs()) { + auto drvPath = checkDerivation( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, + attr2.pos); + if (drvPath && attr_name == settings.thisSystem.get()) { + auto path = DerivedPath::Built{ + .drvPath = makeConstantStorePathRef(*drvPath), + .outputs = OutputsSpec::All{}, + }; + drvPaths.push_back(std::move(path)); + } + } } - } - } + }); } - } - else if (name == "formatter") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - checkDerivation(fmt("%s.%s", name, attr_name), *attr.value, attr.pos); - }; + else if (name == "formatter") { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkDerivation(fmt("%s.%s", name, attr_name), *attr.value, attr.pos); + }; + } } - } - else if (name == "packages" || name == "devShells") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs()) - checkDerivation( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, - attr2.pos); - }; + else if (name == "packages" || name == "devShells") { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) + futures.spawn(3, [&, name]() { + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs()) + checkDerivation( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, + attr2.pos); + }; + }); } - } - else if (name == "apps") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs()) - checkApp( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, - attr2.pos); - }; + else if (name == "apps") { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs()) + checkApp( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, + attr2.pos); + }; + } } - } - else if (name == "defaultPackage" || name == "devShell") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - checkDerivation(fmt("%s.%s", name, attr_name), *attr.value, attr.pos); - }; + else if (name == "defaultPackage" || name == "devShell") { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkDerivation(fmt("%s.%s", name, attr_name), *attr.value, attr.pos); + }; + } } - } - else if (name == "defaultApp") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - checkApp(fmt("%s.%s", name, attr_name), *attr.value, attr.pos); - }; + else if (name == "defaultApp") { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkApp(fmt("%s.%s", name, attr_name), *attr.value, attr.pos); + }; + } } - } - else if (name == "legacyPackages") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - checkSystemName(state->symbols[attr.name], attr.pos); - checkSystemType(state->symbols[attr.name], attr.pos); - // FIXME: do getDerivations? + else if (name == "legacyPackages") { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { + checkSystemName(state->symbols[attr.name], attr.pos); + checkSystemType(state->symbols[attr.name], attr.pos); + // FIXME: do getDerivations? + } } - } - else if (name == "overlay") - checkOverlay(name, vOutput, pos); + else if (name == "overlay") + checkOverlay(name, vOutput, pos); - else if (name == "overlays") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) - checkOverlay(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); - } + else if (name == "overlays") { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) + checkOverlay(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); + } - else if (name == "nixosModule") - checkModule(name, vOutput, pos); + else if (name == "nixosModule") + checkModule(name, vOutput, pos); - else if (name == "nixosModules") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) - checkModule(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); - } + else if (name == "nixosModules") { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) + checkModule(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); + } - else if (name == "nixosConfigurations") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) - checkNixOSConfiguration( - fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); - } + else if (name == "nixosConfigurations") { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) + checkNixOSConfiguration( + fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); + } - else if (name == "hydraJobs") - checkHydraJobs(name, vOutput, pos); + else if (name == "hydraJobs") + checkHydraJobs(std::string(name), vOutput, pos); - else if (name == "defaultTemplate") - checkTemplate(name, vOutput, pos); + else if (name == "defaultTemplate") + checkTemplate(name, vOutput, pos); - else if (name == "templates") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) - checkTemplate(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); - } + else if (name == "templates") { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) + checkTemplate(fmt("%s.%s", name, state->symbols[attr.name]), *attr.value, attr.pos); + } - else if (name == "defaultBundler") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - checkBundler(fmt("%s.%s", name, attr_name), *attr.value, attr.pos); - }; + else if (name == "defaultBundler") { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + checkBundler(fmt("%s.%s", name, attr_name), *attr.value, attr.pos); + }; + } } - } - else if (name == "bundlers") { - state->forceAttrs(vOutput, pos, ""); - for (auto & attr : *vOutput.attrs()) { - const auto & attr_name = state->symbols[attr.name]; - checkSystemName(attr_name, attr.pos); - if (checkSystemType(attr_name, attr.pos)) { - state->forceAttrs(*attr.value, attr.pos, ""); - for (auto & attr2 : *attr.value->attrs()) { - checkBundler( - fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), - *attr2.value, - attr2.pos); - } - }; + else if (name == "bundlers") { + state->forceAttrs(vOutput, pos, ""); + for (auto & attr : *vOutput.attrs()) { + const auto & attr_name = state->symbols[attr.name]; + checkSystemName(attr_name, attr.pos); + if (checkSystemType(attr_name, attr.pos)) { + state->forceAttrs(*attr.value, attr.pos, ""); + for (auto & attr2 : *attr.value->attrs()) { + checkBundler( + fmt("%s.%s.%s", name, attr_name, state->symbols[attr2.name]), + *attr2.value, + attr2.pos); + } + }; + } } - } - else if ( - name == "lib" || name == "darwinConfigurations" || name == "darwinModules" - || name == "flakeModule" || name == "flakeModules" || name == "herculesCI" - || name == "homeConfigurations" || name == "homeModule" || name == "homeModules" - || name == "nixopsConfigurations") - // Known but unchecked community attribute - ; + else if ( + name == "lib" || name == "darwinConfigurations" || name == "darwinModules" + || name == "flakeModule" || name == "flakeModules" || name == "herculesCI" + || name == "homeConfigurations" || name == "homeModule" || name == "homeModules" + || name == "nixopsConfigurations") + // Known but unchecked community attribute + ; - else - warn("unknown flake output '%s'", name); + else + warn("unknown flake output '%s'", name); - } catch (Error & e) { - e.addTrace(resolve(pos), HintFmt("while checking flake output '%s'", name)); - reportError(e); - } + } catch (Error & e) { + e.addTrace(resolve(pos), HintFmt("while checking flake output '%s'", name)); + reportError(e); + } + }); }); - } + }; + + futures.spawn(1, checkFlake); + futures.finishAll(); if (build && !drvPaths.empty()) { - Activity act(*logger, lvlInfo, actUnknown, fmt("running %d flake checks", drvPaths.size())); - store->buildPaths(drvPaths); + // TODO: This filtering of substitutable paths is a temporary workaround until + // https://github.com/NixOS/nix/issues/5025 (union stores) is implemented. + // + // Once union stores are available, this code should be replaced with a proper + // union store configuration. Ideally, we'd use a union of multiple destination + // stores to preserve the current behavior where different substituters can + // cache different check results. + // + // For now, we skip building derivations whose outputs are already available + // via substitution, as `nix flake check` only needs to verify buildability, + // not actually produce the outputs. + state->waitForAllPaths(); + auto missing = store->queryMissing(drvPaths); + // Only occurs if `drvPaths` contains a `DerivedPath::Opaque`, which should never happen + assert(missing.unknown.empty()); + + std::vector toBuild; + for (auto & path : missing.willBuild) { + toBuild.emplace_back( + DerivedPath::Built{ + .drvPath = makeConstantStorePathRef(path), + .outputs = OutputsSpec::All{}, + }); + } + + // FIXME: should start building while evaluating. + Activity act(*logger, lvlInfo, actUnknown, fmt("running %d flake checks", toBuild.size())); + store->buildPaths(toBuild); } + if (hasErrors) throw Error("some errors were encountered during the evaluation"); - if (!omittedSystems.empty()) { + if (!omittedSystems.lock()->empty()) { // TODO: empty system is not visible; render all as nix strings? warn( "The check omitted these incompatible systems: %s\n" "Use '--all-systems' to check all.", - concatStringsSep(", ", omittedSystems)); + concatStringsSep(", ", *omittedSystems.lock())); }; }; }; @@ -807,7 +855,7 @@ static Strings defaultTemplateAttrPaths = {"templates.default", "defaultTemplate struct CmdFlakeInitCommon : virtual Args, EvalCommand { - std::string templateUrl = "templates"; + std::string templateUrl = "https://flakehub.com/f/DeterminateSystems/flake-templates/0.1"; Path destDir; const LockFlags lockFlags{.writeLockFile = false}; @@ -974,7 +1022,7 @@ struct CmdFlakeNew : CmdFlakeInitCommon struct CmdFlakeClone : FlakeCommand { - Path destDir; + std::filesystem::path destDir; std::string description() override { @@ -1004,16 +1052,14 @@ struct CmdFlakeClone : FlakeCommand if (destDir.empty()) throw Error("missing flag '--dest'"); - getFlakeRef().resolve(store).input.clone(destDir); + getFlakeRef().resolve(store).input.clone(store, destDir); } }; -struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun +struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun, MixNoCheckSigs { std::string dstUri; - CheckSigsFlag checkSigs = CheckSigs; - SubstituteFlag substitute = NoSubstitute; CmdFlakeArchive() @@ -1024,11 +1070,6 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun .labels = {"store-uri"}, .handler = {&dstUri}, }); - addFlag({ - .longName = "no-check-sigs", - .description = "Do not require that paths are signed by trusted keys.", - .handler = {&checkSigs, NoCheckSigs}, - }); } std::string description() override @@ -1049,7 +1090,8 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun StorePathSet sources; - auto storePath = store->toStorePath(flake.flake.path.path.abs()).first; + auto storePath = dryRun ? flake.flake.lockedRef.input.computeStorePath(*store) + : std::get(flake.flake.lockedRef.input.fetchToStore(store)); sources.insert(storePath); @@ -1062,7 +1104,7 @@ struct CmdFlakeArchive : FlakeCommand, MixJSON, MixDryRun std::optional storePath; if (!(*inputNode)->lockedRef.input.isRelative()) { storePath = dryRun ? (*inputNode)->lockedRef.input.computeStorePath(*store) - : (*inputNode)->lockedRef.input.fetchToStore(store).first; + : std::get((*inputNode)->lockedRef.input.fetchToStore(store)); sources.insert(*storePath); } if (json) { @@ -1134,125 +1176,57 @@ struct CmdFlakeShow : FlakeCommand, MixJSON auto flake = std::make_shared(lockFlake()); auto localSystem = std::string(settings.thisSystem.get()); - std::function & attrPath, const Symbol & attr)> - hasContent; - - // For frameworks it's important that structures are as lazy as possible - // to prevent infinite recursions, performance issues and errors that - // aren't related to the thing to evaluate. As a consequence, they have - // to emit more attributes than strictly (sic) necessary. - // However, these attributes with empty values are not useful to the user - // so we omit them. - hasContent = - [&](eval_cache::AttrCursor & visitor, const std::vector & attrPath, const Symbol & attr) -> bool { - auto attrPath2(attrPath); - attrPath2.push_back(attr); - auto attrPathS = state->symbols.resolve(attrPath2); - const auto & attrName = state->symbols[attr]; + auto cache = openEvalCache(*state, flake); - auto visitor2 = visitor.getAttr(attrName); + auto j = nlohmann::json::object(); - try { - if ((attrPathS[0] == "apps" || attrPathS[0] == "checks" || attrPathS[0] == "devShells" - || attrPathS[0] == "legacyPackages" || attrPathS[0] == "packages") - && (attrPathS.size() == 1 || attrPathS.size() == 2)) { - for (const auto & subAttr : visitor2->getAttrs()) { - if (hasContent(*visitor2, attrPath2, subAttr)) { - return true; - } - } - return false; - } + std::function visit; - if ((attrPathS.size() == 1) - && (attrPathS[0] == "formatter" || attrPathS[0] == "nixosConfigurations" - || attrPathS[0] == "nixosModules" || attrPathS[0] == "overlays")) { - for (const auto & subAttr : visitor2->getAttrs()) { - if (hasContent(*visitor2, attrPath2, subAttr)) { - return true; - } - } - return false; - } - - // If we don't recognize it, it's probably content - return true; - } catch (EvalError & e) { - // Some attrs may contain errors, e.g. legacyPackages of - // nixpkgs. We still want to recurse into it, instead of - // skipping it at all. - return true; - } - }; - - std::function & attrPath, - const std::string & headerPrefix, - const std::string & nextPrefix)> - visit; - - visit = [&](eval_cache::AttrCursor & visitor, - const std::vector & attrPath, - const std::string & headerPrefix, - const std::string & nextPrefix) -> nlohmann::json { - auto j = nlohmann::json::object(); + FutureVector futures(*state->executor); + visit = [&](eval_cache::AttrCursor & visitor, nlohmann::json & j) { + auto attrPath = visitor.getAttrPath(); auto attrPathS = state->symbols.resolve(attrPath); Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", concatStringsSep(".", attrPathS))); try { auto recurse = [&]() { - if (!json) - logger->cout("%s", headerPrefix); - std::vector attrs; for (const auto & attr : visitor.getAttrs()) { - if (hasContent(visitor, attrPath, attr)) - attrs.push_back(attr); - } - - for (const auto & [i, attr] : enumerate(attrs)) { const auto & attrName = state->symbols[attr]; - bool last = i + 1 == attrs.size(); auto visitor2 = visitor.getAttr(attrName); - auto attrPath2(attrPath); - attrPath2.push_back(attr); - auto j2 = visit( - *visitor2, - attrPath2, - fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, - nextPrefix, - last ? treeLast : treeConn, - attrName), - nextPrefix + (last ? treeNull : treeLine)); - if (json) - j.emplace(attrName, std::move(j2)); + auto & j2 = *j.emplace(attrName, nlohmann::json::object()).first; + futures.spawn(1, [&, visitor2]() { visit(*visitor2, j2); }); } }; auto showDerivation = [&]() { - auto name = visitor.getAttr(state->sName)->getString(); - - if (json) { - std::optional description; - if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) { - if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) - description = aDescription->getString(); - } - j.emplace("type", "derivation"); - j.emplace("name", name); - j.emplace("description", description ? *description : ""); - } else { - logger->cout( - "%s: %s '%s'", - headerPrefix, + auto name = visitor.getAttr(state->s.name)->getString(); + std::optional description; + if (auto aMeta = visitor.maybeGetAttr(state->s.meta)) { + if (auto aDescription = aMeta->maybeGetAttr(state->s.description)) + description = aDescription->getString(); + } + j.emplace("type", "derivation"); + if (!json) + j.emplace( + "subtype", attrPath.size() == 2 && attrPathS[0] == "devShell" ? "development environment" : attrPath.size() >= 2 && attrPathS[0] == "devShells" ? "development environment" : attrPath.size() == 3 && attrPathS[0] == "checks" ? "derivation" : attrPath.size() >= 1 && attrPathS[0] == "hydraJobs" ? "derivation" - : "package", - name); + : "package"); + j.emplace("name", name); + if (description) + j.emplace("description", *description); + }; + + auto omit = [&](std::string_view flag) { + if (json) + logger->warn(fmt("%s omitted (use '%s' to show)", concatStringsSep(".", attrPathS), flag)); + else { + j.emplace("type", "omitted"); + j.emplace("message", fmt(ANSI_WARNING "omitted" ANSI_NORMAL " (use '%s' to show)", flag)); } }; @@ -1274,14 +1248,7 @@ struct CmdFlakeShow : FlakeCommand, MixJSON || (attrPath.size() == 3 && (attrPathS[0] == "checks" || attrPathS[0] == "packages" || attrPathS[0] == "devShells"))) { if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { - if (!json) - logger->cout( - fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", - headerPrefix)); - else { - logger->warn( - fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); - } + omit("--all-systems"); } else { try { if (visitor.isDerivation()) @@ -1289,15 +1256,8 @@ struct CmdFlakeShow : FlakeCommand, MixJSON else throw Error("expected a derivation"); } catch (IFDError & e) { - if (!json) { - logger->cout( - fmt("%s " ANSI_WARNING "omitted due to use of import from derivation" ANSI_NORMAL, - headerPrefix)); - } else { - logger->warn( - fmt("%s omitted due to use of import from derivation", - concatStringsSep(".", attrPathS))); - } + logger->warn(fmt( + "%s omitted due to use of import from derivation", concatStringsSep(".", attrPathS))); } } } @@ -1309,14 +1269,8 @@ struct CmdFlakeShow : FlakeCommand, MixJSON else recurse(); } catch (IFDError & e) { - if (!json) { - logger->cout( - fmt("%s " ANSI_WARNING "omitted due to use of import from derivation" ANSI_NORMAL, - headerPrefix)); - } else { - logger->warn(fmt( - "%s omitted due to use of import from derivation", concatStringsSep(".", attrPathS))); - } + logger->warn( + fmt("%s omitted due to use of import from derivation", concatStringsSep(".", attrPathS))); } } @@ -1324,21 +1278,9 @@ struct CmdFlakeShow : FlakeCommand, MixJSON if (attrPath.size() == 1) recurse(); else if (!showLegacy) { - if (!json) - logger->cout(fmt( - "%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--legacy' to show)", headerPrefix)); - else { - logger->warn(fmt("%s omitted (use '--legacy' to show)", concatStringsSep(".", attrPathS))); - } + omit("--legacy"); } else if (!showAllSystems && std::string(attrPathS[1]) != localSystem) { - if (!json) - logger->cout( - fmt("%s " ANSI_WARNING "omitted" ANSI_NORMAL " (use '--all-systems' to show)", - headerPrefix)); - else { - logger->warn( - fmt("%s omitted (use '--all-systems' to show)", concatStringsSep(".", attrPathS))); - } + omit("--all-systems"); } else { try { if (visitor.isDerivation()) @@ -1347,15 +1289,8 @@ struct CmdFlakeShow : FlakeCommand, MixJSON // FIXME: handle recurseIntoAttrs recurse(); } catch (IFDError & e) { - if (!json) { - logger->cout( - fmt("%s " ANSI_WARNING "omitted due to use of import from derivation" ANSI_NORMAL, - headerPrefix)); - } else { - logger->warn( - fmt("%s omitted due to use of import from derivation", - concatStringsSep(".", attrPathS))); - } + logger->warn(fmt( + "%s omitted due to use of import from derivation", concatStringsSep(".", attrPathS))); } } } @@ -1365,34 +1300,23 @@ struct CmdFlakeShow : FlakeCommand, MixJSON || (attrPath.size() == 3 && attrPathS[0] == "apps")) { auto aType = visitor.maybeGetAttr("type"); std::optional description; - if (auto aMeta = visitor.maybeGetAttr(state->sMeta)) { - if (auto aDescription = aMeta->maybeGetAttr(state->sDescription)) + if (auto aMeta = visitor.maybeGetAttr(state->s.meta)) { + if (auto aDescription = aMeta->maybeGetAttr(state->s.description)) description = aDescription->getString(); } if (!aType || aType->getString() != "app") state->error("not an app definition").debugThrow(); - if (json) { - j.emplace("type", "app"); - if (description) - j.emplace("description", *description); - } else { - logger->cout( - "%s: app: " ANSI_BOLD "%s" ANSI_NORMAL, - headerPrefix, - description ? *description : "no description"); - } + j.emplace("type", "app"); + if (description) + j.emplace("description", *description); } else if ( (attrPath.size() == 1 && attrPathS[0] == "defaultTemplate") || (attrPath.size() == 2 && attrPathS[0] == "templates")) { auto description = visitor.getAttr("description")->getString(); - if (json) { - j.emplace("type", "template"); - j.emplace("description", description); - } else { - logger->cout("%s: template: " ANSI_BOLD "%s" ANSI_NORMAL, headerPrefix, description); - } + j.emplace("type", "template"); + j.emplace("description", description); } else { @@ -1405,25 +1329,85 @@ struct CmdFlakeShow : FlakeCommand, MixJSON || (attrPath.size() == 2 && attrPathS[0] == "nixosModules") ? std::make_pair("nixos-module", "NixOS module") : std::make_pair("unknown", "unknown"); - if (json) { - j.emplace("type", type); - } else { - logger->cout("%s: " ANSI_WARNING "%s" ANSI_NORMAL, headerPrefix, description); - } + j.emplace("type", type); + j.emplace("description", description); } } catch (EvalError & e) { if (!(attrPath.size() > 0 && attrPathS[0] == "legacyPackages")) throw; } - - return j; }; - auto cache = openEvalCache(*state, flake); + futures.spawn(1, [&]() { visit(*cache->getRoot(), j); }); + futures.finishAll(); - auto j = visit(*cache->getRoot(), {}, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), ""); if (json) printJSON(j); + else { + + // For frameworks it's important that structures are as + // lazy as possible to prevent infinite recursions, + // performance issues and errors that aren't related to + // the thing to evaluate. As a consequence, they have to + // emit more attributes than strictly (sic) necessary. + // However, these attributes with empty values are not + // useful to the user so we omit them. + std::function hasContent; + + hasContent = [&](const nlohmann::json & j) -> bool { + if (j.find("type") != j.end()) + return true; + else { + for (auto & j2 : j) + if (hasContent(j2)) + return true; + return false; + } + }; + + // Render the JSON into a tree representation. + std::function + render; + + render = [&](nlohmann::json j, const std::string & headerPrefix, const std::string & nextPrefix) { + if (j.find("type") != j.end()) { + std::string s; + + std::string type = j["type"]; + if (type == "omitted") { + s = j["message"]; + } else if (type == "derivation") { + s = (std::string) j["subtype"] + " '" + (std::string) j["name"] + "'"; + } else { + s = type; + } + + logger->cout("%s: %s", headerPrefix, s); + return; + } + + logger->cout("%s", headerPrefix); + + auto nonEmpty = nlohmann::json::object(); + for (const auto & j2 : j.items()) { + if (hasContent(j2.value())) + nonEmpty[j2.key()] = j2.value(); + } + + for (const auto & [i, j2] : enumerate(nonEmpty.items())) { + bool last = i + 1 == nonEmpty.size(); + render( + j2.value(), + fmt(ANSI_GREEN "%s%s" ANSI_NORMAL ANSI_BOLD "%s" ANSI_NORMAL, + nextPrefix, + last ? treeLast : treeConn, + j2.key()), + nextPrefix + (last ? treeNull : treeLine)); + } + }; + + render(j, fmt(ANSI_BOLD "%s" ANSI_NORMAL, flake->flake.lockedRef), ""); + } } }; @@ -1507,12 +1491,6 @@ struct CmdFlake : NixMultiCommand #include "flake.md" ; } - - void run() override - { - experimentalFeatureSettings.require(Xp::Flakes); - NixMultiCommand::run(); - } }; static auto rCmdFlake = registerCommand("flake"); diff --git a/src/nix/get-env.sh b/src/nix/get-env.sh index 071edf9b94f..5b6162b4ba2 100644 --- a/src/nix/get-env.sh +++ b/src/nix/get-env.sh @@ -1,11 +1,14 @@ +# shellcheck shell=bash set -e +# shellcheck disable=SC1090 # Dynamic sourcing is intentional if [ -e "$NIX_ATTRS_SH_FILE" ]; then source "$NIX_ATTRS_SH_FILE"; fi export IN_NIX_SHELL=impure export dontAddDisableDepTrack=1 if [[ -n $stdenv ]]; then - source $stdenv/setup + # shellcheck disable=SC1091 # setup file is in nix store + source "$stdenv"/setup fi # Better to use compgen, but stdenv bash doesn't have it. @@ -14,13 +17,14 @@ __functions="$(declare -F)" __dumpEnv() { printf '{\n' + printf ' "version": 1,\n' printf ' "bashFunctions": {\n' local __first=1 - while read __line; do + while read -r __line; do if ! [[ $__line =~ ^declare\ -f\ (.*) ]]; then continue; fi __fun_name="${BASH_REMATCH[1]}" - __fun_body="$(type $__fun_name)" + __fun_body="$(type "$__fun_name")" if [[ $__fun_body =~ \{(.*)\} ]]; then if [[ -z $__first ]]; then printf ',\n'; else __first=; fi __fun_body="${BASH_REMATCH[1]}" @@ -37,7 +41,7 @@ __dumpEnv() { printf ' "variables": {\n' local __first=1 - while read __line; do + while read -r __line; do if ! [[ $__line =~ ^declare\ (-[^ ])\ ([^=]*) ]]; then continue; fi local type="${BASH_REMATCH[1]}" local __var_name="${BASH_REMATCH[2]}" @@ -76,7 +80,9 @@ __dumpEnv() { elif [[ $type == -a ]]; then printf '"type": "array", "value": [' local __first2=1 + # shellcheck disable=SC1087 # Complex array manipulation, syntax is correct __var_name="$__var_name[@]" + # shellcheck disable=SC1087 # Complex array manipulation, syntax is correct for __i in "${!__var_name}"; do if [[ -z $__first2 ]]; then printf ', '; else __first2=; fi __escapeString "$__i" @@ -142,6 +148,7 @@ __dumpEnvToOutput() { # array with a format like `outname => /nix/store/hash-drvname-outname`. # Otherwise it is a space-separated list of output variable names. if [ -e "$NIX_ATTRS_SH_FILE" ]; then + # shellcheck disable=SC2154 # outputs is set by sourced file for __output in "${outputs[@]}"; do __dumpEnvToOutput "$__output" done diff --git a/src/nix/ls.cc b/src/nix/ls.cc index dcc46fa1448..0565299e2f5 100644 --- a/src/nix/ls.cc +++ b/src/nix/ls.cc @@ -4,12 +4,13 @@ #include "nix/main/common-args.hh" #include +#include "ls.hh" + using namespace nix; -struct MixLs : virtual Args, MixJSON +struct MixLs : virtual Args, MixJSON, MixLongListing { bool recursive = false; - bool verbose = false; bool showDirectory = false; MixLs() @@ -21,13 +22,6 @@ struct MixLs : virtual Args, MixJSON .handler = {&recursive, true}, }); - addFlag({ - .longName = "long", - .shortName = 'l', - .description = "Show detailed file information.", - .handler = {&verbose, true}, - }); - addFlag({ .longName = "directory", .shortName = 'd', @@ -41,13 +35,13 @@ struct MixLs : virtual Args, MixJSON std::function doPath; auto showFile = [&](const CanonPath & curPath, std::string_view relPath) { - if (verbose) { + if (longListing) { auto st = accessor->lstat(curPath); std::string tp = st.type == SourceAccessor::Type::tRegular ? (st.isExecutable ? "-r-xr-xr-x" : "-r--r--r--") : st.type == SourceAccessor::Type::tSymlink ? "lrwxrwxrwx" : "dr-xr-xr-x"; - auto line = fmt("%s %20d %s", tp, st.fileSize.value_or(0), relPath); + auto line = fmt("%s %9d %s", tp, st.fileSize.value_or(0), relPath); if (st.type == SourceAccessor::Type::tSymlink) line += " -> " + accessor->readLink(curPath); logger->cout(line); @@ -115,7 +109,10 @@ struct CmdLsStore : StoreCommand, MixLs void run(ref store) override { auto [storePath, rest] = store->toStorePath(path); - list(store->getFSAccessor(), CanonPath{storePath.to_string()} / CanonPath{rest}); + auto accessor = store->getFSAccessor(storePath); + if (!accessor) + throw InvalidPath("path '%1%' is not a valid store path", store->printStorePath(storePath)); + list(ref{std::move(accessor)}, CanonPath{rest}); } }; diff --git a/src/nix/ls.hh b/src/nix/ls.hh new file mode 100644 index 00000000000..36e61162035 --- /dev/null +++ b/src/nix/ls.hh @@ -0,0 +1,22 @@ +#pragma once + +#include "nix/util/args.hh" + +namespace nix { + +struct MixLongListing : virtual Args +{ + bool longListing = false; + + MixLongListing() + { + addFlag({ + .longName = "long", + .shortName = 'l', + .description = "Show detailed file information.", + .handler = {&longListing, true}, + }); + } +}; + +} // namespace nix diff --git a/src/nix/main.cc b/src/nix/main.cc index a6077f5e9ad..3621b738fb0 100644 --- a/src/nix/main.cc +++ b/src/nix/main.cc @@ -86,6 +86,20 @@ static bool haveInternet() #endif } +static void disableNet() +{ + // FIXME: should check for command line overrides only. + if (!settings.useSubstitutes.overridden) + // FIXME: should not disable local substituters (like file:///). + settings.useSubstitutes = false; + if (!settings.tarballTtl.overridden) + settings.tarballTtl = std::numeric_limits::max(); + if (!fileTransferSettings.tries.overridden) + fileTransferSettings.tries = 0; + if (!fileTransferSettings.connectTimeout.overridden) + fileTransferSettings.connectTimeout = 1; +} + std::string programPath; struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs @@ -119,7 +133,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs .description = "Print full build logs on standard error.", .category = loggingCategory, .handler = {[&]() { logger->setPrintBuildLogs(true); }}, - .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -135,7 +148,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs .description = "Disable substituters and consider all previously downloaded files up-to-date.", .category = miscCategory, .handler = {[&]() { useNet = false; }}, - .experimentalFeature = Xp::NixCommand, }); addFlag({ @@ -143,7 +155,6 @@ struct NixArgs : virtual MultiCommand, virtual MixCommonArgs, virtual RootArgs .description = "Consider all previously downloaded files out-of-date.", .category = miscCategory, .handler = {[&]() { refresh = true; }}, - .experimentalFeature = Xp::NixCommand, }); aliases = { @@ -226,8 +237,8 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) auto mdName = subcommand.empty() ? "nix" : fmt("nix3-%s", concatStringsSep("-", subcommand)); - evalSettings.restrictEval = false; - evalSettings.pureEval = false; + evalSettings.restrictEval = true; + evalSettings.pureEval = true; EvalState state({}, openStore("dummy://"), fetchSettings, evalSettings); auto vGenerateManpage = state.allocValue(); @@ -256,8 +267,8 @@ static void showHelp(std::vector subcommand, NixArgs & toplevel) vDump->mkString(toplevel.dumpCli()); auto vRes = state.allocValue(); - state.callFunction(*vGenerateManpage, state.getBuiltin("false"), *vRes, noPos); - state.callFunction(*vRes, *vDump, *vRes, noPos); + Value * args[]{&state.getBuiltin("false"), vDump}; + state.callFunction(*vGenerateManpage, args, *vRes, noPos); auto attr = vRes->attrs()->get(state.symbols.create(mdName + ".md")); if (!attr) @@ -423,7 +434,6 @@ void mainWrapped(int argc, char ** argv) if (argc == 2 && std::string(argv[1]) == "__dump-language") { experimentalFeatureSettings.experimentalFeatures = { - Xp::Flakes, Xp::FetchClosure, Xp::DynamicDerivations, Xp::FetchTree, @@ -482,6 +492,12 @@ void mainWrapped(int argc, char ** argv) } }); + if (getEnv("NIX_GET_COMPLETIONS")) + /* Avoid fetching stuff during tab completion. We have to this + early because we haven't checked `haveInternet()` yet + (below). */ + disableNet(); + try { auto isNixCommand = std::regex_search(programName, std::regex("nix$")); auto allowShebang = isNixCommand && argc > 1; @@ -493,6 +509,8 @@ void mainWrapped(int argc, char ** argv) applyJSONLogger(); + printTalkative("Nix %s", version()); + if (args.helpRequested) { std::vector subcommand; MultiCommand * command = &args; @@ -525,17 +543,8 @@ void mainWrapped(int argc, char ** argv) args.useNet = false; } - if (!args.useNet) { - // FIXME: should check for command line overrides only. - if (!settings.useSubstitutes.overridden) - settings.useSubstitutes = false; - if (!settings.tarballTtl.overridden) - settings.tarballTtl = std::numeric_limits::max(); - if (!fileTransferSettings.tries.overridden) - fileTransferSettings.tries = 0; - if (!fileTransferSettings.connectTimeout.overridden) - fileTransferSettings.connectTimeout = 1; - } + if (!args.useNet) + disableNet(); if (args.refresh) { settings.tarballTtl = 0; @@ -561,13 +570,15 @@ void mainWrapped(int argc, char ** argv) int main(int argc, char ** argv) { + using namespace nix; + // The CLI has a more detailed version than the libraries; see nixVersion. - nix::nixVersion = NIX_CLI_VERSION; + nixVersion = NIX_CLI_VERSION; #ifndef _WIN32 // Increase the default stack size for the evaluator and for // libstdc++'s std::regex. - nix::setStackSize(64 * 1024 * 1024); + setStackSize(evalStackSize); #endif - return nix::handleExceptions(argv[0], [&]() { nix::mainWrapped(argc, argv); }); + return handleExceptions(argv[0], [&]() { mainWrapped(argc, argv); }); } diff --git a/src/nix/make-content-addressed.md b/src/nix/make-content-addressed.md index b1f7da525ff..e6a51c83ada 100644 --- a/src/nix/make-content-addressed.md +++ b/src/nix/make-content-addressed.md @@ -51,7 +51,7 @@ be verified without any additional information such as signatures. This means that a command like ```console -# nix store build /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10 \ +# nix build /nix/store/5skmmcb9svys5lj3kbsrjg7vf2irid63-hello-2.10 \ --substituters https://my-cache.example.org ``` diff --git a/src/nix/meson.build b/src/nix/meson.build index e989e80164f..4b4b85eb62a 100644 --- a/src/nix/meson.build +++ b/src/nix/meson.build @@ -56,11 +56,13 @@ config_priv_h = configure_file( ) subdir('nix-meson-build-support/common') +subdir('nix-meson-build-support/asan-options') subdir('nix-meson-build-support/generate-header') nix_sources = [ config_priv_h ] + files( 'add-to-store.cc', 'app.cc', + 'asan-options.cc', 'build.cc', 'bundle.cc', 'cat.cc', @@ -78,6 +80,7 @@ nix_sources = [ config_priv_h ] + files( 'env.cc', 'eval.cc', 'flake-prefetch-inputs.cc', + 'flake-prefetch-inputs.cc', 'flake.cc', 'formatter.cc', 'hash.cc', @@ -87,6 +90,7 @@ nix_sources = [ config_priv_h ] + files( 'make-content-addressed.cc', 'man-pages.cc', 'nar.cc', + 'nario.cc', 'optimise-store.cc', 'path-from-hash-part.cc', 'path-info.cc', diff --git a/src/nix/nario-export.md b/src/nix/nario-export.md new file mode 100644 index 00000000000..2480733c1ca --- /dev/null +++ b/src/nix/nario-export.md @@ -0,0 +1,29 @@ +R""( + +# Examples + +* Export the closure of the build of `nixpkgs#hello`: + + ```console + # nix nario export --format 2 -r nixpkgs#hello > dump.nario + ``` + + It can be imported into another store: + + ```console + # nix nario import --no-check-sigs < dump.nario + ``` + +# Description + +This command prints to standard output a serialization of the specified store paths in `nario` format. This serialization can be imported into another store using `nix nario import`. + +References of a path are not exported by default; use `-r` to export a complete closure. +Paths are exported in topologically sorted order (i.e. if path `X` refers to `Y`, then `Y` appears before `X`). +You must specify the desired `nario` version. Currently the following versions are supported: + +* `1`: This version is compatible with the legacy `nix-store --export` and `nix-store --import` commands. It should be avoided because it is not memory-efficient on import. It does not support signatures, so you have to use `--no-check-sigs` on import. + +* `2`: The latest version. Recommended. + +)"" diff --git a/src/nix/nario-import.md b/src/nix/nario-import.md new file mode 100644 index 00000000000..9cba60c6220 --- /dev/null +++ b/src/nix/nario-import.md @@ -0,0 +1,15 @@ +R""( + +# Examples + +* Import store paths from the file named `dump`: + + ```console + # nix nario import < dump.nario + ``` + +# Description + +This command reads from standard input a serialization of store paths produced by `nix nario export` and adds them to the Nix store. + +)"" diff --git a/src/nix/nario-list.md b/src/nix/nario-list.md new file mode 100644 index 00000000000..ea833586074 --- /dev/null +++ b/src/nix/nario-list.md @@ -0,0 +1,43 @@ +R""( + +# Examples + +* List the contents of a nario file: + + ```console + # nix nario list < dump.nario + /nix/store/4y1jj6cwvslmfh1bzkhbvhx77az6yf00-xgcc-14.2.1.20250322-libgcc: 201856 bytes + /nix/store/d8hnbm5hvbg2vza50garppb63y724i94-libunistring-1.3: 2070240 bytes + … + ``` + +* Use `--json` to get detailed information in JSON format: + + ```console + # nix nario list --json < dump.nario + { + "paths": { + "/nix/store/m1r53pnn…-hello-2.12.1": { + "ca": null, + "deriver": "/nix/store/qa8is0vm…-hello-2.12.1.drv", + "narHash": "sha256-KSCYs4J7tFa+oX7W5M4D7ZYNvrWtdcWTdTL5fQk+za8=", + "narSize": 234672, + "references": [ + "/nix/store/g8zyryr9…-glibc-2.40-66", + "/nix/store/m1r53pnn…-hello-2.12.1" + ], + "registrationTime": 1756900709, + "signatures": [ "cache.nixos.org-1:QbG7A…" ], + "ultimate": false + }, + … + }, + "version": 1 + } + ``` + +# Description + +This command lists the contents of a nario file read from standard input. + +)"" diff --git a/src/nix/nario.cc b/src/nix/nario.cc new file mode 100644 index 00000000000..c655711354c --- /dev/null +++ b/src/nix/nario.cc @@ -0,0 +1,344 @@ +#include "nix/cmd/command.hh" +#include "nix/main/shared.hh" +#include "nix/store/store-api.hh" +#include "nix/store/export-import.hh" +#include "nix/util/callback.hh" +#include "nix/util/fs-sink.hh" +#include "nix/util/archive.hh" + +#include "ls.hh" + +#include + +using namespace nix; + +struct CmdNario : NixMultiCommand +{ + CmdNario() + : NixMultiCommand("nario", RegisterCommand::getCommandsFor({"nario"})) + { + } + + std::string description() override + { + return "operations for manipulating nario files"; + } + + Category category() override + { + return catUtility; + } +}; + +static auto rCmdNario = registerCommand("nario"); + +struct CmdNarioExport : StorePathsCommand +{ + unsigned int version = 0; + + CmdNarioExport() + { + addFlag({ + .longName = "format", + .description = "Version of the nario format to use. Must be `1` or `2`.", + .labels = {"nario-format"}, + .handler = {&version}, + .required = true, + }); + } + + std::string description() override + { + return "serialize store paths to standard output in nario format"; + } + + std::string doc() override + { + return +#include "nario-export.md" + ; + } + + void run(ref store, StorePaths && storePaths) override + { + auto fd = getStandardOutput(); + if (isatty(fd)) + throw UsageError("refusing to write nario to a terminal"); + FdSink sink(std::move(fd)); + exportPaths(*store, StorePathSet(storePaths.begin(), storePaths.end()), sink, version); + } +}; + +static auto rCmdNarioExport = registerCommand2({"nario", "export"}); + +static FdSource getNarioSource() +{ + auto fd = getStandardInput(); + if (isatty(fd)) + throw UsageError("refusing to read nario from a terminal"); + return FdSource(std::move(fd)); +} + +struct CmdNarioImport : StoreCommand, MixNoCheckSigs +{ + std::string description() override + { + return "import store paths from a nario file on standard input"; + } + + std::string doc() override + { + return +#include "nario-import.md" + ; + } + + void run(ref store) override + { + auto source{getNarioSource()}; + importPaths(*store, source, checkSigs); + } +}; + +static auto rCmdNarioImport = registerCommand2({"nario", "import"}); + +nlohmann::json listNar(Source & source) +{ + struct : FileSystemObjectSink + { + nlohmann::json root = nlohmann::json::object(); + + nlohmann::json & makeObject(const CanonPath & path, std::string_view type) + { + auto * cur = &root; + for (auto & c : path) { + assert((*cur)["type"] == "directory"); + auto i = (*cur)["entries"].emplace(c, nlohmann::json::object()).first; + cur = &i.value(); + } + auto inserted = cur->emplace("type", type).second; + assert(inserted); + return *cur; + } + + void createDirectory(const CanonPath & path) override + { + auto & j = makeObject(path, "directory"); + j["entries"] = nlohmann::json::object(); + } + + void createRegularFile(const CanonPath & path, std::function func) override + { + struct : CreateRegularFileSink + { + bool executable = false; + std::optional size; + + void operator()(std::string_view data) override {} + + void preallocateContents(uint64_t s) override + { + size = s; + } + + void isExecutable() override + { + executable = true; + } + } crf; + + crf.skipContents = true; + + func(crf); + + auto & j = makeObject(path, "regular"); + j.emplace("size", crf.size.value()); + if (crf.executable) + j.emplace("executable", true); + } + + void createSymlink(const CanonPath & path, const std::string & target) override + { + auto & j = makeObject(path, "symlink"); + j.emplace("target", target); + } + + } parseSink; + + parseDump(parseSink, source); + + return parseSink.root; +} + +void renderNarListing(const CanonPath & prefix, const nlohmann::json & root, bool longListing) +{ + std::function recurse; + recurse = [&](const nlohmann::json & json, const CanonPath & path) { + auto type = json["type"]; + + if (longListing) { + auto tp = type == "regular" ? (json.find("executable") != json.end() ? "-r-xr-xr-x" : "-r--r--r--") + : type == "symlink" ? "lrwxrwxrwx" + : "dr-xr-xr-x"; + auto line = fmt("%s %9d %s", tp, type == "regular" ? (uint64_t) json["size"] : 0, prefix / path); + if (type == "symlink") + line += " -> " + (std::string) json["target"]; + logger->cout(line); + } else + logger->cout(fmt("%s", prefix / path)); + + if (type == "directory") { + for (auto & entry : json["entries"].items()) { + recurse(entry.value(), path / entry.key()); + } + } + }; + + recurse(root, CanonPath::root); +} + +struct CmdNarioList : Command, MixJSON, MixLongListing +{ + bool listContents = false; + + CmdNarioList() + { + addFlag({ + .longName = "recursive", + .shortName = 'R', + .description = "List the contents of NARs inside the nario.", + .handler = {&listContents, true}, + }); + } + + std::string description() override + { + return "list the contents of a nario file"; + } + + std::string doc() override + { + return +#include "nario-list.md" + ; + } + + void run() override + { + struct Config : StoreConfig + { + Config(const Params & params) + : StoreConfig(params) + { + } + + ref openStore() const override + { + abort(); + } + }; + + struct ListingStore : Store + { + std::optional json; + CmdNarioList & cmd; + + ListingStore(ref config, CmdNarioList & cmd) + : Store{*config} + , cmd(cmd) + { + } + + void queryPathInfoUncached( + const StorePath & path, Callback> callback) noexcept override + { + callback(nullptr); + } + + std::optional isTrustedClient() override + { + return Trusted; + } + + std::optional queryPathFromHashPart(const std::string & hashPart) override + { + return std::nullopt; + } + + void + addToStore(const ValidPathInfo & info, Source & source, RepairFlag repair, CheckSigsFlag checkSigs) override + { + std::optional contents; + if (cmd.listContents) + contents = listNar(source); + else + source.skip(info.narSize); + + if (json) { + auto obj = info.toJSON(*this, true, HashFormat::SRI); + if (contents) + obj.emplace("contents", *contents); + json->emplace(printStorePath(info.path), std::move(obj)); + } else { + if (contents) + renderNarListing(CanonPath(printStorePath(info.path)), *contents, cmd.longListing); + else + logger->cout(fmt("%s: %d bytes", printStorePath(info.path), info.narSize)); + } + } + + StorePath addToStoreFromDump( + Source & dump, + std::string_view name, + FileSerialisationMethod dumpMethod, + ContentAddressMethod hashMethod, + HashAlgorithm hashAlgo, + const StorePathSet & references, + RepairFlag repair) override + { + unsupported("addToStoreFromDump"); + } + + void narFromPath(const StorePath & path, Sink & sink) override + { + unsupported("narFromPath"); + } + + void queryRealisationUncached( + const DrvOutput &, Callback> callback) noexcept override + { + callback(nullptr); + } + + ref getFSAccessor(bool requireValidPath) override + { + return makeEmptySourceAccessor(); + } + + std::shared_ptr getFSAccessor(const StorePath & path, bool requireValidPath) override + { + unsupported("getFSAccessor"); + } + + void registerDrvOutput(const Realisation & output) override + { + unsupported("registerDrvOutput"); + } + }; + + auto source{getNarioSource()}; + auto config = make_ref(StoreConfig::Params()); + ListingStore lister(config, *this); + if (json) + lister.json = nlohmann::json::object(); + importPaths(lister, source, NoCheckSigs); + if (json) { + auto j = nlohmann::json::object(); + j["version"] = 1; + j["paths"] = std::move(*lister.json); + printJSON(j); + } + } +}; + +static auto rCmdNarioList = registerCommand2({"nario", "list"}); diff --git a/src/nix/nix-build/nix-build.cc b/src/nix/nix-build/nix-build.cc index d3902f2a6cd..8a39bf63239 100644 --- a/src/nix/nix-build/nix-build.cc +++ b/src/nix/nix-build/nix-build.cc @@ -285,10 +285,10 @@ static void main_nix_build(int argc, char ** argv) execArgs, interpreter, escapeShellArgAlways(script), - toView(joined)); + joined.view()); } else { envCommand = - fmt("exec %1% %2% %3% %4%", execArgs, interpreter, escapeShellArgAlways(script), toView(joined)); + fmt("exec %1% %2% %3% %4%", execArgs, interpreter, escapeShellArgAlways(script), joined.view()); } } @@ -451,7 +451,9 @@ static void main_nix_build(int argc, char ** argv) throw UsageError("nix-shell requires a single derivation"); auto & packageInfo = drvs.front(); - auto drv = evalStore->derivationFromPath(packageInfo.requireDrvPath()); + auto drvPath = packageInfo.requireDrvPath(); + state->waitForPath(drvPath); + auto drv = evalStore->derivationFromPath(drvPath); std::vector pathsToBuild; RealisedPath::Set pathsToCopy; @@ -475,6 +477,7 @@ static void main_nix_build(int argc, char ** argv) throw Error("the 'bashInteractive' attribute in did not evaluate to a derivation"); auto bashDrv = drv->requireDrvPath(); + state->waitForPath(bashDrv); pathsToBuild.push_back( DerivedPath::Built{ .drvPath = makeConstantStorePathRef(bashDrv), @@ -683,6 +686,7 @@ static void main_nix_build(int argc, char ** argv) for (auto & packageInfo : drvs) { auto drvPath = packageInfo.requireDrvPath(); + state->waitForPath(drvPath); auto outputName = packageInfo.queryOutputName(); if (outputName == "") diff --git a/src/nix/nix-channel/nix-channel.cc b/src/nix/nix-channel/nix-channel.cc index f047dce8f6d..354c44cbc01 100644 --- a/src/nix/nix-channel/nix-channel.cc +++ b/src/nix/nix-channel/nix-channel.cc @@ -183,6 +183,11 @@ static void update(const StringSet & channelNames) static int main_nix_channel(int argc, char ** argv) { + warn( + "nix-channel is deprecated in favor of flakes in Determinate Nix. \ +See https://zero-to-nix.com for a guide to Nix flakes. \ +For details and to offer feedback on the deprecation process, see: https://github.com/DeterminateSystems/nix-src/issues/34."); + { // Figure out the name of the `.nix-channels' file to use auto home = getHome(); diff --git a/src/nix/nix-env/nix-env.cc b/src/nix/nix-env/nix-env.cc index f165c069cd8..de437cc1c89 100644 --- a/src/nix/nix-env/nix-env.cc +++ b/src/nix/nix-env/nix-env.cc @@ -158,7 +158,7 @@ static void loadSourceExpr(EvalState & state, const SourcePath & path, Value & v directory). */ else if (st.type == SourceAccessor::tDirectory) { auto attrs = state.buildBindings(maxAttrs); - attrs.insert(state.symbols.create("_combineChannels"), &state.vEmptyList); + attrs.insert(state.symbols.create("_combineChannels"), &Value::vEmptyList); StringSet seen; getAllExprs(state, path, seen, attrs); v.mkAttrs(attrs); @@ -746,6 +746,8 @@ static void opSet(Globals & globals, Strings opFlags, Strings opArgs) drv.setName(globals.forceName); auto drvPath = drv.queryDrvPath(); + if (drvPath) + globals.state->waitForPath(*drvPath); std::vector paths{ drvPath ? (DerivedPath) (DerivedPath::Built{ .drvPath = makeConstantStorePathRef(*drvPath), diff --git a/src/nix/nix-env/user-env.cc b/src/nix/nix-env/user-env.cc index 1b6e552f724..43b386d37d5 100644 --- a/src/nix/nix-env/user-env.cc +++ b/src/nix/nix-env/user-env.cc @@ -24,7 +24,7 @@ PackageInfos queryInstalled(EvalState & state, const Path & userEnv) if (pathExists(manifestFile)) { Value v; state.evalFile(state.rootPath(CanonPath(manifestFile)).resolveSymlinks(), v); - Bindings & bindings(*state.allocBindings(0)); + Bindings & bindings = Bindings::emptyBindings; getDerivations(state, v, "", bindings, elems, false); } return elems; @@ -37,8 +37,10 @@ bool createUserEnv( exist already. */ std::vector drvsToBuild; for (auto & i : elems) - if (auto drvPath = i.queryDrvPath()) + if (auto drvPath = i.queryDrvPath()) { + state.waitForPath(*drvPath); drvsToBuild.push_back({*drvPath}); + } debug("building user environment dependencies"); state.store->buildPaths(toDerivedPaths(drvsToBuild), state.repair ? bmRepair : bmNormal); @@ -56,21 +58,21 @@ bool createUserEnv( auto attrs = state.buildBindings(7 + outputs.size()); - attrs.alloc(state.sType).mkString("derivation"); - attrs.alloc(state.sName).mkString(i.queryName()); + attrs.alloc(state.s.type).mkStringNoCopy("derivation"); + attrs.alloc(state.s.name).mkString(i.queryName()); auto system = i.querySystem(); if (!system.empty()) - attrs.alloc(state.sSystem).mkString(system); - attrs.alloc(state.sOutPath).mkString(state.store->printStorePath(i.queryOutPath())); + attrs.alloc(state.s.system).mkString(system); + attrs.alloc(state.s.outPath).mkString(state.store->printStorePath(i.queryOutPath())); if (drvPath) - attrs.alloc(state.sDrvPath).mkString(state.store->printStorePath(*drvPath)); + attrs.alloc(state.s.drvPath).mkString(state.store->printStorePath(*drvPath)); // Copy each output meant for installation. auto outputsList = state.buildList(outputs.size()); for (const auto & [m, j] : enumerate(outputs)) { (outputsList[m] = state.allocValue())->mkString(j.first); auto outputAttrs = state.buildBindings(2); - outputAttrs.alloc(state.sOutPath).mkString(state.store->printStorePath(*j.second)); + outputAttrs.alloc(state.s.outPath).mkString(state.store->printStorePath(*j.second)); attrs.alloc(j.first).mkAttrs(outputAttrs); /* This is only necessary when installing store paths, e.g., @@ -80,7 +82,7 @@ bool createUserEnv( references.insert(*j.second); } - attrs.alloc(state.sOutputs).mkList(outputsList); + attrs.alloc(state.s.outputs).mkList(outputsList); // Copy the meta attributes. auto meta = state.buildBindings(metaNames.size()); @@ -91,7 +93,7 @@ bool createUserEnv( meta.insert(state.symbols.create(j), v); } - attrs.alloc(state.sMeta).mkAttrs(meta); + attrs.alloc(state.s.meta).mkAttrs(meta); (list[n] = state.allocValue())->mkAttrs(attrs); @@ -107,8 +109,8 @@ bool createUserEnv( environment. */ auto manifestFile = ({ std::ostringstream str; - printAmbiguous(manifest, state.symbols, str, nullptr, std::numeric_limits::max()); - StringSource source{toView(str)}; + printAmbiguous(state, manifest, str, nullptr, std::numeric_limits::max()); + StringSource source{str.view()}; state.store->addToStoreFromDump( source, "env-manifest.nix", @@ -141,16 +143,17 @@ bool createUserEnv( debug("evaluating user environment builder"); state.forceValue(topLevel, topLevel.determinePos(noPos)); NixStringContext context; - auto & aDrvPath(*topLevel.attrs()->find(state.sDrvPath)); + auto & aDrvPath(*topLevel.attrs()->get(state.s.drvPath)); auto topLevelDrv = state.coerceToStorePath(aDrvPath.pos, *aDrvPath.value, context, ""); topLevelDrv.requireDerivation(); - auto & aOutPath(*topLevel.attrs()->find(state.sOutPath)); + auto & aOutPath(*topLevel.attrs()->get(state.s.outPath)); auto topLevelOut = state.coerceToStorePath(aOutPath.pos, *aOutPath.value, context, ""); /* Realise the resulting store expression. */ debug("building user environment"); std::vector topLevelDrvs; topLevelDrvs.push_back({topLevelDrv}); + state.waitForPath(topLevelDrv); state.store->buildPaths(toDerivedPaths(topLevelDrvs), state.repair ? bmRepair : bmNormal); /* Switch the current user environment to the output path. */ diff --git a/src/nix/nix-instantiate/nix-instantiate.cc b/src/nix/nix-instantiate/nix-instantiate.cc index 3d5c3e26a46..f09b4078a24 100644 --- a/src/nix/nix-instantiate/nix-instantiate.cc +++ b/src/nix/nix-instantiate/nix-instantiate.cc @@ -17,6 +17,8 @@ #include #include +#include + using namespace nix; static Path gcRoot; @@ -56,19 +58,23 @@ void processExpr( else state.autoCallFunction(autoArgs, v, vRes); if (output == okRaw) - std::cout << *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output"); + std::cout << state.devirtualize( + *state.coerceToString(noPos, vRes, context, "while generating the nix-instantiate output"), + context); // We intentionally don't output a newline here. The default PS1 for Bash in NixOS starts with a newline // and other interactive shells like Zsh are smart enough to print a missing newline before the prompt. - else if (output == okXML) - printValueAsXML(state, strict, location, vRes, std::cout, context, noPos); - else if (output == okJSON) { - printValueAsJSON(state, strict, vRes, v.determinePos(noPos), std::cout, context); - std::cout << std::endl; + else if (output == okXML) { + std::ostringstream s; + printValueAsXML(state, strict, location, vRes, s, context, noPos); + std::cout << state.devirtualize(s.str(), context); + } else if (output == okJSON) { + auto j = printValueAsJSON(state, strict, vRes, v.determinePos(noPos), context); + std::cout << state.devirtualize(j.dump(), context) << std::endl; } else { if (strict) state.forceValueDeep(vRes); std::set seen; - printAmbiguous(vRes, state.symbols, std::cout, &seen, std::numeric_limits::max()); + printAmbiguous(state, vRes, std::cout, &seen, std::numeric_limits::max()); std::cout << std::endl; } } else { diff --git a/src/nix/nix-store/nix-store.cc b/src/nix/nix-store/nix-store.cc index 4191ea0d6fc..35563404c16 100644 --- a/src/nix/nix-store/nix-store.cc +++ b/src/nix/nix-store/nix-store.cc @@ -14,6 +14,7 @@ #include "nix/util/posix-source-accessor.hh" #include "nix/store/globals.hh" #include "nix/store/path-with-outputs.hh" +#include "nix/store/export-import.hh" #include "man-pages.hh" @@ -602,7 +603,7 @@ static void registerValidity(bool reregister, bool hashGiven, bool canonicalise) #endif if (!hashGiven) { HashResult hash = hashPath( - {store->getFSAccessor(false), CanonPath{info->path.to_string()}}, + {ref{store->getFSAccessor(info->path, false)}}, FileSerialisationMethod::NixArchive, HashAlgorithm::SHA256); info->narHash = hash.hash; @@ -774,7 +775,7 @@ static void opExport(Strings opFlags, Strings opArgs) paths.insert(store->followLinksToStorePath(i)); FdSink sink(getStandardOutput()); - store->exportPaths(paths, sink); + exportPaths(*store, paths, sink, 1); sink.flush(); } @@ -787,7 +788,7 @@ static void opImport(Strings opFlags, Strings opArgs) throw UsageError("no arguments expected"); FdSource source(STDIN_FILENO); - auto paths = store->importPaths(source, NoCheckSigs); + auto paths = importPaths(*store, source, NoCheckSigs); for (auto & i : paths) cout << fmt("%s\n", store->printStorePath(i)) << std::flush; @@ -985,20 +986,6 @@ static void opServe(Strings opFlags, Strings opArgs) store->narFromPath(store->parseStorePath(readString(in)), out); break; - case ServeProto::Command::ImportPaths: { - if (!writeAllowed) - throw Error("importing paths is not allowed"); - store->importPaths(in, NoCheckSigs); // FIXME: should we skip sig checking? - out << 1; // indicate success - break; - } - - case ServeProto::Command::ExportPaths: { - readInt(in); // obsolete - store->exportPaths(ServeProto::Serialise::read(*store, rconn), out); - break; - } - case ServeProto::Command::BuildPaths: { if (!writeAllowed) diff --git a/src/nix/nix.md b/src/nix/nix.md index 10a2aaee88c..83a1fd0776f 100644 --- a/src/nix/nix.md +++ b/src/nix/nix.md @@ -48,11 +48,6 @@ manual](https://nix.dev/manual/nix/stable/). # Installables -> **Warning** \ -> Installables are part of the unstable -> [`nix-command` experimental feature](@docroot@/development/experimental-features.md#xp-feature-nix-command), -> and subject to change without notice. - Many `nix` subcommands operate on one or more *installables*. These are command line arguments that represent something that can be realised in the Nix store. @@ -72,13 +67,6 @@ That is, Nix will operate on the default flake output attribute of the flake in ### Flake output attribute -> **Warning** \ -> Flake output attribute installables depend on both the -> [`flakes`](@docroot@/development/experimental-features.md#xp-feature-flakes) -> and -> [`nix-command`](@docroot@/development/experimental-features.md#xp-feature-nix-command) -> experimental features, and subject to change without notice. - Example: `nixpkgs#hello` These have the form *flakeref*[`#`*attrpath*], where *flakeref* is a diff --git a/src/nix/prefetch.cc b/src/nix/prefetch.cc index b23b11d0220..18abfa0aae2 100644 --- a/src/nix/prefetch.cc +++ b/src/nix/prefetch.cc @@ -105,7 +105,7 @@ std::tuple prefetchFile( FdSink sink(fd.get()); - FileTransferRequest req(url); + FileTransferRequest req(VerbatimURL{url}); req.decompress = false; getFileTransfer()->download(std::move(req), sink); } diff --git a/src/nix/profile-history.md b/src/nix/profile-history.md index f0bfe503791..0c9a340ddf0 100644 --- a/src/nix/profile-history.md +++ b/src/nix/profile-history.md @@ -7,7 +7,7 @@ R""( ```console # nix profile history Version 508 (2020-04-10): - flake:nixpkgs#legacyPackages.x86_64-linux.awscli: ∅ -> 1.17.13 + flake:nixpkgs#legacyPackages.x86_64-linux.awscli: 1.17.13 added Version 509 (2020-05-16) <- 508: flake:nixpkgs#legacyPackages.x86_64-linux.awscli: 1.17.13 -> 1.18.211 @@ -20,7 +20,7 @@ between subsequent versions of a profile. It only shows top-level packages, not dependencies; for that, use [`nix profile diff-closures`](./nix3-profile-diff-closures.md). -The addition of a package to a profile is denoted by the string `∅ ->` -*version*, whereas the removal is denoted by *version* `-> ∅`. +The addition of a package to a profile is denoted by the string +*version* `added`, whereas the removal is denoted by *version* ` removed`. )"" diff --git a/src/nix/profile.cc b/src/nix/profile.cc index 0ed1face509..3509ee11244 100644 --- a/src/nix/profile.cc +++ b/src/nix/profile.cc @@ -177,8 +177,8 @@ struct ProfileManifest else if (std::filesystem::exists(profile / "manifest.nix")) { // FIXME: needed because of pure mode; ugly. - state.allowPath(state.store->followLinksToStore(profile.string())); - state.allowPath(state.store->followLinksToStore((profile / "manifest.nix").string())); + state.allowPath(state.store->followLinksToStorePath(profile.string())); + state.allowPath(state.store->followLinksToStorePath((profile / "manifest.nix").string())); auto packageInfos = queryInstalled(state, state.store->followLinksToStore(profile.string())); @@ -257,7 +257,7 @@ struct ProfileManifest auto narHash = hashString(HashAlgorithm::SHA256, sink.s); - ValidPathInfo info{ + auto info = ValidPathInfo::makeFromCA( *store, "profile", FixedOutputInfo{ @@ -270,8 +270,7 @@ struct ProfileManifest .self = false, }, }, - narHash, - }; + narHash); info.narSize = sink.s.size(); StringSource source(sink.s); @@ -289,11 +288,11 @@ struct ProfileManifest while (i != prev.elements.end() || j != cur.elements.end()) { if (j != cur.elements.end() && (i == prev.elements.end() || i->first > j->first)) { - logger->cout("%s%s: ∅ -> %s", indent, j->second.identifier(), j->second.versions()); + logger->cout("%s%s: %s added", indent, j->second.identifier(), j->second.versions()); changes = true; ++j; } else if (i != prev.elements.end() && (j == cur.elements.end() || i->first < j->first)) { - logger->cout("%s%s: %s -> ∅", indent, i->second.identifier(), i->second.versions()); + logger->cout("%s%s: %s removed", indent, i->second.identifier(), i->second.versions()); changes = true; ++i; } else { diff --git a/src/nix/realisation.cc b/src/nix/realisation.cc index a0e400f54de..8dd608d23b0 100644 --- a/src/nix/realisation.cc +++ b/src/nix/realisation.cc @@ -59,7 +59,7 @@ struct CmdRealisationInfo : BuiltPathsCommand, MixJSON for (auto & path : realisations) { nlohmann::json currentPath; if (auto realisation = std::get_if(&path.raw)) - currentPath = realisation->toJSON(); + currentPath = *realisation; else currentPath["opaquePath"] = store->printStorePath(path.path()); diff --git a/src/nix/repl.md b/src/nix/repl.md index 32c08e24b24..e608dabf6f9 100644 --- a/src/nix/repl.md +++ b/src/nix/repl.md @@ -36,7 +36,7 @@ R""( Loading Installable ''... Added 1 variables. - # nix repl --extra-experimental-features 'flakes' nixpkgs + # nix repl nixpkgs Loading Installable 'flake:nixpkgs#'... Added 5 variables. diff --git a/src/nix/search.cc b/src/nix/search.cc index 562af31518e..3323db01057 100644 --- a/src/nix/search.cc +++ b/src/nix/search.cc @@ -11,6 +11,7 @@ #include "nix/expr/attr-path.hh" #include "nix/util/hilite.hh" #include "nix/util/strings-inline.hh" +#include "nix/expr/parallel-eval.hh" #include #include @@ -84,11 +85,13 @@ struct CmdSearch : InstallableValueCommand, MixJSON auto state = getEvalState(); - std::optional jsonOut; + std::optional> jsonOut; if (json) - jsonOut = json::object(); + jsonOut.emplace(json::object()); - uint64_t results = 0; + std::atomic results = 0; + + FutureVector futures(*state->executor); std::function & attrPath, bool initialRecurse)> visit; @@ -96,22 +99,29 @@ struct CmdSearch : InstallableValueCommand, MixJSON visit = [&](eval_cache::AttrCursor & cursor, const std::vector & attrPath, bool initialRecurse) { auto attrPathS = state->symbols.resolve(attrPath); - Activity act(*logger, lvlInfo, actUnknown, fmt("evaluating '%s'", concatStringsSep(".", attrPathS))); + /* + Activity act(*logger, lvlInfo, actUnknown, + fmt("evaluating '%s'", concatStringsSep(".", attrPathS))); + */ try { auto recurse = [&]() { + std::vector> work; for (const auto & attr : cursor.getAttrs()) { auto cursor2 = cursor.getAttr(state->symbols[attr]); auto attrPath2(attrPath); attrPath2.push_back(attr); - visit(*cursor2, attrPath2, false); + work.emplace_back( + [cursor2, attrPath2, visit]() { visit(*cursor2, attrPath2, false); }, + std::string_view(state->symbols[attr]).find("Packages") != std::string_view::npos ? 0 : 2); } + futures.spawn(std::move(work)); }; if (cursor.isDerivation()) { - DrvName name(cursor.getAttr(state->sName)->getString()); + DrvName name(cursor.getAttr(state->s.name)->getString()); - auto aMeta = cursor.maybeGetAttr(state->sMeta); - auto aDescription = aMeta ? aMeta->maybeGetAttr(state->sDescription) : nullptr; + auto aMeta = cursor.maybeGetAttr(state->s.meta); + auto aDescription = aMeta ? aMeta->maybeGetAttr(state->s.description) : nullptr; auto description = aDescription ? aDescription->getString() : ""; std::replace(description.begin(), description.end(), '\n', ' '); auto attrPath2 = concatStringsSep(".", attrPathS); @@ -148,21 +158,21 @@ struct CmdSearch : InstallableValueCommand, MixJSON if (found) { results++; if (json) { - (*jsonOut)[attrPath2] = { + (*jsonOut->lock())[attrPath2] = { {"pname", name.name}, {"version", name.version}, {"description", description}, }; } else { - if (results > 1) - logger->cout(""); - logger->cout( - "* %s%s", - wrap("\e[0;1m", hiliteMatches(attrPath2, attrPathMatches, ANSI_GREEN, "\e[0;1m")), - name.version != "" ? " (" + name.version + ")" : ""); + auto out = + fmt("%s* %s%s", + results > 1 ? "\n" : "", + wrap("\e[0;1m", hiliteMatches(attrPath2, attrPathMatches, ANSI_GREEN, "\e[0;1m")), + name.version != "" ? " (" + name.version + ")" : ""); if (description != "") - logger->cout( - " %s", hiliteMatches(description, descriptionMatches, ANSI_GREEN, ANSI_NORMAL)); + out += fmt( + "\n %s", hiliteMatches(description, descriptionMatches, ANSI_GREEN, ANSI_NORMAL)); + logger->cout(out); } } } @@ -176,7 +186,7 @@ struct CmdSearch : InstallableValueCommand, MixJSON recurse(); else if (attrPathS[0] == "legacyPackages" && attrPath.size() > 2) { - auto attr = cursor.maybeGetAttr(state->sRecurseForDerivations); + auto attr = cursor.maybeGetAttr(state->s.recurseForDerivations); if (attr && attr->getBool()) recurse(); } @@ -187,14 +197,21 @@ struct CmdSearch : InstallableValueCommand, MixJSON } }; - for (auto & cursor : installable->getCursors(*state)) - visit(*cursor, cursor->getAttrPath(), true); + std::vector> work; + for (auto & cursor : installable->getCursors(*state)) { + work.emplace_back([cursor, visit]() { visit(*cursor, cursor->getAttrPath(), true); }, 1); + } + + futures.spawn(std::move(work)); + futures.finishAll(); if (json) - printJSON(*jsonOut); + printJSON(*(jsonOut->lock())); if (!json && !results) throw Error("no results for the given search term(s)!"); + + notice("Found %d matching packages.", results); } }; diff --git a/src/nix/sigs.cc b/src/nix/sigs.cc index 92bb0050058..e82f0d284b9 100644 --- a/src/nix/sigs.cc +++ b/src/nix/sigs.cc @@ -3,6 +3,7 @@ #include "nix/main/shared.hh" #include "nix/store/store-open.hh" #include "nix/util/thread-pool.hh" +#include "nix/store/filetransfer.hh" #include @@ -28,6 +29,13 @@ struct CmdCopySigs : StorePathsCommand return "copy store path signatures from substituters"; } + std::string doc() override + { + return +#include "store-copy-sigs.md" + ; + } + void run(ref store, StorePaths && storePaths) override { if (substituterUris.empty()) @@ -38,7 +46,7 @@ struct CmdCopySigs : StorePathsCommand for (auto & s : substituterUris) substituters.push_back(openStore(s)); - ThreadPool pool; + ThreadPool pool{fileTransferSettings.httpConnections}; std::atomic added{0}; @@ -104,6 +112,7 @@ struct CmdSign : StorePathsCommand .labels = {"file"}, .handler = {&secretKeyFile}, .completer = completePath, + .required = true, }); } @@ -114,9 +123,6 @@ struct CmdSign : StorePathsCommand void run(ref store, StorePaths && storePaths) override { - if (secretKeyFile.empty()) - throw UsageError("you must specify a secret key file using '-k'"); - SecretKey secretKey(readFile(secretKeyFile)); LocalSigner signer(std::move(secretKey)); @@ -144,7 +150,7 @@ static auto rCmdSign = registerCommand2({"store", "sign"}); struct CmdKeyGenerateSecret : Command { - std::optional keyName; + std::string keyName; CmdKeyGenerateSecret() { @@ -153,6 +159,7 @@ struct CmdKeyGenerateSecret : Command .description = "Identifier of the key (e.g. `cache.example.org-1`).", .labels = {"name"}, .handler = {&keyName}, + .required = true, }); } @@ -170,11 +177,8 @@ struct CmdKeyGenerateSecret : Command void run() override { - if (!keyName) - throw UsageError("required argument '--key-name' is missing"); - logger->stop(); - writeFull(getStandardOutput(), SecretKey::generate(*keyName).to_string()); + writeFull(getStandardOutput(), SecretKey::generate(keyName).to_string()); } }; diff --git a/src/nix/store-copy-sigs.md b/src/nix/store-copy-sigs.md new file mode 100644 index 00000000000..67875622156 --- /dev/null +++ b/src/nix/store-copy-sigs.md @@ -0,0 +1,30 @@ +R""( + +# Examples + +* To copy signatures from a binary cache to the local store: + + ```console + # nix store copy-sigs --substituter https://cache.nixos.org \ + --recursive /nix/store/y1x7ng5bmc9s8lqrf98brcpk1a7lbcl5-hello-2.12.1 + ``` + +* To copy signatures from one binary cache to another: + + ```console + # nix store copy-sigs --substituter https://cache.nixos.org \ + --store file:///tmp/binary-cache \ + --recursive -v \ + /nix/store/y1x7ng5bmc9s8lqrf98brcpk1a7lbcl5-hello-2.12.1 + imported 2 signatures + ``` + +# Description + +`nix store copy-sigs` copies store path signatures from one store to another. + +It is not advised to copy signatures to binary cache stores. Binary cache signatures are stored in `.narinfo` files. Since these are cached aggressively, clients may not see the new signatures quickly. It is therefore better to set any required signatures when the paths are first uploaded to the binary cache. + +Store paths are processed in parallel. The amount of parallelism is controlled by the [`http-connections`](@docroot@/command-ref/conf-file.md#conf-http-connections) settings. + +)"" diff --git a/src/nix/upgrade-nix.cc b/src/nix/upgrade-nix.cc index f6668f6dc44..f5ca094c6af 100644 --- a/src/nix/upgrade-nix.cc +++ b/src/nix/upgrade-nix.cc @@ -15,26 +15,6 @@ using namespace nix; struct CmdUpgradeNix : MixDryRun, StoreCommand { - std::filesystem::path profileDir; - - CmdUpgradeNix() - { - addFlag({ - .longName = "profile", - .shortName = 'p', - .description = "The path to the Nix profile to upgrade.", - .labels = {"profile-dir"}, - .handler = {&profileDir}, - }); - - addFlag({ - .longName = "nix-store-paths-url", - .description = "The URL of the file that contains the store paths of the latest Nix release.", - .labels = {"url"}, - .handler = {&(std::string &) settings.upgradeNixStorePathUrl}, - }); - } - /** * This command is stable before the others */ @@ -45,7 +25,7 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand std::string description() override { - return "upgrade Nix to the latest stable version"; + return "deprecated in favor of determinate-nixd upgrade"; } std::string doc() override @@ -62,111 +42,9 @@ struct CmdUpgradeNix : MixDryRun, StoreCommand void run(ref store) override { - evalSettings.pureEval = true; - - if (profileDir == "") - profileDir = getProfileDir(store); - - printInfo("upgrading Nix in profile %s", profileDir); - - auto storePath = getLatestNix(store); - - auto version = DrvName(storePath.name()).version; - - if (dryRun) { - logger->stop(); - warn("would upgrade to version %s", version); - return; - } - - { - Activity act(*logger, lvlInfo, actUnknown, fmt("downloading '%s'...", store->printStorePath(storePath))); - store->ensurePath(storePath); - } - - { - Activity act( - *logger, lvlInfo, actUnknown, fmt("verifying that '%s' works...", store->printStorePath(storePath))); - auto program = store->printStorePath(storePath) + "/bin/nix-env"; - auto s = runProgram(program, false, {"--version"}); - if (s.find("Nix") == std::string::npos) - throw Error("could not verify that '%s' works", program); - } - - logger->stop(); - - { - Activity act( - *logger, - lvlInfo, - actUnknown, - fmt("installing '%s' into profile %s...", store->printStorePath(storePath), profileDir)); - - // FIXME: don't call an external process. - runProgram( - getNixBin("nix-env").string(), - false, - {"--profile", profileDir.string(), "-i", store->printStorePath(storePath), "--no-sandbox"}); - } - - printInfo(ANSI_GREEN "upgrade to version %s done" ANSI_NORMAL, version); - } - - /* Return the profile in which Nix is installed. */ - std::filesystem::path getProfileDir(ref store) - { - auto whereOpt = ExecutablePath::load().findName(OS_STR("nix-env")); - if (!whereOpt) - throw Error("couldn't figure out how Nix is installed, so I can't upgrade it"); - const auto & where = whereOpt->parent_path(); - - printInfo("found Nix in %s", where); - - if (hasPrefix(where.string(), "/run/current-system")) - throw Error("Nix on NixOS must be upgraded via 'nixos-rebuild'"); - - auto profileDir = where.parent_path(); - - // Resolve profile to /nix/var/nix/profiles/ link. - while (canonPath(profileDir.string()).find("/profiles/") == std::string::npos - && std::filesystem::is_symlink(profileDir)) - profileDir = readLink(profileDir.string()); - - printInfo("found profile %s", profileDir); - - Path userEnv = canonPath(profileDir.string(), true); - - if (std::filesystem::exists(profileDir / "manifest.json")) - throw Error( - "directory %s is managed by 'nix profile' and currently cannot be upgraded by 'nix upgrade-nix'", - profileDir); - - if (!std::filesystem::exists(profileDir / "manifest.nix")) - throw Error("directory %s does not appear to be part of a Nix profile", profileDir); - - if (!store->isValidPath(store->parseStorePath(userEnv))) - throw Error("directory '%s' is not in the Nix store", userEnv); - - return profileDir; - } - - /* Return the store path of the latest stable Nix. */ - StorePath getLatestNix(ref store) - { - Activity act(*logger, lvlInfo, actUnknown, "querying latest Nix version"); - - // FIXME: use nixos.org? - auto req = FileTransferRequest((std::string &) settings.upgradeNixStorePathUrl); - auto res = getFileTransfer()->download(req); - - auto state = std::make_unique(LookupPath{}, store, fetchSettings, evalSettings); - auto v = state->allocValue(); - state->eval(state->parseExprFromString(res.data, state->rootPath(CanonPath("/no-such-path"))), *v); - Bindings & bindings(*state->allocBindings(0)); - auto v2 = findAlongAttrPath(*state, settings.thisSystem, bindings, *v).first; - - return store->parseStorePath( - state->forceString(*v2, noPos, "while evaluating the path tho latest nix version")); + throw Error( + "The upgrade-nix command isn't available in Determinate Nix; use %s instead", + "sudo determinate-nixd upgrade"); } }; diff --git a/src/nix/upgrade-nix.md b/src/nix/upgrade-nix.md index 3a3bf61b9b0..bb515717582 100644 --- a/src/nix/upgrade-nix.md +++ b/src/nix/upgrade-nix.md @@ -1,33 +1,11 @@ R""( -# Examples - -* Upgrade Nix to the stable version declared in Nixpkgs: - - ```console - # nix upgrade-nix - ``` - -* Upgrade Nix in a specific profile: - - ```console - # nix upgrade-nix --profile ~alice/.local/state/nix/profiles/profile - ``` - # Description -This command upgrades Nix to the stable version. - -By default, the latest stable version is defined by Nixpkgs, in -[nix-fallback-paths.nix](https://github.com/NixOS/nixpkgs/raw/master/nixos/modules/installer/tools/nix-fallback-paths.nix) -and updated manually. It may not always be the latest tagged release. - -By default, it locates the directory containing the `nix` binary in the `$PATH` -environment variable. If that directory is a Nix profile, it will -upgrade the `nix` package in that profile to the latest stable binary -release. +This command isn't available in Determinate Nix but is present in order to guide +users to the new upgrade path. -You cannot use this command to upgrade Nix in the system profile of a -NixOS system (that is, if `nix` is found in `/run/current-system`). +Use `sudo determinate-nixd upgrade` to upgrade Determinate Nix on systems that manage it imperatively. +In practice, this is any system that isn't running NixOS. )"" diff --git a/src/nix/why-depends.cc b/src/nix/why-depends.cc index 7869e33a7be..473827a9344 100644 --- a/src/nix/why-depends.cc +++ b/src/nix/why-depends.cc @@ -108,8 +108,6 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions auto dependencyPath = *optDependencyPath; auto dependencyPathHash = dependencyPath.hashPart(); - auto accessor = store->getFSAccessor(); - auto const inf = std::numeric_limits::max(); struct Node @@ -172,8 +170,6 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions {}; printNode = [&](Node & node, const std::string & firstPad, const std::string & tailPad) { - CanonPath pathS(node.path.to_string()); - assert(node.dist != inf); if (precise) { logger->cout( @@ -181,7 +177,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions firstPad, node.visited ? "\e[38;5;244m" : "", firstPad != "" ? "→ " : "", - pathS.abs()); + store->printStorePath(node.path)); } if (node.path == dependencyPath && !all && packagePath != dependencyPath) @@ -211,13 +207,13 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions contain the reference. */ std::map hits; - std::function visitPath; + auto accessor = store->getFSAccessor(node.path); - visitPath = [&](const CanonPath & p) { + auto visitPath = [&](this auto && recur, const CanonPath & p) -> void { auto st = accessor->maybeLstat(p); assert(st); - auto p2 = p == pathS ? "/" : p.abs().substr(pathS.abs().size() + 1); + auto p2 = p.isRoot() ? p.abs() : p.rel(); auto getColour = [&](const std::string & hash) { return hash == dependencyPathHash ? ANSI_GREEN : ANSI_BLUE; @@ -226,7 +222,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions if (st->type == SourceAccessor::Type::tDirectory) { auto names = accessor->readDirectory(p); for (auto & [name, type] : names) - visitPath(p / name); + recur(p / name); } else if (st->type == SourceAccessor::Type::tRegular) { @@ -264,7 +260,7 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions // FIXME: should use scanForReferences(). if (precise) - visitPath(pathS); + visitPath(CanonPath::root); for (auto & ref : refs) { std::string hash(ref.second->path.hashPart()); @@ -280,13 +276,12 @@ struct CmdWhyDepends : SourceExprCommand, MixOperateOnOptions } if (!precise) { - auto pathS = store->printStorePath(ref.second->path); logger->cout( "%s%s%s%s" ANSI_NORMAL, firstPad, ref.second->visited ? "\e[38;5;244m" : "", last ? treeLast : treeConn, - pathS); + store->printStorePath(ref.second->path)); node.visited = true; } diff --git a/src/perl/lib/Nix/Store.xs b/src/perl/lib/Nix/Store.xs index edcb6d72a5e..6de26f0d284 100644 --- a/src/perl/lib/Nix/Store.xs +++ b/src/perl/lib/Nix/Store.xs @@ -11,6 +11,7 @@ #include "nix/store/globals.hh" #include "nix/store/store-open.hh" #include "nix/util/posix-source-accessor.hh" +#include "nix/store/export-import.hh" #include #include @@ -167,7 +168,7 @@ StoreWrapper::queryRawRealisation(char * outputId) try { auto realisation = THIS->store->queryRealisation(DrvOutput::parse(outputId)); if (realisation) - XPUSHs(sv_2mortal(newSVpv(realisation->toJSON().dump().c_str(), 0))); + XPUSHs(sv_2mortal(newSVpv(static_cast(*realisation).dump().c_str(), 0))); else XPUSHs(sv_2mortal(newSVpv("", 0))); } catch (Error & e) { @@ -233,7 +234,7 @@ StoreWrapper::exportPaths(int fd, ...) StorePathSet paths; for (int n = 2; n < items; ++n) paths.insert(THIS->store->parseStorePath(SvPV_nolen(ST(n)))); FdSink sink(fd); - THIS->store->exportPaths(paths, sink); + exportPaths(*THIS->store, paths, sink, 1); } catch (Error & e) { croak("%s", e.what()); } @@ -244,7 +245,7 @@ StoreWrapper::importPaths(int fd, int dontCheckSigs) PPCODE: try { FdSource source(fd); - THIS->store->importPaths(source, dontCheckSigs ? NoCheckSigs : CheckSigs); + importPaths(*THIS->store, source, dontCheckSigs ? NoCheckSigs : CheckSigs); } catch (Error & e) { croak("%s", e.what()); } diff --git a/src/perl/package.nix b/src/perl/package.nix index 10d84de771a..424e38d3070 100644 --- a/src/perl/package.nix +++ b/src/perl/package.nix @@ -18,7 +18,7 @@ in perl.pkgs.toPerlModule ( mkMesonDerivation (finalAttrs: { - pname = "nix-perl"; + pname = "determinate-nix-perl"; inherit version; workDir = ./.; diff --git a/tests/functional/build-delete.sh b/tests/functional/build-delete.sh index 18841509d50..66b14fd1438 100755 --- a/tests/functional/build-delete.sh +++ b/tests/functional/build-delete.sh @@ -43,6 +43,10 @@ issue_6572_dependent_outputs() { nix-store --delete "$p" # Clean up for next test # Make sure that 'nix build' tracks input-outputs correctly when a single output is already present. + if [[ -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + # Resolved derivations interferre with the deletion + nix-store --delete "${NIX_STORE_DIR}"/*.drv + fi nix-store --delete "$(jq -r <"$TEST_ROOT"/a.json .[0].outputs.second)" p=$(nix build -f multiple-outputs.nix use-a --no-link --print-out-paths) cmp "$p" < $out/file + ''; + }; + + # Same output, different drv + a-prime = mkDerivation { + name = "issue-13247-a"; + builder = builtins.toFile "builder.sh" '' + echo 'will make the same stuff as `a`, but different drv hash' + + mkdir $out + test -z $all + echo "output" > $out/file + ''; + }; + + # Multiple outputs in a derivation that depends on other derivations + f = + dep: + mkDerivation { + name = "use-a-more-outputs"; + outputs = [ + "first" + "second" + ]; + inherit dep; + builder = builtins.toFile "builder.sh" '' + ln -s $dep/file $first + ln -s $first $second + ''; + }; + + use-a-more-outputs = f a; + + use-a-prime-more-outputs = f a-prime; + +} diff --git a/tests/functional/ca/issue-13247.sh b/tests/functional/ca/issue-13247.sh new file mode 100755 index 00000000000..686d90cede6 --- /dev/null +++ b/tests/functional/ca/issue-13247.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# https://github.com/NixOS/nix/issues/13247 + +export NIX_TESTS_CA_BY_DEFAULT=1 + +source common.sh + +clearStoreIfPossible + +set -x + +# Build derivation (both outputs) +nix build -f issue-13247.nix --json a a-prime use-a-more-outputs --no-link > "$TEST_ROOT"/a.json + +cache="file://$TEST_ROOT/cache" + +# Copy all outputs and realisations to cache +declare -a drvs +for d in "$NIX_STORE_DIR"/*-issue-13247-a.drv "$NIX_STORE_DIR"/*-use-a-more-outputs.drv; do + drvs+=("$d" "$d"^*) +done +nix copy --to "$cache" "${drvs[@]}" + +function delete () { + # Delete local copy + # shellcheck disable=SC2046 + nix-store --delete \ + $(jq -r <"$TEST_ROOT"/a.json '.[] | .drvPath, .outputs.[]') \ + "$NIX_STORE_DIR"/*-issue-13247-a.drv \ + "$NIX_STORE_DIR"/*-use-a-more-outputs.drv + + [[ ! -e "$(jq -r <"$TEST_ROOT"/a.json '.[0].outputs.out')" ]] + [[ ! -e "$(jq -r <"$TEST_ROOT"/a.json '.[1].outputs.out')" ]] + [[ ! -e "$(jq -r <"$TEST_ROOT"/a.json '.[2].outputs.first')" ]] + [[ ! -e "$(jq -r <"$TEST_ROOT"/a.json '.[2].outputs.second')" ]] +} + +delete + +buildViaSubstitute () { + nix build -f issue-13247.nix "$1" --no-link --max-jobs 0 --substituters "$cache" --no-require-sigs --offline --substitute +} + +# Substitue just the first output +buildViaSubstitute use-a-more-outputs^first + +# Should only fetch the output we asked for +[[ -d "$(jq -r <"$TEST_ROOT"/a.json '.[0].outputs.out')" ]] +[[ -f "$(jq -r <"$TEST_ROOT"/a.json '.[2].outputs.first')" ]] +[[ ! -e "$(jq -r <"$TEST_ROOT"/a.json '.[2].outputs.second')" ]] + +delete + +# Failure with 2.28 encountered in CI +requireDaemonNewerThan "2.29" + +# Substitue just the first output +# +# This derivation is the same after normalization, so we should get +# early cut-off, and thus a chance to download just the output we want +# rather than building more +buildViaSubstitute use-a-prime-more-outputs^first + +# Should only fetch the output we asked for +[[ -d "$(jq -r <"$TEST_ROOT"/a.json '.[0].outputs.out')" ]] +[[ -f "$(jq -r <"$TEST_ROOT"/a.json '.[2].outputs.first')" ]] + +# Output should *not* be here, this is the bug +[[ -e "$(jq -r <"$TEST_ROOT"/a.json '.[2].outputs.second')" ]] +skipTest "bug is not yet fixed" diff --git a/tests/functional/ca/meson.build b/tests/functional/ca/meson.build index ec34e9644b3..b1912fd869d 100644 --- a/tests/functional/ca/meson.build +++ b/tests/functional/ca/meson.build @@ -9,6 +9,7 @@ suites += { 'deps' : [], 'tests' : [ 'build-cache.sh', + 'build-delete.sh', 'build-with-garbage-path.sh', 'build.sh', 'concurrent-builds.sh', @@ -18,6 +19,8 @@ suites += { 'eval-store.sh', 'gc.sh', 'import-from-derivation.sh', + 'issue-13247.sh', + 'multiple-outputs.sh', 'new-build-cmd.sh', 'nix-copy.sh', 'nix-run.sh', diff --git a/tests/functional/ca/multiple-outputs.sh b/tests/functional/ca/multiple-outputs.sh new file mode 100644 index 00000000000..e4e05b5f5c1 --- /dev/null +++ b/tests/functional/ca/multiple-outputs.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +source common.sh + +export NIX_TESTS_CA_BY_DEFAULT=1 +cd .. +# shellcheck source=/dev/null +source ./multiple-outputs.sh diff --git a/tests/functional/ca/new-build-cmd.sh b/tests/functional/ca/new-build-cmd.sh index 408bfb0f698..e5cb644d117 100644 --- a/tests/functional/ca/new-build-cmd.sh +++ b/tests/functional/ca/new-build-cmd.sh @@ -4,4 +4,5 @@ source common.sh export NIX_TESTS_CA_BY_DEFAULT=1 cd .. +# shellcheck source=/dev/null source ./build.sh diff --git a/tests/functional/ca/nix-shell.sh b/tests/functional/ca/nix-shell.sh index 7b30b2ac858..05115c1262e 100755 --- a/tests/functional/ca/nix-shell.sh +++ b/tests/functional/ca/nix-shell.sh @@ -2,6 +2,8 @@ source common.sh +# shellcheck disable=SC2034 NIX_TESTS_CA_BY_DEFAULT=true cd .. +# shellcheck source=/dev/null source ./nix-shell.sh diff --git a/tests/functional/ca/post-hook.sh b/tests/functional/ca/post-hook.sh index 705bde9d4a0..e1adffc473d 100755 --- a/tests/functional/ca/post-hook.sh +++ b/tests/functional/ca/post-hook.sh @@ -6,6 +6,7 @@ requireDaemonNewerThan "2.4pre20210626" export NIX_TESTS_CA_BY_DEFAULT=1 cd .. +# shellcheck source=/dev/null source ./post-hook.sh diff --git a/tests/functional/ca/recursive.sh b/tests/functional/ca/recursive.sh index cd6736b24a3..e3fb98ab28e 100755 --- a/tests/functional/ca/recursive.sh +++ b/tests/functional/ca/recursive.sh @@ -6,4 +6,5 @@ requireDaemonNewerThan "2.4pre20210623" export NIX_TESTS_CA_BY_DEFAULT=1 cd .. +# shellcheck source=/dev/null source ./recursive.sh diff --git a/tests/functional/ca/repl.sh b/tests/functional/ca/repl.sh index 0bbbebd8578..f96ecfcf232 100644 --- a/tests/functional/ca/repl.sh +++ b/tests/functional/ca/repl.sh @@ -3,5 +3,5 @@ source common.sh export NIX_TESTS_CA_BY_DEFAULT=1 - +# shellcheck source=/dev/null cd .. && source repl.sh diff --git a/tests/functional/ca/selfref-gc.sh b/tests/functional/ca/selfref-gc.sh index 24877889459..fdee6b07ab0 100755 --- a/tests/functional/ca/selfref-gc.sh +++ b/tests/functional/ca/selfref-gc.sh @@ -4,8 +4,9 @@ source common.sh requireDaemonNewerThan "2.4pre20210626" -enableFeatures "ca-derivations nix-command flakes" +enableFeatures "ca-derivations" export NIX_TESTS_CA_BY_DEFAULT=1 cd .. +# shellcheck source=/dev/null source ./selfref-gc.sh diff --git a/tests/functional/ca/why-depends.sh b/tests/functional/ca/why-depends.sh index 0af8a544006..2a3c7d083c5 100644 --- a/tests/functional/ca/why-depends.sh +++ b/tests/functional/ca/why-depends.sh @@ -3,5 +3,5 @@ source common.sh export NIX_TESTS_CA_BY_DEFAULT=1 - +# shellcheck source=/dev/null cd .. && source why-depends.sh diff --git a/tests/functional/characterisation-test-infra.sh b/tests/functional/characterisation-test-infra.sh index 27945455061..fecae29e809 100755 --- a/tests/functional/characterisation-test-infra.sh +++ b/tests/functional/characterisation-test-infra.sh @@ -40,7 +40,7 @@ echo Bye! > "$TEST_ROOT/expected" diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" (( "$badDiff" == 1 )) ) -[[ "$(echo Bye! )" == $(< "$TEST_ROOT/expected") ]] +[[ "Bye!" == $(< "$TEST_ROOT/expected") ]] # _NIX_TEST_ACCEPT=1 matches non-empty echo Hi! > "$TEST_ROOT/got" @@ -57,7 +57,7 @@ echo Bye! > "$TEST_ROOT/expected" _NIX_TEST_ACCEPT=1 diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" (( "$badDiff" == 1 )) ) -[[ "$(echo Hi! )" == $(< "$TEST_ROOT/expected") ]] +[[ "Hi!" == $(< "$TEST_ROOT/expected") ]] # second time succeeds ( diffAndAcceptInner test "$TEST_ROOT/got" "$TEST_ROOT/expected" diff --git a/tests/functional/check.sh b/tests/functional/check.sh index a1c6decf5b5..26050613872 100755 --- a/tests/functional/check.sh +++ b/tests/functional/check.sh @@ -52,10 +52,10 @@ test_custom_build_dir() { nix-build check.nix -A failed --argstr checkBuildId "$checkBuildId" \ --no-out-link --keep-failed --option build-dir "$TEST_ROOT/custom-build-dir" 2> "$TEST_ROOT/log" || status=$? [ "$status" = "100" ] - [[ 1 == "$(count "$customBuildDir/nix-build-"*)" ]] - local buildDir=("$customBuildDir/nix-build-"*) + [[ 1 == "$(count "$customBuildDir/nix-"*)" ]] + local buildDir=("$customBuildDir/nix-"*) if [[ "${#buildDir[@]}" -ne 1 ]]; then - echo "expected one nix-build-* directory, got: ${buildDir[*]}" >&2 + echo "expected one nix-* directory, got: ${buildDir[*]}" >&2 exit 1 fi if [[ -e ${buildDir[*]}/build ]]; then diff --git a/tests/functional/common/functions.sh b/tests/functional/common/functions.sh index 1b2ec8fe0e8..fd59385762d 100644 --- a/tests/functional/common/functions.sh +++ b/tests/functional/common/functions.sh @@ -73,6 +73,7 @@ startDaemon() { fi # Start the daemon, wait for the socket to appear. rm -f "$NIX_DAEMON_SOCKET_PATH" + # TODO: remove the nix-command feature when we're no longer testing against old daemons. PATH=$DAEMON_PATH nix --extra-experimental-features 'nix-command' daemon & _NIX_TEST_DAEMON_PID=$! export _NIX_TEST_DAEMON_PID @@ -132,11 +133,11 @@ restartDaemon() { } isDaemonNewer () { - [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 - local requiredVersion="$1" - local daemonVersion - daemonVersion=$("$NIX_DAEMON_PACKAGE/bin/nix" daemon --version | cut -d' ' -f3) - [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] + [[ -n "${NIX_DAEMON_PACKAGE:-}" ]] || return 0 + local requiredVersion="$1" + local daemonVersion + daemonVersion=$("$NIX_DAEMON_PACKAGE/bin/nix" daemon --version | sed 's/.*) //') + [[ $(nix eval --expr "builtins.compareVersions ''$daemonVersion'' ''$requiredVersion''") -ge 0 ]] } skipTest () { diff --git a/tests/functional/common/init.sh b/tests/functional/common/init.sh index 66b44c76f69..7f28a09d753 100755 --- a/tests/functional/common/init.sh +++ b/tests/functional/common/init.sh @@ -12,7 +12,7 @@ if isTestOnNixOS; then ! test -e "$test_nix_conf" cat > "$test_nix_conf" < "$NIX_CONF_DIR"/nix.conf < "$NIX_CONF_DIR"/nix.conf.extra < $out/foobar +# shellcheck disable=SC2154 +mkdir "$out" +echo "$(cat "$input1"/foo)$(cat "$input2"/bar)" > "$out"/foobar -ln -s $input2 $out/reference-to-input-2 +ln -s "$input2" "$out"/reference-to-input-2 # Self-reference. -ln -s $out $out/self +ln -s "$out" "$out"/self # Executable. -echo program > $out/program -chmod +x $out/program +echo program > "$out"/program +chmod +x "$out"/program + +echo '1 + 2' > "$out"/foo.nix echo FOO diff --git a/tests/functional/dependencies.sh b/tests/functional/dependencies.sh index 972bc5a9bd6..1f529a3f9f6 100755 --- a/tests/functional/dependencies.sh +++ b/tests/functional/dependencies.sh @@ -11,22 +11,22 @@ echo "derivation is $drvPath" nix-store -q --tree "$drvPath" | grep '───.*builder-dependencies-input-1.sh' # Test Graphviz graph generation. -nix-store -q --graph "$drvPath" > $TEST_ROOT/graph +nix-store -q --graph "$drvPath" > "$TEST_ROOT"/graph if test -n "$dot"; then # Does it parse? - $dot < $TEST_ROOT/graph + $dot < "$TEST_ROOT"/graph fi # Test GraphML graph generation -nix-store -q --graphml "$drvPath" > $TEST_ROOT/graphml +nix-store -q --graphml "$drvPath" > "$TEST_ROOT"/graphml outPath=$(nix-store -rvv "$drvPath") || fail "build failed" # Test Graphviz graph generation. -nix-store -q --graph "$outPath" > $TEST_ROOT/graph +nix-store -q --graph "$outPath" > "$TEST_ROOT"/graph if test -n "$dot"; then # Does it parse? - $dot < $TEST_ROOT/graph + $dot < "$TEST_ROOT"/graph fi nix-store -q --tree "$outPath" | grep '───.*dependencies-input-2' @@ -53,7 +53,7 @@ input2OutPath=$(echo "$deps" | grep "dependencies-input-2") nix-store -q --referrers-closure "$input2OutPath" | grep "$outPath" # Check that the derivers are set properly. -test $(nix-store -q --deriver "$outPath") = "$drvPath" +test "$(nix-store -q --deriver "$outPath")" = "$drvPath" nix-store -q --deriver "$input2OutPath" | grepQuiet -- "-input-2.drv" # --valid-derivers returns the currently single valid .drv file @@ -68,9 +68,9 @@ test "$(nix-store -q --valid-derivers "$outPath" | sort)" = "$(sort <<< "$drvPat TODO_NixOS # The following --delete fails, because it seems to be still alive. This might be caused by a different test using the same path. We should try make the derivations unique, e.g. naming after tests, and adding a timestamp that's constant for that test script run. # check that nix-store --valid-derivers only returns existing drv -nix-store --delete "$drvPath" +nix-store --delete "$drvPath" --ignore-liveness test "$(nix-store -q --valid-derivers "$outPath")" = "$drvPath2" # check that --valid-derivers returns nothing when there are no valid derivers -nix-store --delete "$drvPath2" +nix-store --delete "$drvPath2" --ignore-liveness test -z "$(nix-store -q --valid-derivers "$outPath")" diff --git a/tests/functional/dump-db.sh b/tests/functional/dump-db.sh index 14181b4b63d..70d79e9fbfa 100755 --- a/tests/functional/dump-db.sh +++ b/tests/functional/dump-db.sh @@ -8,19 +8,18 @@ needLocalStore "--dump-db requires a local store" clearStore -path=$(nix-build dependencies.nix -o $TEST_ROOT/result) +nix-build dependencies.nix -o "$TEST_ROOT"/result +deps="$(nix-store -qR "$TEST_ROOT"/result)" -deps="$(nix-store -qR $TEST_ROOT/result)" +nix-store --dump-db > "$TEST_ROOT"/dump -nix-store --dump-db > $TEST_ROOT/dump +rm -rf "$NIX_STATE_DIR"/db -rm -rf $NIX_STATE_DIR/db +nix-store --load-db < "$TEST_ROOT"/dump -nix-store --load-db < $TEST_ROOT/dump - -deps2="$(nix-store -qR $TEST_ROOT/result)" +deps2="$(nix-store -qR "$TEST_ROOT"/result)" [ "$deps" = "$deps2" ]; -nix-store --dump-db > $TEST_ROOT/dump2 -cmp $TEST_ROOT/dump $TEST_ROOT/dump2 +nix-store --dump-db > "$TEST_ROOT"/dump2 +cmp "$TEST_ROOT"/dump "$TEST_ROOT"/dump2 diff --git a/tests/functional/dyn-drv/build-built-drv.sh b/tests/functional/dyn-drv/build-built-drv.sh index 49d61c6ce26..78db413274a 100644 --- a/tests/functional/dyn-drv/build-built-drv.sh +++ b/tests/functional/dyn-drv/build-built-drv.sh @@ -23,4 +23,4 @@ requireDaemonNewerThan "2.30pre20250515" out2=$(nix build "${drvDep}^out^out" --no-link) -test $out1 == $out2 +test "$out1" == "$out2" diff --git a/tests/functional/dyn-drv/common.sh b/tests/functional/dyn-drv/common.sh index 0d95881b6ab..ca24498d0bb 100644 --- a/tests/functional/dyn-drv/common.sh +++ b/tests/functional/dyn-drv/common.sh @@ -1,3 +1,4 @@ +# shellcheck shell=bash source ../common.sh # Need backend to support text-hashing too diff --git a/tests/functional/dyn-drv/dep-built-drv.sh b/tests/functional/dyn-drv/dep-built-drv.sh index e9a8b6b832c..f5be23645c7 100644 --- a/tests/functional/dyn-drv/dep-built-drv.sh +++ b/tests/functional/dyn-drv/dep-built-drv.sh @@ -11,4 +11,4 @@ clearStore out2=$(nix-build ./text-hashed-output.nix -A wrapper --no-out-link) -diff -r $out1 $out2 +diff -r "$out1" "$out2" diff --git a/tests/functional/dyn-drv/eval-outputOf.sh b/tests/functional/dyn-drv/eval-outputOf.sh index 3681bd09889..82aa04ca241 100644 --- a/tests/functional/dyn-drv/eval-outputOf.sh +++ b/tests/functional/dyn-drv/eval-outputOf.sh @@ -3,7 +3,7 @@ source ./common.sh # Without the dynamic-derivations XP feature, we don't have the builtin. -nix --experimental-features 'nix-command' eval --impure --expr \ +nix --experimental-features '' eval --impure --expr \ 'assert ! (builtins ? outputOf); ""' # Test that a string is required. @@ -12,14 +12,14 @@ nix --experimental-features 'nix-command' eval --impure --expr \ # object that could be coerced to a string. We might liberalise this in # the future so it does work, but there are some design questions to # resolve first. Adding a test so we don't liberalise it by accident. -expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \ +expectStderr 1 nix --experimental-features 'dynamic-derivations' eval --impure --expr \ 'builtins.outputOf (import ../dependencies.nix {}) "out"' \ | grepQuiet "expected a string but found a set" # Test that "DrvDeep" string contexts are not supported at this time # # Like the above, this is a restriction we could relax later. -expectStderr 1 nix --experimental-features 'nix-command dynamic-derivations' eval --impure --expr \ +expectStderr 1 nix --experimental-features 'dynamic-derivations' eval --impure --expr \ 'builtins.outputOf (import ../dependencies.nix {}).drvPath "out"' \ | grepQuiet "has a context which refers to a complete source and binary closure. This is not supported at this time" diff --git a/tests/functional/dyn-drv/failing-outer.sh b/tests/functional/dyn-drv/failing-outer.sh index 3feda74fbed..596efe43dbd 100644 --- a/tests/functional/dyn-drv/failing-outer.sh +++ b/tests/functional/dyn-drv/failing-outer.sh @@ -9,4 +9,4 @@ expected=100 if [[ -v NIX_DAEMON_PACKAGE ]]; then expected=1; fi # work around the daemon not returning a 100 status correctly expectStderr "$expected" nix-build ./text-hashed-output.nix -A failingWrapper --no-out-link \ - | grepQuiet "build of '.*use-dynamic-drv-in-non-dynamic-drv-wrong.drv' failed" + | grepQuiet "build of resolved derivation '.*use-dynamic-drv-in-non-dynamic-drv-wrong.drv' failed" diff --git a/tests/functional/dyn-drv/non-trivial.nix b/tests/functional/dyn-drv/non-trivial.nix index 5cfafbb62f5..3c24ac2ee4b 100644 --- a/tests/functional/dyn-drv/non-trivial.nix +++ b/tests/functional/dyn-drv/non-trivial.nix @@ -62,12 +62,15 @@ builtins.outputOf "hashAlgo": "sha256" } }, - "system": "${system}" + "system": "${system}", + "version": 3 } EOF - drvs[$word]="$(echo "$json" | nix derivation add)" + drvPath=$(echo "$json" | nix derivation add) + storeDir=$(dirname "$drvPath") + drvs[$word]="$(basename "$drvPath")" done - cp "''${drvs[e]}" $out + cp "''${storeDir}/''${drvs[e]}" $out ''; __contentAddressed = true; diff --git a/tests/functional/dyn-drv/old-daemon-error-hack.sh b/tests/functional/dyn-drv/old-daemon-error-hack.sh index 43b04997396..02129bd734d 100644 --- a/tests/functional/dyn-drv/old-daemon-error-hack.sh +++ b/tests/functional/dyn-drv/old-daemon-error-hack.sh @@ -1,3 +1,4 @@ +# shellcheck shell=bash # Purposely bypassing our usual common for this subgroup source ../common.sh diff --git a/tests/functional/dyn-drv/recursive-mod-json.nix b/tests/functional/dyn-drv/recursive-mod-json.nix index 2d46e4e2e02..2a7a1ec1a81 100644 --- a/tests/functional/dyn-drv/recursive-mod-json.nix +++ b/tests/functional/dyn-drv/recursive-mod-json.nix @@ -13,7 +13,7 @@ mkDerivation rec { drv = builtins.unsafeDiscardOutputDependency (import ./text-hashed-output.nix).hello.drvPath; buildCommand = '' - export NIX_CONFIG='experimental-features = nix-command ca-derivations' + export NIX_CONFIG='experimental-features = ca-derivations' PATH=${builtins.getEnv "EXTRA_PATH"}:$PATH diff --git a/tests/functional/dyn-drv/recursive-mod-json.sh b/tests/functional/dyn-drv/recursive-mod-json.sh index 0698b81bd11..01e8f16e956 100644 --- a/tests/functional/dyn-drv/recursive-mod-json.sh +++ b/tests/functional/dyn-drv/recursive-mod-json.sh @@ -1,3 +1,4 @@ +# shellcheck shell=bash source common.sh # FIXME @@ -10,18 +11,18 @@ restartDaemon clearStore -rm -f $TEST_ROOT/result +rm -f "$TEST_ROOT"/result -EXTRA_PATH=$(dirname $(type -p nix)):$(dirname $(type -p jq)) +EXTRA_PATH=$(dirname "$(type -p nix)"):$(dirname "$(type -p jq)") export EXTRA_PATH # Will produce a drv metaDrv=$(nix-instantiate ./recursive-mod-json.nix) # computed "dynamic" derivation -drv=$(nix-store -r $metaDrv) +drv=$(nix-store -r "$metaDrv") # build that dyn drv -res=$(nix-store -r $drv) +res=$(nix-store -r "$drv") -grep 'I am alive!' $res/hello +grep 'I am alive!' "$res"/hello diff --git a/tests/functional/eval-store.sh b/tests/functional/eval-store.sh index 202e7b00413..9f4b3b03646 100755 --- a/tests/functional/eval-store.sh +++ b/tests/functional/eval-store.sh @@ -6,6 +6,7 @@ TODO_NixOS # Using `--eval-store` with the daemon will eventually copy everything # to the build store, invalidating most of the tests here +# shellcheck disable=SC1111 needLocalStore "“--eval-store” doesn't achieve much with the daemon" eval_store=$TEST_ROOT/eval-store @@ -15,7 +16,7 @@ rm -rf "$eval_store" nix build -f dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result" [[ -e $TEST_ROOT/result/foobar ]] -if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then +if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then # Resolved CA derivations are written to store for building # # TODO when we something more systematic @@ -23,32 +24,36 @@ if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then # between scratch storage for building and the final destination # store, we'll be able to make this unconditional again -- resolved # derivations should only appear in the scratch store. - (! ls $NIX_STORE_DIR/*.drv) + (! ls "$NIX_STORE_DIR"/*.drv) fi -ls $eval_store/nix/store/*.drv +ls "$eval_store"/nix/store/*.drv clearStore rm -rf "$eval_store" nix-instantiate dependencies.nix --eval-store "$eval_store" -(! ls $NIX_STORE_DIR/*.drv) -ls $eval_store/nix/store/*.drv +(! ls "$NIX_STORE_DIR"/*.drv) +ls "$eval_store"/nix/store/*.drv clearStore rm -rf "$eval_store" nix-build dependencies.nix --eval-store "$eval_store" -o "$TEST_ROOT/result" [[ -e $TEST_ROOT/result/foobar ]] -if [[ ! -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then +if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then # See above - (! ls $NIX_STORE_DIR/*.drv) + (! ls "$NIX_STORE_DIR"/*.drv) fi -ls $eval_store/nix/store/*.drv +ls "$eval_store"/nix/store/*.drv clearStore rm -rf "$eval_store" # Confirm that import-from-derivation builds on the build store [[ $(nix eval --eval-store "$eval_store?require-sigs=false" --impure --raw --file ./ifd.nix) = hi ]] -ls $NIX_STORE_DIR/*dependencies-top/foobar -(! ls $eval_store/nix/store/*dependencies-top/foobar) +ls "$NIX_STORE_DIR"/*dependencies-top/foobar +(! ls "$eval_store"/nix/store/*dependencies-top/foobar) + +# Can't write .drv by default +(! nix-instantiate dependencies.nix --eval-store "dummy://") +nix-instantiate dependencies.nix --eval-store "dummy://?read-only=false" diff --git a/tests/functional/experimental-features.sh b/tests/functional/experimental-features.sh index d7216992d4e..43514e2295a 100755 --- a/tests/functional/experimental-features.sh +++ b/tests/functional/experimental-features.sh @@ -11,12 +11,12 @@ source common.sh # # With flakes, flake options should show up # # function grep_both_ways { -# nix --experimental-features 'nix-command' "$@" | grepQuietInverse flake -# nix --experimental-features 'nix-command flakes' "$@" | grepQuiet flake +# nix --experimental-features '' "$@" | grepQuietInverse flake +# nix --experimental-features '' "$@" | grepQuiet flake # # # Also, the order should not matter -# nix "$@" --experimental-features 'nix-command' | grepQuietInverse flake -# nix "$@" --experimental-features 'nix-command flakes' | grepQuiet flake +# nix "$@" --experimental-features '' | grepQuietInverse flake +# nix "$@" --experimental-features '' | grepQuiet flake # } # # # Simple case, the configuration effects the running command @@ -29,62 +29,53 @@ source common.sh # with a warning if the experimental feature is not enabled. The order of the # `setting = value` lines in the configuration should not matter. -# 'flakes' experimental-feature is disabled before, ignore and warn -NIX_CONFIG=' - experimental-features = nix-command - accept-flake-config = true -' expect 1 nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +xpFeature=auto-allocate-uids +gatedSetting=auto-allocate-uids + +# Experimental feature is disabled before, ignore and warn. +NIX_CONFIG=" + experimental-features = + $gatedSetting = true +" expect 1 nix config show $gatedSetting 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr [[ $(cat "$TEST_ROOT/stdout") = '' ]] -grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" "$TEST_ROOT/stderr" -grepQuiet "error: could not find setting 'accept-flake-config'" "$TEST_ROOT/stderr" +grepQuiet "error: could not find setting '$gatedSetting'" "$TEST_ROOT/stderr" -# 'flakes' experimental-feature is disabled after, ignore and warn -NIX_CONFIG=' - accept-flake-config = true - experimental-features = nix-command -' expect 1 nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +# Experimental feature is disabled after, ignore and warn. +NIX_CONFIG=" + $gatedSetting = true + experimental-features = +" expect 1 nix config show $gatedSetting 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr [[ $(cat "$TEST_ROOT/stdout") = '' ]] -grepQuiet "Ignoring setting 'accept-flake-config' because experimental feature 'flakes' is not enabled" "$TEST_ROOT/stderr" -grepQuiet "error: could not find setting 'accept-flake-config'" "$TEST_ROOT/stderr" +grepQuiet "error: could not find setting '$gatedSetting'" "$TEST_ROOT/stderr" -# 'flakes' experimental-feature is enabled before, process -NIX_CONFIG=' - experimental-features = nix-command flakes - accept-flake-config = true -' nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +# Experimental feature is enabled before, process. +NIX_CONFIG=" + experimental-features = $xpFeature + $gatedSetting = true +" nix config show $gatedSetting 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr grepQuiet "true" "$TEST_ROOT/stdout" -grepQuietInverse "Ignoring setting 'accept-flake-config'" "$TEST_ROOT/stderr" -# 'flakes' experimental-feature is enabled after, process -NIX_CONFIG=' - accept-flake-config = true - experimental-features = nix-command flakes -' nix config show accept-flake-config 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr +# Experimental feature is enabled after, process. +NIX_CONFIG=" + $gatedSetting = true + experimental-features = $xpFeature +" nix config show $gatedSetting 1>"$TEST_ROOT"/stdout 2>"$TEST_ROOT"/stderr grepQuiet "true" "$TEST_ROOT/stdout" -grepQuietInverse "Ignoring setting 'accept-flake-config'" "$TEST_ROOT/stderr" +grepQuietInverse "Ignoring setting '$gatedSetting'" "$TEST_ROOT/stderr" function exit_code_both_ways { - expect 1 nix --experimental-features 'nix-command' "$@" 1>/dev/null - nix --experimental-features 'nix-command flakes' "$@" 1>/dev/null + expect 1 nix --experimental-features '' "$@" 1>/dev/null + nix --experimental-features "$xpFeature" "$@" 1>/dev/null # Also, the order should not matter - expect 1 nix "$@" --experimental-features 'nix-command' 1>/dev/null - nix "$@" --experimental-features 'nix-command flakes' 1>/dev/null + expect 1 nix "$@" --experimental-features '' 1>/dev/null + nix "$@" --experimental-features "$xpFeature" 1>/dev/null } -exit_code_both_ways show-config --flake-registry 'https://no' +exit_code_both_ways config show --auto-allocate-uids # Double check these are stable nix --experimental-features '' --help 1>/dev/null nix --experimental-features '' doctor --help 1>/dev/null nix --experimental-features '' repl --help 1>/dev/null nix --experimental-features '' upgrade-nix --help 1>/dev/null - -# These 3 arguments are currently given to all commands, which is wrong (as not -# all care). To deal with fixing later, we simply make them require the -# nix-command experimental features --- it so happens that the commands we wish -# stabilizing to do not need them anyways. -for arg in '--print-build-logs' '--offline' '--refresh'; do - nix --experimental-features 'nix-command' "$arg" --help 1>/dev/null - expect 1 nix --experimental-features '' "$arg" --help 1>/dev/null -done diff --git a/tests/functional/export-graph.sh b/tests/functional/export-graph.sh index b507b6d3a12..0490b580d1b 100755 --- a/tests/functional/export-graph.sh +++ b/tests/functional/export-graph.sh @@ -8,27 +8,29 @@ clearStore clearProfiles checkRef() { - nix-store -q --references $TEST_ROOT/result | grepQuiet "$1"'$' || fail "missing reference $1" + nix-store -q --references "$TEST_ROOT"/result | grepQuiet "$1"'$' || fail "missing reference $1" } # Test the export of the runtime dependency graph. -outPath=$(nix-build ./export-graph.nix -A 'foo."bar.runtimeGraph"' -o $TEST_ROOT/result) +outPath=$(nix-build ./export-graph.nix -A 'foo."bar.runtimeGraph"' -o "$TEST_ROOT"/result) -test $(nix-store -q --references $TEST_ROOT/result | wc -l) = 3 || fail "bad nr of references" +test "$(nix-store -q --references "$TEST_ROOT"/result | wc -l)" = 3 || fail "bad nr of references" checkRef input-2 -for i in $(cat $outPath); do checkRef $i; done +# shellcheck disable=SC2013 +for i in $(cat "$outPath"); do checkRef "$i"; done # Test the export of the build-time dependency graph. nix-store --gc # should force rebuild of input-1 -outPath=$(nix-build ./export-graph.nix -A 'foo."bar.buildGraph"' -o $TEST_ROOT/result) +outPath=$(nix-build ./export-graph.nix -A 'foo."bar.buildGraph"' -o "$TEST_ROOT"/result) checkRef input-1 checkRef input-1.drv checkRef input-2 checkRef input-2.drv -for i in $(cat $outPath); do checkRef $i; done +# shellcheck disable=SC2013 +for i in $(cat "$outPath"); do checkRef "$i"; done diff --git a/tests/functional/export.sh b/tests/functional/export.sh index 3e895a5402d..a74efa91d80 100755 --- a/tests/functional/export.sh +++ b/tests/functional/export.sh @@ -7,34 +7,74 @@ TODO_NixOS clearStore outPath=$(nix-build dependencies.nix --no-out-link) +drvPath=$(nix path-info --json "$outPath" | jq -r .\""$outPath"\".deriver) -nix-store --export $outPath > $TEST_ROOT/exp +nix-store --export "$outPath" > "$TEST_ROOT"/exp +expectStderr 1 nix nario export "$outPath" | grepQuiet "required argument.*missing" +nix nario export --format 1 "$outPath" > "$TEST_ROOT/exp2" +cmp "$TEST_ROOT/exp" "$TEST_ROOT/exp2" -nix-store --export $(nix-store -qR $outPath) > $TEST_ROOT/exp_all +# shellcheck disable=SC2046 +nix-store --export $(nix-store -qR "$outPath") > "$TEST_ROOT"/exp_all -if nix-store --export $outPath >/dev/full ; then +nix nario export --format 1 -r "$outPath" > "$TEST_ROOT"/exp_all2 +cmp "$TEST_ROOT/exp_all" "$TEST_ROOT/exp_all2" + +if nix-store --export "$outPath" >/dev/full ; then echo "exporting to a bad file descriptor should fail" exit 1 fi - clearStore -if nix-store --import < $TEST_ROOT/exp; then +if nix-store --import < "$TEST_ROOT"/exp; then echo "importing a non-closure should fail" exit 1 fi - clearStore -nix-store --import < $TEST_ROOT/exp_all - -nix-store --export $(nix-store -qR $outPath) > $TEST_ROOT/exp_all2 +nix-store --import < "$TEST_ROOT"/exp_all +# shellcheck disable=SC2046 +nix-store --export $(nix-store -qR "$outPath") > "$TEST_ROOT"/exp_all2 clearStore # Regression test: the derivers in exp_all2 are empty, which shouldn't # cause a failure. -nix-store --import < $TEST_ROOT/exp_all2 +nix-store --import < "$TEST_ROOT"/exp_all2 + +# Test `nix nario import` on files created by `nix-store --export`. +clearStore +expectStderr 1 nix nario import < "$TEST_ROOT"/exp_all | grepQuiet "lacks a signature" +nix nario import --no-check-sigs < "$TEST_ROOT"/exp_all +nix path-info "$outPath" + +# Test `nix nario list`. +nix nario list < "$TEST_ROOT"/exp_all +nix nario list < "$TEST_ROOT"/exp_all | grepQuiet ".*dependencies-input-0.*bytes" +nix nario list -lR < "$TEST_ROOT"/exp_all | grepQuiet "dr-xr-xr-x .*0 $outPath" +nix nario list -lR < "$TEST_ROOT"/exp_all | grepQuiet "lrwxrwxrwx .*0 $outPath/self -> $outPath" +nix nario list -lR < "$TEST_ROOT"/exp_all | grepQuiet -- "-r--r--r-- .*7 $outPath/foobar" + +# Test format 2 (including signatures). +nix key generate-secret --key-name my-key > "$TEST_ROOT"/secret +public_key=$(nix key convert-secret-to-public < "$TEST_ROOT"/secret) +nix store sign --key-file "$TEST_ROOT/secret" -r "$outPath" +nix nario export --format 2 -r "$outPath" > "$TEST_ROOT"/exp_all +clearStore +expectStderr 1 nix nario import < "$TEST_ROOT"/exp_all | grepQuiet "lacks a signature" +nix nario import --trusted-public-keys "$public_key" < "$TEST_ROOT"/exp_all +[[ $(nix path-info --json "$outPath" | jq -r .[].signatures[]) =~ my-key: ]] + +# Test json listing. +json=$(nix nario list --json -R < "$TEST_ROOT/exp_all") +[[ $(printf "%s" "$json" | jq -r ".paths.\"$outPath\".deriver") = "$drvPath" ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$outPath\".contents.type") = directory ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$outPath\".contents.entries.foobar.type") = regular ]] +[[ $(printf "%s" "$json" | jq ".paths.\"$outPath\".contents.entries.foobar.size") = 7 ]] + +json=$(nix nario list --json < "$TEST_ROOT/exp_all") +[[ $(printf "%s" "$json" | jq -r ".paths.\"$outPath\".deriver") = "$drvPath" ]] +[[ $(printf "%s" "$json" | jq -r ".paths.\"$outPath\".contents.type") = null ]] diff --git a/tests/functional/external-builders.sh b/tests/functional/external-builders.sh new file mode 100644 index 00000000000..4c1d5636a0a --- /dev/null +++ b/tests/functional/external-builders.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +source common.sh + +TODO_NixOS + +needLocalStore "'--external-builders' can’t be used with the daemon" + +expr="$TEST_ROOT/expr.nix" +cat > "$expr" < \$out + ''; +} +EOF + +external_builder="$TEST_ROOT/external-builder.sh" +cat > "$external_builder" <> \$out +EOF +chmod +x "$external_builder" + +nix build -L --file "$expr" --out-link "$TEST_ROOT/result" \ + --extra-experimental-features external-builders \ + --external-builders "[{\"systems\": [\"x68_46-xunil\"], \"args\": [\"bla\"], \"program\": \"$external_builder\"}]" + +[[ $(cat "$TEST_ROOT/result") = foobar ]] diff --git a/tests/functional/fetchClosure.sh b/tests/functional/fetchClosure.sh index 7ef635d3677..85a83d19225 100755 --- a/tests/functional/fetchClosure.sh +++ b/tests/functional/fetchClosure.sh @@ -17,14 +17,14 @@ requireDaemonNewerThan "2.16.0pre20230524" # Initialize binary cache. nonCaPath=$(nix build --json --file ./dependencies.nix --no-link | jq -r .[].outputs.out) -caPath=$(nix store make-content-addressed --json $nonCaPath | jq -r '.rewrites | map(.) | .[]') -nix copy --to file://$cacheDir $nonCaPath +caPath=$(nix store make-content-addressed --json "$nonCaPath" | jq -r '.rewrites | map(.) | .[]') +nix copy --to file://"$cacheDir" "$nonCaPath" # Test basic fetchClosure rewriting from non-CA to CA. clearStore -[ ! -e $nonCaPath ] -[ ! -e $caPath ] +[ ! -e "$nonCaPath" ] +[ ! -e "$caPath" ] [[ $(nix eval -v --raw --expr " builtins.fetchClosure { @@ -32,10 +32,10 @@ clearStore fromPath = $nonCaPath; toPath = $caPath; } -") = $caPath ]] +") = "$caPath" ]] -[ ! -e $nonCaPath ] -[ -e $caPath ] +[ ! -e "$nonCaPath" ] +[ -e "$caPath" ] clearStore @@ -55,7 +55,7 @@ if [[ "$NIX_REMOTE" != "daemon" ]]; then # TODO: Should the closure be rejected, despite single user mode? # [ ! -e $nonCaPath ] - [ ! -e $caPath ] + [ ! -e "$caPath" ] # We can use non-CA paths when we ask explicitly. [[ $(nix eval --raw --no-require-sigs --expr " @@ -64,15 +64,15 @@ if [[ "$NIX_REMOTE" != "daemon" ]]; then fromPath = $nonCaPath; inputAddressed = true; } - ") = $nonCaPath ]] + ") = "$nonCaPath" ]] - [ -e $nonCaPath ] - [ ! -e $caPath ] + [ -e "$nonCaPath" ] + [ ! -e "$caPath" ] fi -[ ! -e $caPath ] +[ ! -e "$caPath" ] # 'toPath' set to empty string should fail but print the expected path. expectStderr 1 nix eval -v --json --expr " @@ -84,39 +84,49 @@ expectStderr 1 nix eval -v --json --expr " " | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath" # If fromPath is CA, then toPath isn't needed. -nix copy --to file://$cacheDir $caPath +nix copy --to file://"$cacheDir" "$caPath" clearStore -[ ! -e $caPath ] +[ ! -e "$caPath" ] [[ $(nix eval -v --raw --expr " builtins.fetchClosure { fromStore = \"file://$cacheDir\"; fromPath = $caPath; } -") = $caPath ]] +") = "$caPath" ]] -[ -e $caPath ] +[ -e "$caPath" ] + +# Test import-from-derivation on the result of fetchClosure. +[[ $(nix eval -v --expr " + import \"\${builtins.fetchClosure { + fromStore = \"file://$cacheDir\"; + fromPath = $caPath; + }}/foo.nix\" +") = 3 ]] # Check that URL query parameters aren't allowed. clearStore narCache=$TEST_ROOT/nar-cache -rm -rf $narCache +rm -rf "$narCache" (! nix eval -v --raw --expr " builtins.fetchClosure { fromStore = \"file://$cacheDir?local-nar-cache=$narCache\"; fromPath = $caPath; } ") -(! [ -e $narCache ]) +# shellcheck disable=SC2235 +(! [ -e "$narCache" ]) # If toPath is specified but wrong, we check it (only) when the path is missing. clearStore -badPath=$(echo $caPath | sed -e 's!/store/................................-!/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-!') +# shellcheck disable=SC2001 +badPath=$(echo "$caPath" | sed -e 's!/store/................................-!/store/xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-!') -[ ! -e $badPath ] +[ ! -e "$badPath" ] expectStderr 1 nix eval -v --raw --expr " builtins.fetchClosure { @@ -126,11 +136,11 @@ expectStderr 1 nix eval -v --raw --expr " } " | grep "error: rewriting.*$nonCaPath.*yielded.*$caPath.*while.*$badPath.*was expected" -[ ! -e $badPath ] +[ ! -e "$badPath" ] # We only check it when missing, as a performance optimization similar to what we do for fixed output derivations. So if it's already there, we don't check it. # It would be nice for this to fail, but checking it would be too(?) slow. -[ -e $caPath ] +[ -e "$caPath" ] [[ $(nix eval -v --raw --expr " builtins.fetchClosure { @@ -138,7 +148,7 @@ expectStderr 1 nix eval -v --raw --expr " fromPath = $badPath; toPath = $caPath; } -") = $caPath ]] +") = "$caPath" ]] # However, if the output address is unexpected, we can report it diff --git a/tests/functional/fetchGit.sh b/tests/functional/fetchGit.sh index e7c9c77a5a1..be8b5cb34af 100755 --- a/tests/functional/fetchGit.sh +++ b/tests/functional/fetchGit.sh @@ -12,25 +12,25 @@ repo=$TEST_ROOT/./git export _NIX_FORCE_HTTP=1 -rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix $TEST_ROOT/worktree $TEST_ROOT/minimal - -git init $repo -git -C $repo config user.email "foobar@example.com" -git -C $repo config user.name "Foobar" - -echo utrecht > $repo/hello -touch $repo/.gitignore -git -C $repo add hello .gitignore -git -C $repo commit -m 'Bla1' -rev1=$(git -C $repo rev-parse HEAD) -git -C $repo tag -a tag1 -m tag1 - -echo world > $repo/hello -git -C $repo commit -m 'Bla2' -a -git -C $repo worktree add $TEST_ROOT/worktree -echo hello >> $TEST_ROOT/worktree/hello -rev2=$(git -C $repo rev-parse HEAD) -git -C $repo tag -a tag2 -m tag2 +rm -rf "$repo" "${repo}"-tmp "$TEST_HOME"/.cache/nix "$TEST_ROOT"/worktree "$TEST_ROOT"/minimal + +git init "$repo" +git -C "$repo" config user.email "foobar@example.com" +git -C "$repo" config user.name "Foobar" + +echo utrecht > "$repo"/hello +touch "$repo"/.gitignore +git -C "$repo" add hello .gitignore +git -C "$repo" commit -m 'Bla1' +rev1=$(git -C "$repo" rev-parse HEAD) +git -C "$repo" tag -a tag1 -m tag1 + +echo world > "$repo"/hello +git -C "$repo" commit -m 'Bla2' -a +git -C "$repo" worktree add "$TEST_ROOT"/worktree +echo hello >> "$TEST_ROOT"/worktree/hello +rev2=$(git -C "$repo" rev-parse HEAD) +git -C "$repo" tag -a tag2 -m tag2 # Check whether fetching in read-only mode works. nix-instantiate --eval -E "builtins.readFile ((builtins.fetchGit file://$TEST_ROOT/worktree) + \"/hello\") == \"utrecht\\n\"" @@ -40,52 +40,52 @@ unset _NIX_FORCE_HTTP expectStderr 0 nix eval -vvvv --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath" | grepQuiet "copying '$TEST_ROOT/worktree/' to the store" path0=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$TEST_ROOT/worktree).outPath") path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree { type = \"git\"; url = file://$TEST_ROOT/worktree; }).outPath") -[[ $path0 = $path0_ ]] +[[ $path0 = "$path0_" ]] path0_=$(nix eval --impure --raw --expr "(builtins.fetchTree git+file://$TEST_ROOT/worktree).outPath") -[[ $path0 = $path0_ ]] +[[ $path0 = "$path0_" ]] export _NIX_FORCE_HTTP=1 -[[ $(tail -n 1 $path0/hello) = "hello" ]] +[[ $(tail -n 1 "$path0"/hello) = "hello" ]] # Nuke the cache -rm -rf $TEST_HOME/.cache/nix +rm -rf "$TEST_HOME"/.cache/nix # Fetch the default branch. path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") -[[ $(cat $path/hello) = world ]] +[[ $(cat "$path"/hello) = world ]] # Fetch again. This should be cached. # NOTE: This has to be done before the test case below which tries to pack-refs # the reason being that the lookup on the cache uses the ref-file `/refs/heads/master` # which does not exist after packing. -mv $repo ${repo}-tmp +mv "$repo" "${repo}"-tmp path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") -[[ $path = $path2 ]] +[[ $path = "$path2" ]] [[ $(nix eval --impure --expr "(builtins.fetchGit file://$repo).revCount") = 2 ]] -[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).rev") = $rev2 ]] -[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).shortRev") = ${rev2:0:7} ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).rev") = "$rev2" ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).shortRev") = "${rev2:0:7}" ]] # Fetching with a explicit hash should succeed. path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") -[[ $path = $path2 ]] +[[ $path = "$path2" ]] path2=$(nix eval --refresh --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev1\"; }).outPath") -[[ $(cat $path2/hello) = utrecht ]] +[[ $(cat "$path2"/hello) = utrecht ]] -mv ${repo}-tmp $repo +mv "${repo}"-tmp "$repo" # Fetch when the cache has packed-refs # Regression test of #8822 -git -C $TEST_HOME/.cache/nix/gitv3/*/ pack-refs --all +git -C "$TEST_HOME"/.cache/nix/gitv3/*/ pack-refs --all path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") # Fetch a rev from another branch -git -C $repo checkout -b devtest -echo "different file" >> $TEST_ROOT/git/differentbranch -git -C $repo add differentbranch -git -C $repo commit -m 'Test2' -git -C $repo checkout master -devrev=$(git -C $repo rev-parse devtest) +git -C "$repo" checkout -b devtest +echo "different file" >> "$TEST_ROOT"/git/differentbranch +git -C "$repo" add differentbranch +git -C "$repo" commit -m 'Test2' +git -C "$repo" checkout master +devrev=$(git -C "$repo" rev-parse devtest) nix eval --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; }" [[ $(nix eval --raw --expr "builtins.readFile (builtins.fetchGit { url = file://$repo; rev = \"$devrev\"; allRefs = true; } + \"/differentbranch\")") = 'different file' ]] @@ -96,7 +96,7 @@ nix eval --raw --expr "builtins.fetchGit { url = file://$repo; rev = \"$devrev\" # Fetch using an explicit revision hash. path2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev2\"; }).outPath") -[[ $path = $path2 ]] +[[ $path = "$path2" ]] # In pure eval mode, fetchGit with a revision should succeed. [[ $(nix eval --raw --expr "builtins.readFile (fetchGit { url = file://$repo; rev = \"$rev2\"; } + \"/hello\")") = world ]] @@ -106,23 +106,23 @@ expectStderr 1 nix eval --expr 'builtins.fetchGit "file:///foo"' | grepQuiet "'f # Using a clean working tree should produce the same result. path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") -[[ $path = $path2 ]] +[[ $path = "$path2" ]] # Using an unclean tree should yield the tracked but uncommitted changes. -mkdir $repo/dir1 $repo/dir2 -echo foo > $repo/dir1/foo -echo bar > $repo/bar -echo bar > $repo/dir2/bar -git -C $repo add dir1/foo -git -C $repo rm hello +mkdir "$repo"/dir1 "$repo"/dir2 +echo foo > "$repo"/dir1/foo +echo bar > "$repo"/bar +echo bar > "$repo"/dir2/bar +git -C "$repo" add dir1/foo +git -C "$repo" rm hello unset _NIX_FORCE_HTTP path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") -[ ! -e $path2/hello ] -[ ! -e $path2/bar ] -[ ! -e $path2/dir2/bar ] -[ ! -e $path2/.git ] -[[ $(cat $path2/dir1/foo) = foo ]] +[ ! -e "$path2"/hello ] +[ ! -e "$path2"/bar ] +[ ! -e "$path2"/dir2/bar ] +[ ! -e "$path2"/.git ] +[[ $(cat "$path2"/dir1/foo) = foo ]] [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = 0000000000000000000000000000000000000000 ]] [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).dirtyRev") = "${rev2}-dirty" ]] @@ -130,16 +130,16 @@ path2=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") # ... unless we're using an explicit ref or rev. path3=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"master\"; }).outPath") -[[ $path = $path3 ]] +[[ $path = "$path3" ]] path3=$(nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; }).outPath") -[[ $path = $path3 ]] +[[ $path = "$path3" ]] # Committing should not affect the store path. -git -C $repo commit -m 'Bla3' -a +git -C "$repo" commit -m 'Bla3' -a path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$repo).outPath") -[[ $path2 = $path4 ]] +[[ $path2 = "$path4" ]] [[ $(nix eval --impure --expr "builtins.hasAttr \"rev\" (builtins.fetchGit $repo)") == "true" ]] [[ $(nix eval --impure --expr "builtins.hasAttr \"dirtyRev\" (builtins.fetchGit $repo)") == "false" ]] @@ -148,7 +148,7 @@ path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchGit file://$rep expect 102 nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-B5yIPHhEm0eysJKEsO7nqxprh9vcblFxpJG11gXJus1=\"; }).outPath" path5=$(nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-Hr8g6AqANb3xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath") -[[ $path = $path5 ]] +[[ $path = "$path5" ]] # Ensure that NAR hashes are checked. expectStderr 102 nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = \"$rev2\"; narHash = \"sha256-Hr8g6AqANb4xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath" | grepQuiet "error: NAR hash mismatch" @@ -157,22 +157,22 @@ expectStderr 102 nix eval --raw --expr "(builtins.fetchGit { url = $repo; rev = expectStderr 0 nix eval --raw --expr "(builtins.fetchGit { url = $repo; ref = \"tag2\"; narHash = \"sha256-Hr8g6AqANb3xqX28eu1XnjK/3ab8Gv6TJSnkb1LezG9=\"; }).outPath" | grepQuiet "warning: Input .* is unlocked" # tarball-ttl should be ignored if we specify a rev -echo delft > $repo/hello -git -C $repo add hello -git -C $repo commit -m 'Bla4' -rev3=$(git -C $repo rev-parse HEAD) +echo delft > "$repo"/hello +git -C "$repo" add hello +git -C "$repo" commit -m 'Bla4' +rev3=$(git -C "$repo" rev-parse HEAD) nix eval --tarball-ttl 3600 --expr "builtins.fetchGit { url = $repo; rev = \"$rev3\"; }" >/dev/null # Update 'path' to reflect latest master path=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") # Check behavior when non-master branch is used -git -C $repo checkout $rev2 -b dev -echo dev > $repo/hello +git -C "$repo" checkout "$rev2" -b dev +echo dev > "$repo"/hello # File URI uses dirty tree unless specified otherwise path2=$(nix eval --impure --raw --expr "(builtins.fetchGit file://$repo).outPath") -[ $(cat $path2/hello) = dev ] +[ "$(cat "$path2"/hello)" = dev ] # Using local path with branch other than 'master' should work when clean or dirty path3=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") @@ -181,53 +181,53 @@ path3=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") [[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).shortRev") = 0000000 ]] # Making a dirty tree clean again and fetching it should # record correct revision information. See: #4140 -echo world > $repo/hello -[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = $rev2 ]] +echo world > "$repo"/hello +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit $repo).rev") = "$rev2" ]] # Committing shouldn't change store path, or switch to using 'master' -echo dev > $repo/hello -git -C $repo commit -m 'Bla5' -a +echo dev > "$repo"/hello +git -C "$repo" commit -m 'Bla5' -a path4=$(nix eval --impure --raw --expr "(builtins.fetchGit $repo).outPath") -[[ $(cat $path4/hello) = dev ]] -[[ $path3 = $path4 ]] +[[ $(cat "$path4"/hello) = dev ]] +[[ $path3 = "$path4" ]] # Using remote path with branch other than 'master' should fetch the HEAD revision. # (--tarball-ttl 0 to prevent using the cached repo above) export _NIX_FORCE_HTTP=1 path4=$(nix eval --tarball-ttl 0 --impure --raw --expr "(builtins.fetchGit $repo).outPath") -[[ $(cat $path4/hello) = dev ]] -[[ $path3 = $path4 ]] +[[ $(cat "$path4"/hello) = dev ]] +[[ $path3 = "$path4" ]] unset _NIX_FORCE_HTTP # Confirm same as 'dev' branch path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") -[[ $path3 = $path5 ]] +[[ $path3 = "$path5" ]] # Nuke the cache -rm -rf $TEST_HOME/.cache/nix +rm -rf "$TEST_HOME"/.cache/nix # Try again. This should work. path5=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = $repo; ref = \"dev\"; }).outPath") -[[ $path3 = $path5 ]] +[[ $path3 = "$path5" ]] # Fetching from a repo with only a specific revision and no branches should # not fall back to copying files and record correct revision information. See: #5302 -mkdir $TEST_ROOT/minimal -git -C $TEST_ROOT/minimal init -git -C $TEST_ROOT/minimal fetch $repo $rev2 -git -C $TEST_ROOT/minimal checkout $rev2 -[[ $(nix eval --impure --raw --expr "(builtins.fetchGit { url = $TEST_ROOT/minimal; }).rev") = $rev2 ]] +mkdir "$TEST_ROOT"/minimal +git -C "$TEST_ROOT"/minimal init +git -C "$TEST_ROOT"/minimal fetch "$repo" "$rev2" +git -C "$TEST_ROOT"/minimal checkout "$rev2" +[[ $(nix eval --impure --raw --expr "(builtins.fetchGit { url = $TEST_ROOT/minimal; }).rev") = "$rev2" ]] # Explicit ref = "HEAD" should work, and produce the same outPath as without ref path7=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).outPath") path8=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; }).outPath") -[[ $path7 = $path8 ]] +[[ $path7 = "$path8" ]] # ref = "HEAD" should fetch the HEAD revision -rev4=$(git -C $repo rev-parse HEAD) +rev4=$(git -C "$repo" rev-parse HEAD) rev4_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; }).rev") -[[ $rev4 = $rev4_nix ]] +[[ $rev4 = "$rev4_nix" ]] # The name argument should be handled path9=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"HEAD\"; name = \"foo\"; }).outPath") @@ -236,33 +236,36 @@ path9=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$rep # Specifying a ref without a rev shouldn't pick a cached rev for a different ref export _NIX_FORCE_HTTP=1 rev_tag1_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/tags/tag1\"; }).rev") -rev_tag1=$(git -C $repo rev-parse refs/tags/tag1^{commit}) -[[ $rev_tag1_nix = $rev_tag1 ]] +# shellcheck disable=SC1083 +rev_tag1=$(git -C "$repo" rev-parse refs/tags/tag1^{commit}) +[[ $rev_tag1_nix = "$rev_tag1" ]] rev_tag2_nix=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repo\"; ref = \"refs/tags/tag2\"; }).rev") -rev_tag2=$(git -C $repo rev-parse refs/tags/tag2^{commit}) -[[ $rev_tag2_nix = $rev_tag2 ]] +# shellcheck disable=SC1083 +rev_tag2=$(git -C "$repo" rev-parse refs/tags/tag2^{commit}) +[[ $rev_tag2_nix = "$rev_tag2" ]] unset _NIX_FORCE_HTTP # Ensure .gitattributes is respected -touch $repo/not-exported-file -touch $repo/exported-wonky -echo "/not-exported-file export-ignore" >> $repo/.gitattributes -echo "/exported-wonky export-ignore=wonk" >> $repo/.gitattributes -git -C $repo add not-exported-file exported-wonky .gitattributes -git -C $repo commit -m 'Bla6' -rev5=$(git -C $repo rev-parse HEAD) +touch "$repo"/not-exported-file +touch "$repo"/exported-wonky +echo "/not-exported-file export-ignore" >> "$repo"/.gitattributes +echo "/exported-wonky export-ignore=wonk" >> "$repo"/.gitattributes +git -C "$repo" add not-exported-file exported-wonky .gitattributes +git -C "$repo" commit -m 'Bla6' +rev5=$(git -C "$repo" rev-parse HEAD) path12=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$repo; rev = \"$rev5\"; }).outPath") [[ ! -e $path12/not-exported-file ]] [[ -e $path12/exported-wonky ]] # should fail if there is no repo -rm -rf $repo/.git -rm -rf $TEST_HOME/.cache/nix +rm -rf "$repo"/.git +rm -rf "$TEST_HOME"/.cache/nix (! nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") # should succeed for a repo without commits -git init $repo -git -C $repo add hello # need to add at least one file to cause the root of the repo to be visible +git init "$repo" +git -C "$repo" add hello # need to add at least one file to cause the root of the repo to be visible +# shellcheck disable=SC2034 path10=$(nix eval --impure --raw --expr "(builtins.fetchGit \"file://$repo\").outPath") # should succeed for a path with a space @@ -277,6 +280,7 @@ touch "$repo/.gitignore" git -C "$repo" add hello .gitignore git -C "$repo" commit -m 'Bla1' cd "$repo" +# shellcheck disable=SC2034 path11=$(nix eval --impure --raw --expr "(builtins.fetchGit ./.).outPath") # Test a workdir with no commits. diff --git a/tests/functional/fetchGitRefs.sh b/tests/functional/fetchGitRefs.sh index 258a6552592..288b26591ad 100755 --- a/tests/functional/fetchGitRefs.sh +++ b/tests/functional/fetchGitRefs.sh @@ -38,16 +38,16 @@ path=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = \ # 10. They cannot contain a \. valid_ref() { - { set +x; printf >&2 '\n>>>>>>>>>> valid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; } + { set +x; printf >&2 '\n>>>>>>>>>> valid_ref %s\b <<<<<<<<<<\n' "$(printf %s "$1" | sed -n -e l)"; set -x; } git check-ref-format --branch "$1" >/dev/null git -C "$repo" branch "$1" master >/dev/null path1=$(nix eval --raw --impure --expr "(builtins.fetchGit { url = $repo; ref = ''$1''; }).outPath") - [[ $path1 = $path ]] + [[ $path1 = "$path" ]] git -C "$repo" branch -D "$1" >/dev/null } invalid_ref() { - { set +x; printf >&2 '\n>>>>>>>>>> invalid_ref %s\b <<<<<<<<<<\n' $(printf %s "$1" | sed -n -e l); set -x; } + { set +x; printf >&2 '\n>>>>>>>>>> invalid_ref %s\b <<<<<<<<<<\n' "$(printf %s "$1" | sed -n -e l)"; set -x; } # special case for a sole @: # --branch @ will try to interpret @ as a branch reference and not fail. Thus we need --allow-onelevel if [ "$1" = "@" ]; then @@ -68,6 +68,7 @@ valid_ref 'heads/foo@bar' valid_ref "$(printf 'heads/fu\303\237')" valid_ref 'foo-bar-baz' valid_ref 'branch#' +# shellcheck disable=SC2016 valid_ref '$1' valid_ref 'foo.locke' diff --git a/tests/functional/fetchGitSubmodules.sh b/tests/functional/fetchGitSubmodules.sh index cd3b51674cf..2a25245be75 100755 --- a/tests/functional/fetchGitSubmodules.sh +++ b/tests/functional/fetchGitSubmodules.sh @@ -11,7 +11,7 @@ clearStoreIfPossible rootRepo=$TEST_ROOT/gitSubmodulesRoot subRepo=$TEST_ROOT/gitSubmodulesSub -rm -rf ${rootRepo} ${subRepo} $TEST_HOME/.cache/nix +rm -rf "${rootRepo}" "${subRepo}" "$TEST_HOME"/.cache/nix # Submodules can't be fetched locally by default, which can cause # information leakage vulnerabilities, but for these tests our @@ -23,35 +23,35 @@ export XDG_CONFIG_HOME=$TEST_HOME/.config git config --global protocol.file.allow always initGitRepo() { - git init $1 - git -C $1 config user.email "foobar@example.com" - git -C $1 config user.name "Foobar" + git init "$1" + git -C "$1" config user.email "foobar@example.com" + git -C "$1" config user.name "Foobar" } addGitContent() { - echo "lorem ipsum" > $1/content - git -C $1 add content - git -C $1 commit -m "Initial commit" + echo "lorem ipsum" > "$1"/content + git -C "$1" add content + git -C "$1" commit -m "Initial commit" } -initGitRepo $subRepo -addGitContent $subRepo +initGitRepo "$subRepo" +addGitContent "$subRepo" -initGitRepo $rootRepo +initGitRepo "$rootRepo" -git -C $rootRepo submodule init -git -C $rootRepo submodule add $subRepo sub -git -C $rootRepo add sub -git -C $rootRepo commit -m "Add submodule" +git -C "$rootRepo" submodule init +git -C "$rootRepo" submodule add "$subRepo" sub +git -C "$rootRepo" add sub +git -C "$rootRepo" commit -m "Add submodule" -rev=$(git -C $rootRepo rev-parse HEAD) +rev=$(git -C "$rootRepo" rev-parse HEAD) r1=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; }).outPath") r2=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = false; }).outPath") r3=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }).outPath") -[[ $r1 == $r2 ]] -[[ $r2 != $r3 ]] +[[ $r1 == "$r2" ]] +[[ $r2 != "$r3" ]] r4=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; }).outPath") r5=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = false; }).outPath") @@ -59,11 +59,11 @@ r6=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \ r7=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") r8=$(nix eval --raw --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; submodules = true; }).outPath") -[[ $r1 == $r4 ]] -[[ $r4 == $r5 ]] -[[ $r3 == $r6 ]] -[[ $r6 == $r7 ]] -[[ $r7 == $r8 ]] +[[ $r1 == "$r4" ]] +[[ $r4 == "$r5" ]] +[[ $r3 == "$r6" ]] +[[ $r6 == "$r7" ]] +[[ $r7 == "$r8" ]] have_submodules=$(nix eval --expr "(builtins.fetchGit { url = $rootRepo; rev = \"$rev\"; }).submodules") [[ $have_submodules == false ]] @@ -80,13 +80,13 @@ pathWithSubmodulesAgain=$(nix eval --raw --expr "(builtins.fetchGit { url = file pathWithSubmodulesAgainWithRef=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; ref = \"master\"; rev = \"$rev\"; submodules = true; }).outPath") # The resulting store path cannot be the same. -[[ $pathWithoutSubmodules != $pathWithSubmodules ]] +[[ $pathWithoutSubmodules != "$pathWithSubmodules" ]] # Checking out the same repo with submodules returns in the same store path. -[[ $pathWithSubmodules == $pathWithSubmodulesAgain ]] +[[ $pathWithSubmodules == "$pathWithSubmodulesAgain" ]] # Checking out the same repo with submodules returns in the same store path. -[[ $pathWithSubmodulesAgain == $pathWithSubmodulesAgainWithRef ]] +[[ $pathWithSubmodulesAgain == "$pathWithSubmodulesAgainWithRef" ]] # The submodules flag is actually honored. [[ ! -e $pathWithoutSubmodules/sub/content ]] @@ -98,14 +98,14 @@ pathWithSubmodulesAgainWithRef=$(nix eval --raw --expr "(builtins.fetchGit { url test "$(find "$pathWithSubmodules" -name .git)" = "" # Git repos without submodules can be fetched with submodules = true. -subRev=$(git -C $subRepo rev-parse HEAD) +subRev=$(git -C "$subRepo" rev-parse HEAD) noSubmoduleRepoBaseline=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; }).outPath") noSubmoduleRepo=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$subRepo; rev = \"$subRev\"; submodules = true; }).outPath") -[[ $noSubmoduleRepoBaseline == $noSubmoduleRepo ]] +[[ $noSubmoduleRepoBaseline == "$noSubmoduleRepo" ]] # Test .gitmodules with entries that refer to non-existent objects or objects that are not submodules. -cat >> $rootRepo/.gitmodules <> "$rootRepo"/.gitmodules <> $rootRepo/.gitmodules < $rootRepo/file -git -C $rootRepo add file -git -C $rootRepo commit -a -m "Add bad submodules" +echo foo > "$rootRepo"/file +git -C "$rootRepo" add file +git -C "$rootRepo" commit -a -m "Add bad submodules" -rev=$(git -C $rootRepo rev-parse HEAD) +rev=$(git -C "$rootRepo" rev-parse HEAD) r=$(nix eval --raw --expr "builtins.fetchGit { url = file://$rootRepo; rev = \"$rev\"; submodules = true; }") @@ -126,44 +126,44 @@ r=$(nix eval --raw --expr "builtins.fetchGit { url = file://$rootRepo; rev = \"$ [[ ! -e $r/missing ]] # Test relative submodule URLs. -rm $TEST_HOME/.cache/nix/fetcher-cache* -rm -rf $rootRepo/.git $rootRepo/.gitmodules $rootRepo/sub -initGitRepo $rootRepo -git -C $rootRepo submodule add ../gitSubmodulesSub sub -git -C $rootRepo commit -m "Add submodule" -rev2=$(git -C $rootRepo rev-parse HEAD) +rm "$TEST_HOME"/.cache/nix/fetcher-cache* +rm -rf "$rootRepo"/.git "$rootRepo"/.gitmodules "$rootRepo"/sub +initGitRepo "$rootRepo" +git -C "$rootRepo" submodule add ../gitSubmodulesSub sub +git -C "$rootRepo" commit -m "Add submodule" +rev2=$(git -C "$rootRepo" rev-parse HEAD) pathWithRelative=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$rootRepo; rev = \"$rev2\"; submodules = true; }).outPath") -diff -r -x .gitmodules $pathWithSubmodules $pathWithRelative +diff -r -x .gitmodules "$pathWithSubmodules" "$pathWithRelative" # Test clones that have an upstream with relative submodule URLs. -rm $TEST_HOME/.cache/nix/fetcher-cache* +rm "$TEST_HOME"/.cache/nix/fetcher-cache* cloneRepo=$TEST_ROOT/a/b/gitSubmodulesClone # NB /a/b to make the relative path not work relative to $cloneRepo -git clone $rootRepo $cloneRepo +git clone "$rootRepo" "$cloneRepo" pathIndirect=$(nix eval --raw --expr "(builtins.fetchGit { url = file://$cloneRepo; rev = \"$rev2\"; submodules = true; }).outPath") -[[ $pathIndirect = $pathWithRelative ]] +[[ $pathIndirect = "$pathWithRelative" ]] # Test submodule export-ignore interaction -git -C $rootRepo/sub config user.email "foobar@example.com" -git -C $rootRepo/sub config user.name "Foobar" +git -C "$rootRepo"/sub config user.email "foobar@example.com" +git -C "$rootRepo"/sub config user.name "Foobar" -echo "/exclude-from-root export-ignore" >> $rootRepo/.gitattributes +echo "/exclude-from-root export-ignore" >> "$rootRepo"/.gitattributes # TBD possible semantics for submodules + exportIgnore # echo "/sub/exclude-deep export-ignore" >> $rootRepo/.gitattributes -echo nope > $rootRepo/exclude-from-root -git -C $rootRepo add .gitattributes exclude-from-root -git -C $rootRepo commit -m "Add export-ignore" +echo nope > "$rootRepo"/exclude-from-root +git -C "$rootRepo" add .gitattributes exclude-from-root +git -C "$rootRepo" commit -m "Add export-ignore" -echo "/exclude-from-sub export-ignore" >> $rootRepo/sub/.gitattributes -echo nope > $rootRepo/sub/exclude-from-sub +echo "/exclude-from-sub export-ignore" >> "$rootRepo"/sub/.gitattributes +echo nope > "$rootRepo"/sub/exclude-from-sub # TBD possible semantics for submodules + exportIgnore # echo aye > $rootRepo/sub/exclude-from-root -git -C $rootRepo/sub add .gitattributes exclude-from-sub -git -C $rootRepo/sub commit -m "Add export-ignore (sub)" +git -C "$rootRepo"/sub add .gitattributes exclude-from-sub +git -C "$rootRepo"/sub commit -m "Add export-ignore (sub)" -git -C $rootRepo add sub -git -C $rootRepo commit -m "Update submodule" +git -C "$rootRepo" add sub +git -C "$rootRepo" commit -m "Update submodule" -git -C $rootRepo status +git -C "$rootRepo" status # # TBD: not supported yet, because semantics are undecided and current implementation leaks rules from the root to submodules # # exportIgnore can be used with submodules @@ -199,39 +199,40 @@ test_submodule_nested() { local repoB=$TEST_ROOT/submodule_nested/b local repoC=$TEST_ROOT/submodule_nested/c - rm -rf $repoA $repoB $repoC $TEST_HOME/.cache/nix + rm -rf "$repoA" "$repoB" "$repoC" "$TEST_HOME"/.cache/nix - initGitRepo $repoC - touch $repoC/inside-c - git -C $repoC add inside-c - addGitContent $repoC + initGitRepo "$repoC" + touch "$repoC"/inside-c + git -C "$repoC" add inside-c + addGitContent "$repoC" - initGitRepo $repoB - git -C $repoB submodule add $repoC c - git -C $repoB add c - addGitContent $repoB + initGitRepo "$repoB" + git -C "$repoB" submodule add "$repoC" c + git -C "$repoB" add c + addGitContent "$repoB" - initGitRepo $repoA - git -C $repoA submodule add $repoB b - git -C $repoA add b - addGitContent $repoA + initGitRepo "$repoA" + git -C "$repoA" submodule add "$repoB" b + git -C "$repoA" add b + addGitContent "$repoA" # Check non-worktree fetch - local rev=$(git -C $repoA rev-parse HEAD) + local rev + rev=$(git -C "$repoA" rev-parse HEAD) out=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repoA\"; rev = \"$rev\"; submodules = true; }).outPath") - test -e $out/b/c/inside-c - test -e $out/content - test -e $out/b/content - test -e $out/b/c/content + test -e "$out"/b/c/inside-c + test -e "$out"/content + test -e "$out"/b/content + test -e "$out"/b/c/content local nonWorktree=$out # Check worktree based fetch # TODO: make it work without git submodule update - git -C $repoA submodule update --init --recursive + git -C "$repoA" submodule update --init --recursive out=$(nix eval --impure --raw --expr "(builtins.fetchGit { url = \"file://$repoA\"; submodules = true; }).outPath") - find $out - [[ $out == $nonWorktree ]] || { find $out; false; } + find "$out" + [[ $out == "$nonWorktree" ]] || { find "$out"; false; } } test_submodule_nested diff --git a/tests/functional/fetchGitVerification.sh b/tests/functional/fetchGitVerification.sh index 4012d82290e..79c78d0c9f6 100755 --- a/tests/functional/fetchGitVerification.sh +++ b/tests/functional/fetchGitVerification.sh @@ -21,29 +21,29 @@ ssh-keygen -f "$keysDir/testkey2" -t rsa -P "" -C "test key 2" key2File="$keysDir/testkey2.pub" publicKey2=$(awk '{print $2}' "$key2File") -git init $repo -git -C $repo config user.email "foobar@example.com" -git -C $repo config user.name "Foobar" -git -C $repo config gpg.format ssh +git init "$repo" +git -C "$repo" config user.email "foobar@example.com" +git -C "$repo" config user.name "Foobar" +git -C "$repo" config gpg.format ssh -echo 'hello' > $repo/text -git -C $repo add text -git -C $repo -c "user.signingkey=$key1File" commit -S -m 'initial commit' +echo 'hello' > "$repo"/text +git -C "$repo" add text +git -C "$repo" -c "user.signingkey=$key1File" commit -S -m 'initial commit' out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; keytype = \"ssh-rsa\"; publicKey = \"$publicKey2\"; }" 2>&1) || status=$? [[ $status == 1 ]] -[[ $out =~ 'No principal matched.' ]] +[[ $out == *'No principal matched.'* ]] [[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKey = \"$publicKey1\"; } + \"/text\")") = 'hello' ]] -echo 'hello world' > $repo/text +echo 'hello world' > "$repo"/text # Verification on a dirty repo should fail. out=$(nix eval --impure --raw --expr "builtins.fetchGit { url = \"file://$repo\"; keytype = \"ssh-rsa\"; publicKey = \"$publicKey2\"; }" 2>&1) || status=$? [[ $status == 1 ]] [[ $out =~ 'dirty' ]] -git -C $repo add text -git -C $repo -c "user.signingkey=$key2File" commit -S -m 'second commit' +git -C "$repo" add text +git -C "$repo" -c "user.signingkey=$key2File" commit -S -m 'second commit' [[ $(nix eval --impure --raw --expr "builtins.readFile (builtins.fetchGit { url = \"file://$repo\"; publicKeys = [{key = \"$publicKey1\";} {type = \"ssh-rsa\"; key = \"$publicKey2\";}]; } + \"/text\")") = 'hello world' ]] @@ -80,5 +80,6 @@ cat > "$flakeDir/flake.nix" <&1) || status=$? + [[ $status == 1 ]] -[[ $out =~ 'No principal matched.' ]] +[[ $out == *'No principal matched.'* ]] diff --git a/tests/functional/fetchMercurial.sh b/tests/functional/fetchMercurial.sh index 6de19286587..6293fb76ac2 100755 --- a/tests/functional/fetchMercurial.sh +++ b/tests/functional/fetchMercurial.sh @@ -12,34 +12,35 @@ clearStore # See https://github.com/NixOS/nix/issues/6195 repo=$TEST_ROOT/./hg -rm -rf $repo ${repo}-tmp $TEST_HOME/.cache/nix - -hg init $repo -echo '[ui]' >> $repo/.hg/hgrc -echo 'username = Foobar ' >> $repo/.hg/hgrc - -# Set ui.tweakdefaults to ensure HGPLAIN is being set. -echo 'tweakdefaults = True' >> $repo/.hg/hgrc - -echo utrecht > $repo/hello -touch $repo/.hgignore -hg add --cwd $repo hello .hgignore -hg commit --cwd $repo -m 'Bla1' -rev1=$(hg log --cwd $repo -r tip --template '{node}') - -echo world > $repo/hello -hg commit --cwd $repo -m 'Bla2' -rev2=$(hg log --cwd $repo -r tip --template '{node}') +rm -rf "$repo" "${repo}"-tmp "$TEST_HOME"/.cache/nix + +hg init "$repo" +{ + echo '[ui]' + echo 'username = Foobar ' + # Set ui.tweakdefaults to ensure HGPLAIN is being set. + echo 'tweakdefaults = True' +} >> "$repo"/.hg/hgrc + +echo utrecht > "$repo"/hello +touch "$repo"/.hgignore +hg add --cwd "$repo" hello .hgignore +hg commit --cwd "$repo" -m 'Bla1' +rev1=$(hg log --cwd "$repo" -r tip --template '{node}') + +echo world > "$repo"/hello +hg commit --cwd "$repo" -m 'Bla2' +rev2=$(hg log --cwd "$repo" -r tip --template '{node}') # Fetch an unclean branch. -echo unclean > $repo/hello +echo unclean > "$repo"/hello path=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).outPath") -[[ $(cat $path/hello) = unclean ]] -hg revert --cwd $repo --all +[[ $(cat "$path"/hello) = unclean ]] +hg revert --cwd "$repo" --all # Fetch the default branch. path=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).outPath") -[[ $(cat $path/hello) = world ]] +[[ $(cat "$path"/hello) = world ]] # In pure eval mode, fetchGit without a revision should fail. [[ $(nix eval --impure --raw --expr "(builtins.readFile (fetchMercurial file://$repo + \"/hello\"))") = world ]] @@ -47,64 +48,64 @@ path=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).ou # Fetch using an explicit revision hash. path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") -[[ $path = $path2 ]] +[[ $path = "$path2" ]] # In pure eval mode, fetchGit with a revision should succeed. [[ $(nix eval --raw --expr "builtins.readFile (fetchMercurial { url = file://$repo; rev = \"$rev2\"; } + \"/hello\")") = world ]] # Fetch again. This should be cached. -mv $repo ${repo}-tmp +mv "$repo" "${repo}"-tmp path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).outPath") -[[ $path = $path2 ]] +[[ $path = "$path2" ]] [[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).branch") = default ]] [[ $(nix eval --impure --expr "(builtins.fetchMercurial file://$repo).revCount") = 1 ]] -[[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).rev") = $rev2 ]] +[[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial file://$repo).rev") = "$rev2" ]] # But with TTL 0, it should fail. (! nix eval --impure --refresh --expr "builtins.fetchMercurial file://$repo") # Fetching with a explicit hash should succeed. path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev2\"; }).outPath") -[[ $path = $path2 ]] +[[ $path = "$path2" ]] path2=$(nix eval --refresh --raw --expr "(builtins.fetchMercurial { url = file://$repo; rev = \"$rev1\"; }).outPath") -[[ $(cat $path2/hello) = utrecht ]] +[[ $(cat "$path2"/hello) = utrecht ]] -mv ${repo}-tmp $repo +mv "${repo}"-tmp "$repo" # Using a clean working tree should produce the same result. path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).outPath") -[[ $path = $path2 ]] +[[ $path = "$path2" ]] # Using an unclean tree should yield the tracked but uncommitted changes. -mkdir $repo/dir1 $repo/dir2 -echo foo > $repo/dir1/foo -echo bar > $repo/bar -echo bar > $repo/dir2/bar -hg add --cwd $repo dir1/foo -hg rm --cwd $repo hello +mkdir "$repo"/dir1 "$repo"/dir2 +echo foo > "$repo"/dir1/foo +echo bar > "$repo"/bar +echo bar > "$repo"/dir2/bar +hg add --cwd "$repo" dir1/foo +hg rm --cwd "$repo" hello path2=$(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).outPath") -[ ! -e $path2/hello ] -[ ! -e $path2/bar ] -[ ! -e $path2/dir2/bar ] -[ ! -e $path2/.hg ] -[[ $(cat $path2/dir1/foo) = foo ]] +[ ! -e "$path2"/hello ] +[ ! -e "$path2"/bar ] +[ ! -e "$path2"/dir2/bar ] +[ ! -e "$path2"/.hg ] +[[ $(cat "$path2"/dir1/foo) = foo ]] [[ $(nix eval --impure --raw --expr "(builtins.fetchMercurial $repo).rev") = 0000000000000000000000000000000000000000 ]] # ... unless we're using an explicit ref. path3=$(nix eval --impure --raw --expr "(builtins.fetchMercurial { url = $repo; rev = \"default\"; }).outPath") -[[ $path = $path3 ]] +[[ $path = "$path3" ]] # Committing should not affect the store path. -hg commit --cwd $repo -m 'Bla3' +hg commit --cwd "$repo" -m 'Bla3' path4=$(nix eval --impure --refresh --raw --expr "(builtins.fetchMercurial file://$repo).outPath") -[[ $path2 = $path4 ]] +[[ $path2 = "$path4" ]] -echo paris > $repo/hello +echo paris > "$repo"/hello # Passing a `name` argument should be reflected in the output path path5=$(nix eval -vvvvv --impure --refresh --raw --expr "(builtins.fetchMercurial { url = \"file://$repo\"; name = \"foo\"; } ).outPath") diff --git a/tests/functional/fetchurl.sh b/tests/functional/fetchurl.sh index c25ac321668..96d46abf468 100755 --- a/tests/functional/fetchurl.sh +++ b/tests/functional/fetchurl.sh @@ -71,7 +71,7 @@ echo "$outPath" | grepQuiet 'xyzzy' test -x "$outPath/fetchurl.sh" test -L "$outPath/symlink" -nix-store --delete "$outPath" +nix-store --delete "$outPath" --ignore-liveness # Test unpacking a compressed NAR. narxz="$TEST_ROOT/archive.nar.xz" diff --git a/tests/functional/fixed.builder1.sh b/tests/functional/fixed.builder1.sh index c41bb2b9a61..172f65e6b25 100644 --- a/tests/functional/fixed.builder1.sh +++ b/tests/functional/fixed.builder1.sh @@ -1,3 +1,5 @@ +# shellcheck shell=bash if test "$IMPURE_VAR1" != "foo"; then exit 1; fi if test "$IMPURE_VAR2" != "bar"; then exit 1; fi -echo "Hello World!" > $out +# shellcheck disable=SC2154 +echo "Hello World!" > "$out" diff --git a/tests/functional/fixed.builder2.sh b/tests/functional/fixed.builder2.sh index 31ea1579a51..9fbcf022ed9 100644 --- a/tests/functional/fixed.builder2.sh +++ b/tests/functional/fixed.builder2.sh @@ -1,6 +1,9 @@ -echo dummy: $dummy +# shellcheck shell=bash +# shellcheck disable=SC2154 +echo dummy: "$dummy" if test -n "$dummy"; then sleep 2; fi -mkdir $out -mkdir $out/bla -echo "Hello World!" > $out/foo -ln -s foo $out/bar +# shellcheck disable=SC2154 +mkdir "$out" +mkdir "$out"/bla +echo "Hello World!" > "$out"/foo +ln -s foo "$out"/bar diff --git a/tests/functional/fixed.sh b/tests/functional/fixed.sh index d98769e6435..edf6f88d4ed 100755 --- a/tests/functional/fixed.sh +++ b/tests/functional/fixed.sh @@ -6,7 +6,7 @@ TODO_NixOS clearStore -path=$(nix-store -q $(nix-instantiate fixed.nix -A good.0)) +path=$(nix-store -q "$(nix-instantiate fixed.nix -A good.0)") echo 'testing bad...' nix-build fixed.nix -A bad --no-out-link && fail "should fail" @@ -14,7 +14,7 @@ nix-build fixed.nix -A bad --no-out-link && fail "should fail" # Building with the bad hash should produce the "good" output path as # a side-effect. [[ -e $path ]] -nix path-info --json $path | grep fixed:md5:2qk15sxzzjlnpjk9brn7j8ppcd +nix path-info --json "$path" | grep fixed:md5:2qk15sxzzjlnpjk9brn7j8ppcd echo 'testing good...' nix-build fixed.nix -A good --no-out-link @@ -37,7 +37,7 @@ fi # While we're at it, check attribute selection a bit more. echo 'testing attribute selection...' -test $(nix-instantiate fixed.nix -A good.1 | wc -l) = 1 +test "$(nix-instantiate fixed.nix -A good.1 | wc -l)" = 1 # Test parallel builds of derivations that produce the same output. # Only one should run at the same time. @@ -51,16 +51,16 @@ echo 'testing sameAsAdd...' out=$(nix-build fixed.nix -A sameAsAdd --no-out-link) # This is what fixed.builder2 produces... -rm -rf $TEST_ROOT/fixed -mkdir $TEST_ROOT/fixed -mkdir $TEST_ROOT/fixed/bla -echo "Hello World!" > $TEST_ROOT/fixed/foo -ln -s foo $TEST_ROOT/fixed/bar +rm -rf "$TEST_ROOT"/fixed +mkdir "$TEST_ROOT"/fixed +mkdir "$TEST_ROOT"/fixed/bla +echo "Hello World!" > "$TEST_ROOT"/fixed/foo +ln -s foo "$TEST_ROOT"/fixed/bar -out2=$(nix-store --add $TEST_ROOT/fixed) +out2=$(nix-store --add "$TEST_ROOT"/fixed) [ "$out" = "$out2" ] -out3=$(nix-store --add-fixed --recursive sha256 $TEST_ROOT/fixed) +out3=$(nix-store --add-fixed --recursive sha256 "$TEST_ROOT"/fixed) [ "$out" = "$out3" ] out4=$(nix-store --print-fixed-path --recursive sha256 "1ixr6yd3297ciyp9im522dfxpqbkhcw0pylkb2aab915278fqaik" fixed) diff --git a/tests/functional/flakes/absolute-paths.sh b/tests/functional/flakes/absolute-paths.sh index a355a7a1c07..6565857cb85 100755 --- a/tests/functional/flakes/absolute-paths.sh +++ b/tests/functional/flakes/absolute-paths.sh @@ -7,13 +7,13 @@ requireGit flake1Dir=$TEST_ROOT/flake1 flake2Dir=$TEST_ROOT/flake2 -createGitRepo $flake1Dir -cat > $flake1Dir/flake.nix < "$flake1Dir"/flake.nix < "$lazy/who" +git -C "$lazy" add who +git -C "$lazy" commit -a -m foo + +repo="$TEST_ROOT/repo" + +createGitRepo "$repo" + +cat > "$repo/flake.nix" < "$lazy/who" +git -C "$lazy" commit -a -m foo + +nix flake update --flake "$repo" + +clearStore + +nix build --out-link "$TEST_ROOT/result" -L "$repo" +[[ $(cat "$TEST_ROOT/result") = utrecht ]] + +rm -rf "$lazy" + +clearStore + +expectStderr 1 nix build --out-link "$TEST_ROOT/result" -L "$repo" | grepQuiet "Cannot build.*source.drv" diff --git a/tests/functional/flakes/check.sh b/tests/functional/flakes/check.sh index 27e73444ae0..55cd3805ff2 100755 --- a/tests/functional/flakes/check.sh +++ b/tests/functional/flakes/check.sh @@ -3,9 +3,9 @@ source common.sh flakeDir=$TEST_ROOT/flake3 -mkdir -p $flakeDir +mkdir -p "$flakeDir" -cat > $flakeDir/flake.nix < "$flakeDir"/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < "$flakeDir"/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < "$flakeDir"/flake.nix < $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +# shellcheck disable=SC2015 +checkRes=$(nix flake check "$flakeDir" 2>&1 && fail "nix flake check --all-systems should have failed" || true) echo "$checkRes" | grepQuiet "error: overlay is not a function, but a set instead" -cat > $flakeDir/flake.nix < "$flakeDir"/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < "$flakeDir"/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < "$flakeDir"/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < "$flakeDir"/flake.nix < $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +# shellcheck disable=SC2015 +checkRes=$(nix flake check --all-systems --keep-going "$flakeDir" 2>&1 && fail "nix flake check --all-systems should have failed" || true) echo "$checkRes" | grepQuiet "packages.system-1.default" echo "$checkRes" | grepQuiet "packages.system-2.default" -cat > $flakeDir/flake.nix < "$flakeDir"/flake.nix < $flakeDir/flake.nix < $flakeDir/flake.nix < "$flakeDir"/flake.nix < $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +# shellcheck disable=SC2015 +checkRes=$(nix flake check --all-systems "$flakeDir" 2>&1 && fail "nix flake check --all-systems should have failed" || true) echo "$checkRes" | grepQuiet "unknown-attr" -cat > $flakeDir/flake.nix < "$flakeDir"/flake.nix < $flakeDir/flake.nix <&1 && fail "nix flake check --all-systems should have failed" || true) +# shellcheck disable=SC2015 +checkRes=$(nix flake check --all-systems "$flakeDir" 2>&1 && fail "nix flake check --all-systems should have failed" || true) echo "$checkRes" | grepQuiet "formatter.system-1" + +# Test whether `nix flake check` builds checks. +cat > "$flakeDir"/flake.nix < "$flakeDir"/flake.nix < "$flakeDir"/flake.nix <&1 && fail "nix flake check should have failed" || true) +echo "$checkRes" | grepQuiet -E "builder( for .*)? failed with exit code 1" diff --git a/tests/functional/flakes/common.sh b/tests/functional/flakes/common.sh index 422cab96cc2..77bc030605f 100644 --- a/tests/functional/flakes/common.sh +++ b/tests/functional/flakes/common.sh @@ -2,6 +2,8 @@ source ../common.sh +export _NIX_TEST_BARF_ON_UNCACHEABLE=1 + # shellcheck disable=SC2034 # this variable is used by tests that source this file registry=$TEST_ROOT/registry.json diff --git a/tests/functional/flakes/config.sh b/tests/functional/flakes/config.sh index ab2d9f47cf0..87714b5db61 100755 --- a/tests/functional/flakes/config.sh +++ b/tests/functional/flakes/config.sh @@ -2,9 +2,9 @@ source common.sh -cp ../simple.nix ../simple.builder.sh "${config_nix}" $TEST_HOME +cp ../simple.nix ../simple.builder.sh "${config_nix}" "$TEST_HOME" -cd $TEST_HOME +cd "$TEST_HOME" rm -f post-hook-ran cat < echoing-post-hook.sh @@ -37,6 +37,7 @@ if type -p script >/dev/null && script -q -c true /dev/null; then else echo "script is not available or not GNU-like, so we skip testing with an added tty" fi +# shellcheck disable=SC2235 (! [[ -f post-hook-ran ]]) TODO_NixOS clearStore diff --git a/tests/functional/flakes/eval-cache.sh b/tests/functional/flakes/eval-cache.sh index 75a2c8cacbf..f4e48d8bb77 100755 --- a/tests/functional/flakes/eval-cache.sh +++ b/tests/functional/flakes/eval-cache.sh @@ -35,11 +35,11 @@ EOF git -C "$flake1Dir" add flake.nix git -C "$flake1Dir" commit -m "Init" -expect 1 nix build "$flake1Dir#foo.bar" 2>&1 | grepQuiet 'error: breaks' -expect 1 nix build "$flake1Dir#foo.bar" 2>&1 | grepQuiet 'error: breaks' +expect 1 nix build --no-link "$flake1Dir#foo.bar" 2>&1 | grepQuiet 'error: breaks' +expect 1 nix build --no-link "$flake1Dir#foo.bar" 2>&1 | grepQuiet 'error: breaks' # Stack overflow error must not be cached -expect 1 nix build --max-call-depth 50 "$flake1Dir#stack-depth" 2>&1 \ +expect 1 nix build --no-link --max-call-depth 50 "$flake1Dir#stack-depth" 2>&1 \ | grepQuiet 'error: stack overflow; max-call-depth exceeded' # If the SO is cached, the following invocation will produce a cached failure; we expect it to succeed nix build --no-link "$flake1Dir#stack-depth" @@ -48,3 +48,11 @@ nix build --no-link "$flake1Dir#stack-depth" expect 1 nix build "$flake1Dir#ifd" --option allow-import-from-derivation false 2>&1 \ | grepQuiet 'error: cannot build .* during evaluation because the option '\''allow-import-from-derivation'\'' is disabled' nix build --no-link "$flake1Dir#ifd" + +# Test that a store derivation is recreated when it has been deleted +# but the corresponding attribute is still cached. +if ! isTestOnNixOS; then + nix build --no-link "$flake1Dir#drv" + clearStore + nix build --no-link "$flake1Dir#drv" +fi diff --git a/tests/functional/flakes/flake-in-submodule.sh b/tests/functional/flakes/flake-in-submodule.sh index fe5acf26dec..a7d86698de8 100755 --- a/tests/functional/flakes/flake-in-submodule.sh +++ b/tests/functional/flakes/flake-in-submodule.sh @@ -62,8 +62,8 @@ flakeref=git+file://$rootRepo\?submodules=1\&dir=submodule # Check that dirtying a submodule makes the entire thing dirty. [[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) != null ]] echo '"foo"' > "$rootRepo"/submodule/sub.nix -[[ $(nix eval --json "$flakeref#sub" ) = '"foo"' ]] -[[ $(nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] +[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval --json "$flakeref#sub" ) = '"foo"' ]] +[[ $(_NIX_TEST_BARF_ON_UNCACHEABLE='' nix flake metadata --json "$flakeref" | jq -r .locked.rev) = null ]] # Test that `nix flake metadata` parses `submodule` correctly. cat > "$rootRepo"/flake.nix < "$flake1Dir/foo" -git -C "$flake1Dir" add $flake1Dir/foo +git -C "$flake1Dir" add "$flake1Dir"/foo [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "$hash1-dirty" ]] +[[ $(_NIX_TEST_FAIL_ON_LARGE_PATH=1 nix flake metadata flake1 --json --refresh --warn-large-path-threshold 1 --lazy-trees | jq -r .dirtyRevision) == "$hash1-dirty" ]] [[ "$(nix flake metadata flake1 --json | jq -r .fingerprint)" != null ]] echo -n '# foo' >> "$flake1Dir/flake.nix" flake1OriginalCommit=$(git -C "$flake1Dir" rev-parse HEAD) git -C "$flake1Dir" commit -a -m 'Foo' +# shellcheck disable=SC2034 flake1NewCommit=$(git -C "$flake1Dir" rev-parse HEAD) hash2=$(nix flake metadata flake1 --json --refresh | jq -r .revision) [[ $(nix flake metadata flake1 --json --refresh | jq -r .dirtyRevision) == "null" ]] -[[ $hash1 != $hash2 ]] +[[ $hash1 != "$hash2" ]] # Test 'nix build' on a flake. nix build -o "$TEST_ROOT/result" flake1#foo @@ -109,6 +113,11 @@ nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir#default" nix build -o "$TEST_ROOT/result" "$flake1Dir?ref=HEAD#default" nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" +# Check that the fetcher cache works. +if [[ $(nix config show lazy-trees) = false ]]; then + nix build -o "$TEST_ROOT/result" "git+file://$flake1Dir?ref=HEAD#default" -vvvvv 2>&1 | grepQuiet "source path.*cache hit" +fi + # Check that relative paths are allowed for git flakes. # This may change in the future once git submodule support is refined. # See: https://discourse.nixos.org/t/57783 and #9708. @@ -160,7 +169,12 @@ expect 1 nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --no-update-lock-file nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" --commit-lock-file [[ -e "$flake2Dir/flake.lock" ]] [[ -z $(git -C "$flake2Dir" diff main || echo failed) ]] -[[ $(jq --indent 0 --compact-output . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'.*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +[[ $(jq --indent 0 --compact-output . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"narHash":"sha256-'.*'","ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +if [[ $(nix config show lazy-trees) = true ]]; then + # Test that `lazy-locks` causes NAR hashes to be omitted from the lock file. + nix flake update --flake "$flake2Dir" --commit-lock-file --lazy-locks + [[ $(jq --indent 0 --compact-output . < "$flake2Dir/flake.lock") =~ ^'{"nodes":{"flake1":{"locked":{"lastModified":'[0-9]*',"ref":"refs/heads/master","rev":"'.*'","revCount":2,"type":"git","url":"file:///'.*'"},"original":{"id":"flake1","type":"indirect"}},"root":{"inputs":{"flake1":"flake1"}}},"root":"root","version":7}'$ ]] +fi # Rerunning the build should not change the lockfile. nix build -o "$TEST_ROOT/result" "$flake2Dir#bar" @@ -204,8 +218,8 @@ git -C "$flake3Dir" add flake.nix git -C "$flake3Dir" commit -m 'Update flake.nix' # Check whether `nix build` works with an incomplete lockfile -nix build -o $TEST_ROOT/result "$flake3Dir#sth sth" -nix build -o $TEST_ROOT/result "$flake3Dir#sth%20sth" +nix build -o "$TEST_ROOT"/result "$flake3Dir#sth sth" +nix build -o "$TEST_ROOT"/result "$flake3Dir#sth%20sth" # Check whether it saved the lockfile [[ -n $(git -C "$flake3Dir" diff master) ]] @@ -249,7 +263,7 @@ nix flake lock "$flake3Dir" [[ -z $(git -C "$flake3Dir" diff master || echo failed) ]] nix flake update --flake "$flake3Dir" --override-flake flake2 nixpkgs -[[ ! -z $(git -C "$flake3Dir" diff master || echo failed) ]] +[[ -n $(git -C "$flake3Dir" diff master || echo failed) ]] # Testing the nix CLI nix registry add flake1 flake3 @@ -262,7 +276,7 @@ nix registry remove flake1 [[ $(nix registry list | wc -l) == 4 ]] # Test 'nix registry list' with a disabled global registry. -nix registry add user-flake1 git+file://$flake1Dir +nix registry add user-flake1 git+file://"$flake1Dir" nix registry add user-flake2 "git+file://$percentEncodedFlake2Dir" [[ $(nix --flake-registry "" registry list | wc -l) == 2 ]] nix --flake-registry "" registry list | grepQuietInverse '^global' # nothing in global registry @@ -273,9 +287,9 @@ nix registry remove user-flake2 [[ $(nix registry list | wc -l) == 4 ]] # Test 'nix flake clone'. -rm -rf $TEST_ROOT/flake1-v2 -nix flake clone flake1 --dest $TEST_ROOT/flake1-v2 -[ -e $TEST_ROOT/flake1-v2/flake.nix ] +rm -rf "$TEST_ROOT"/flake1-v2 +nix flake clone flake1 --dest "$TEST_ROOT"/flake1-v2 +[ -e "$TEST_ROOT"/flake1-v2/flake.nix ] # Test 'follows' inputs. cat > "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < $badFlakeDir/flake.nix -nix store delete $(nix store add-path $badFlakeDir) +rm -rf "$badFlakeDir" +mkdir "$badFlakeDir" +echo INVALID > "$badFlakeDir"/flake.nix +nix store delete --ignore-liveness "$(nix store add-path "$badFlakeDir")" -[[ $(nix path-info $(nix store add-path $flake1Dir)) =~ flake1 ]] -[[ $(nix path-info path:$(nix store add-path $flake1Dir)) =~ simple ]] +[[ $(nix path-info "$(nix store add-path "$flake1Dir")") =~ flake1 ]] +[[ $(nix path-info path:"$(nix store add-path "$flake1Dir")") =~ simple ]] # Test fetching flakerefs in the legacy CLI. [[ $(nix-instantiate --eval flake:flake3 -A x) = 123 ]] @@ -423,15 +444,15 @@ nix store delete $(nix store add-path $badFlakeDir) [[ $(NIX_PATH=flake3=flake:flake3 nix-instantiate --eval '' -A x) = 123 ]] # Test alternate lockfile paths. -nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2.lock -cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one +nix flake lock "$flake2Dir" --output-lock-file "$TEST_ROOT"/flake2.lock +cmp "$flake2Dir/flake.lock" "$TEST_ROOT"/flake2.lock >/dev/null # lockfiles should be identical, since we're referencing flake2's original one -nix flake lock "$flake2Dir" --output-lock-file $TEST_ROOT/flake2-overridden.lock --override-input flake1 git+file://$flake1Dir?rev=$flake1OriginalCommit -expectStderr 1 cmp "$flake2Dir/flake.lock" $TEST_ROOT/flake2-overridden.lock -nix flake metadata "$flake2Dir" --reference-lock-file $TEST_ROOT/flake2-overridden.lock | grepQuiet $flake1OriginalCommit +nix flake lock "$flake2Dir" --output-lock-file "$TEST_ROOT"/flake2-overridden.lock --override-input flake1 git+file://"$flake1Dir"?rev="$flake1OriginalCommit" +expectStderr 1 cmp "$flake2Dir/flake.lock" "$TEST_ROOT"/flake2-overridden.lock +nix flake metadata "$flake2Dir" --reference-lock-file "$TEST_ROOT"/flake2-overridden.lock | grepQuiet "$flake1OriginalCommit" # reference-lock-file can only be used if allow-dirty is set. -expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file $TEST_ROOT/flake2-overridden.lock +expectStderr 1 nix flake metadata "$flake2Dir" --no-allow-dirty --reference-lock-file "$TEST_ROOT"/flake2-overridden.lock # After changing an input (flake2 from newFlake2Rev to prevFlake2Rev), we should have the transitive inputs locked by revision $prevFlake2Rev of flake2. prevFlake1Rev=$(nix flake metadata --json "$flake1Dir" | jq -r .revision) @@ -458,7 +479,7 @@ git -C "$flake3Dir" commit flake.nix -m 'bla' rm "$flake3Dir/flake.lock" nix flake lock "$flake3Dir" -[[ "$(nix flake metadata --json "$flake3Dir" | jq -r .locks.nodes.flake1.locked.rev)" = $newFlake1Rev ]] +[[ "$(nix flake metadata --json "$flake3Dir" | jq -r .locks.nodes.flake1.locked.rev)" = "$newFlake1Rev" ]] cat > "$flake3Dir/flake.nix" < "$flake3Dir/flake.nix" < "$subdirFlakeDir1"/flake.nix < "$subdirFlakeDir2"/flake.nix < $flakeFollowsA/flake.nix < "$flakeFollowsA"/flake.nix < $flakeFollowsA/flake.nix < $flakeFollowsB/flake.nix < "$flakeFollowsB"/flake.nix < $flakeFollowsB/flake.nix < $flakeFollowsC/flake.nix < "$flakeFollowsC"/flake.nix < $flakeFollowsC/flake.nix < $flakeFollowsD/flake.nix < "$flakeFollowsD"/flake.nix < $flakeFollowsD/flake.nix < $flakeFollowsE/flake.nix < "$flakeFollowsE"/flake.nix < $flakeFollowsE/flake.nix < $flakeFollowsA/flake.nix < "$flakeFollowsA"/flake.nix < $flakeFollowsA/flake.nix < $flakeFollowsA/flake.nix < "$flakeFollowsA"/flake.nix < $flakeFollowsA/flake.nix <&1 | grep '/flakeB.*is forbidden in pure evaluation mode' -expect 1 nix flake lock --impure $flakeFollowsA 2>&1 | grep '/flakeB.*does not exist' +expect 1 nix flake lock "$flakeFollowsA" 2>&1 | grep '/flakeB.*is forbidden in pure evaluation mode' +expect 1 nix flake lock --impure "$flakeFollowsA" 2>&1 | grep "error: 'flakeB' is too short to be a valid store path" # Test relative non-flake inputs. -cat > $flakeFollowsA/flake.nix < "$flakeFollowsA"/flake.nix < $flakeFollowsA/flake.nix < $flakeFollowsA/foo.nix +echo 123 > "$flakeFollowsA"/foo.nix -git -C $flakeFollowsA add flake.nix foo.nix +git -C "$flakeFollowsA" add flake.nix foo.nix -nix flake lock $flakeFollowsA +nix flake lock "$flakeFollowsA" -[[ $(nix eval --json $flakeFollowsA#e) = 123 ]] +[[ $(nix eval --json "$flakeFollowsA"#e) = 123 ]] # Non-existant follows should print a warning. -cat >$flakeFollowsA/flake.nix <"$flakeFollowsA"/flake.nix <$flakeFollowsA/flake.nix <&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid'" nix flake lock "$flakeFollowsA" 2>&1 | grep "warning: input 'B' has an override for a non-existent input 'invalid2'" @@ -269,7 +269,7 @@ flakeFollowCycle="$TEST_ROOT/follows/followCycle" # Test following path flakerefs. mkdir -p "$flakeFollowCycle" -cat > $flakeFollowCycle/flake.nix < "$flakeFollowCycle"/flake.nix < $flakeFollowCycle/flake.nix <&1 && fail "nix flake lock should have failed." || true) -echo $checkRes | grep -F "error: follow cycle detected: [baz -> foo -> bar -> baz]" +echo "$checkRes" | grep -F "error: follow cycle detected: [baz -> foo -> bar -> baz]" # Test transitive input url locking @@ -362,22 +363,22 @@ echo "$json" | jq .locks.nodes.C.original # Test deep overrides, e.g. `inputs.B.inputs.C.inputs.D.follows = ...`. -cat < $flakeFollowsD/flake.nix +cat < "$flakeFollowsD"/flake.nix { outputs = _: {}; } EOF -cat < $flakeFollowsC/flake.nix +cat < "$flakeFollowsC"/flake.nix { inputs.D.url = "path:nosuchflake"; outputs = _: {}; } EOF -cat < $flakeFollowsB/flake.nix +cat < "$flakeFollowsB"/flake.nix { inputs.C.url = "path:$flakeFollowsC"; outputs = _: {}; } EOF -cat < $flakeFollowsA/flake.nix +cat < "$flakeFollowsA"/flake.nix { inputs.B.url = "path:$flakeFollowsB"; inputs.D.url = "path:$flakeFollowsD"; @@ -386,26 +387,26 @@ cat < $flakeFollowsA/flake.nix } EOF -nix flake lock $flakeFollowsA +nix flake lock "$flakeFollowsA" -[[ $(jq -c .nodes.C.inputs.D $flakeFollowsA/flake.lock) = '["D"]' ]] +[[ $(jq -c .nodes.C.inputs.D "$flakeFollowsA"/flake.lock) = '["D"]' ]] # Test overlapping flake follows: B has D follow C/D, while A has B/C follow C -cat < $flakeFollowsC/flake.nix +cat < "$flakeFollowsC"/flake.nix { inputs.D.url = "path:$flakeFollowsD"; outputs = _: {}; } EOF -cat < $flakeFollowsB/flake.nix +cat < "$flakeFollowsB"/flake.nix { inputs.C.url = "path:nosuchflake"; inputs.D.follows = "C/D"; outputs = _: {}; } EOF -cat < $flakeFollowsA/flake.nix +cat < "$flakeFollowsA"/flake.nix { inputs.B.url = "path:$flakeFollowsB"; inputs.C.url = "path:$flakeFollowsC"; @@ -415,12 +416,12 @@ cat < $flakeFollowsA/flake.nix EOF # bug was not triggered without recreating the lockfile -nix flake lock $flakeFollowsA --recreate-lock-file +nix flake lock "$flakeFollowsA" --recreate-lock-file -[[ $(jq -c .nodes.B.inputs.D $flakeFollowsA/flake.lock) = '["B","C","D"]' ]] +[[ $(jq -c .nodes.B.inputs.D "$flakeFollowsA"/flake.lock) = '["B","C","D"]' ]] # Check that you can't have both a flakeref and a follows attribute on an input. -cat < $flakeFollowsB/flake.nix +cat < "$flakeFollowsB"/flake.nix { inputs.C.url = "path:nosuchflake"; inputs.D.url = "path:nosuchflake"; @@ -429,4 +430,4 @@ cat < $flakeFollowsB/flake.nix } EOF -expectStderr 1 nix flake lock $flakeFollowsA --recreate-lock-file | grepQuiet "flake input has both a flake reference and a follows attribute" +expectStderr 1 nix flake lock "$flakeFollowsA" --recreate-lock-file | grepQuiet "flake input has both a flake reference and a follows attribute" diff --git a/tests/functional/flakes/init.sh b/tests/functional/flakes/init.sh index 9e484f71cc0..0a17fd18dfe 100755 --- a/tests/functional/flakes/init.sh +++ b/tests/functional/flakes/init.sh @@ -61,8 +61,8 @@ nix flake show templates nix flake show templates --json | jq createGitRepo "$flakeDir" -(cd "$flakeDir" && nix flake init) -(cd "$flakeDir" && nix flake init) # check idempotence +(cd "$flakeDir" && nix flake init --template "git+file://$templatesDir") +(cd "$flakeDir" && nix flake init --template "git+file://$templatesDir") # check idempotence git -C "$flakeDir" add flake.nix nix flake check "$flakeDir" nix flake show "$flakeDir" @@ -72,13 +72,13 @@ git -C "$flakeDir" commit -a -m 'Initial' # Test 'nix flake init' with benign conflicts createGitRepo "$flakeDir" echo a > "$flakeDir/a" -(cd "$flakeDir" && nix flake init) # check idempotence +(cd "$flakeDir" && nix flake init --template "git+file://$templatesDir") # check idempotence # Test 'nix flake init' with conflicts createGitRepo "$flakeDir" echo b > "$flakeDir/a" pushd "$flakeDir" -(! nix flake init) |& grep "refusing to overwrite existing file '$flakeDir/a'" +(! nix flake init --template "git+file://$templatesDir") |& grep "refusing to overwrite existing file '$flakeDir/a'" popd git -C "$flakeDir" commit -a -m 'Changed' diff --git a/tests/functional/flakes/mercurial.sh b/tests/functional/flakes/mercurial.sh index b9045bf6bad..b6c14fc2605 100755 --- a/tests/functional/flakes/mercurial.sh +++ b/tests/functional/flakes/mercurial.sh @@ -27,9 +27,9 @@ nix build -o "$TEST_ROOT/result" "hg+file://$flake2Dir" (! nix flake metadata --json "hg+file://$flake2Dir" | jq -e -r .revision) -nix eval "hg+file://$flake2Dir"#expr +_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval "hg+file://$flake2Dir"#expr -nix eval "hg+file://$flake2Dir"#expr +_NIX_TEST_BARF_ON_UNCACHEABLE='' nix eval "hg+file://$flake2Dir"#expr (! nix eval "hg+file://$flake2Dir"#expr --no-allow-dirty) diff --git a/tests/functional/flakes/meson.build b/tests/functional/flakes/meson.build index de76a55804a..9a6511f2b19 100644 --- a/tests/functional/flakes/meson.build +++ b/tests/functional/flakes/meson.build @@ -34,6 +34,8 @@ suites += { 'source-paths.sh', 'old-lockfiles.sh', 'trace-ifd.sh', + 'build-time-flake-inputs.sh', + 'substitution.sh', ], 'workdir' : meson.current_source_dir(), } diff --git a/tests/functional/flakes/non-flake-inputs.sh b/tests/functional/flakes/non-flake-inputs.sh index 05e65604226..6b1c6a94106 100644 --- a/tests/functional/flakes/non-flake-inputs.sh +++ b/tests/functional/flakes/non-flake-inputs.sh @@ -81,7 +81,7 @@ nix build -o "$TEST_ROOT/result" "$flake3Dir#sth" --commit-lock-file nix registry add --registry "$registry" flake3 "git+file://$flake3Dir" -nix build -o "$TEST_ROOT/result" flake3#fnord +_NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#fnord [[ $(cat "$TEST_ROOT/result") = FNORD ]] # Check whether flake input fetching is lazy: flake3#sth does not @@ -91,16 +91,17 @@ clearStore mv "$flake2Dir" "$flake2Dir.tmp" mv "$nonFlakeDir" "$nonFlakeDir.tmp" nix build -o "$TEST_ROOT/result" flake3#sth -(! nix build -o "$TEST_ROOT/result" flake3#xyzzy) -(! nix build -o "$TEST_ROOT/result" flake3#fnord) +(! _NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#xyzzy) +(! _NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#fnord) mv "$flake2Dir.tmp" "$flake2Dir" mv "$nonFlakeDir.tmp" "$nonFlakeDir" -nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord +_NIX_TEST_BARF_ON_UNCACHEABLE='' nix build -o "$TEST_ROOT/result" flake3#xyzzy flake3#fnord # Check non-flake inputs have a sourceInfo and an outPath # # This may look redundant, but the other checks below happen in a command # substitution subshell, so failures there will not exit this shell +export _NIX_TEST_BARF_ON_UNCACHEABLE='' # FIXME nix eval --raw flake3#inputs.nonFlake.outPath nix eval --raw flake3#inputs.nonFlake.sourceInfo.outPath nix eval --raw flake3#inputs.nonFlakeFile.outPath diff --git a/tests/functional/flakes/prefetch.sh b/tests/functional/flakes/prefetch.sh index a451b712009..999270c1ea3 100755 --- a/tests/functional/flakes/prefetch.sh +++ b/tests/functional/flakes/prefetch.sh @@ -3,6 +3,6 @@ source common.sh # Test symlinks in zip files (#10649). -path=$(nix flake prefetch --json file://$(pwd)/tree.zip | jq -r .storePath) -[[ $(cat $path/foo) = foo ]] -[[ $(readlink $path/bar) = foo ]] +path=$(nix flake prefetch --json file://"$(pwd)"/tree.zip | jq -r .storePath) +[[ $(cat "$path"/foo) = foo ]] +[[ $(readlink "$path"/bar) = foo ]] diff --git a/tests/functional/flakes/relative-paths-lockfile.sh b/tests/functional/flakes/relative-paths-lockfile.sh index d91aedd16cd..662c9329ca7 100644 --- a/tests/functional/flakes/relative-paths-lockfile.sh +++ b/tests/functional/flakes/relative-paths-lockfile.sh @@ -4,6 +4,8 @@ source ./common.sh requireGit +unset _NIX_TEST_BARF_ON_UNCACHEABLE + # Test a "vendored" subflake dependency. This is a relative path flake # which doesn't reference the root flake and has its own lock file. # diff --git a/tests/functional/flakes/run.sh b/tests/functional/flakes/run.sh index 0a294782592..107b3dfb8ee 100755 --- a/tests/functional/flakes/run.sh +++ b/tests/functional/flakes/run.sh @@ -5,10 +5,10 @@ source ../common.sh TODO_NixOS clearStore -rm -rf $TEST_HOME/.cache $TEST_HOME/.config $TEST_HOME/.local +rm -rf "$TEST_HOME"/.cache "$TEST_HOME"/.config "$TEST_HOME"/.local -cp ../shell-hello.nix "${config_nix}" $TEST_HOME -cd $TEST_HOME +cp ../shell-hello.nix "${config_nix}" "$TEST_HOME" +cd "$TEST_HOME" cat < flake.nix { @@ -34,8 +34,8 @@ nix run --no-write-lock-file .#pkgAsPkg # For instance, we might set an environment variable temporarily to affect some # initialization or whatnot, but this must not leak into the environment of the # command being run. -env > $TEST_ROOT/expected-env -nix run -f shell-hello.nix env > $TEST_ROOT/actual-env +env > "$TEST_ROOT"/expected-env +nix run -f shell-hello.nix env > "$TEST_ROOT"/actual-env # Remove/reset variables we expect to be different. # - PATH is modified by nix shell # - we unset TMPDIR on macOS if it contains /var/folders. bad. https://github.com/NixOS/nix/issues/7731 @@ -48,12 +48,12 @@ sed -i \ -e '/^TMPDIR=\/var\/folders\/.*/d' \ -e '/^__CF_USER_TEXT_ENCODING=.*$/d' \ -e '/^__LLVM_PROFILE_RT_INIT_ONCE=.*$/d' \ - $TEST_ROOT/expected-env $TEST_ROOT/actual-env -sort $TEST_ROOT/expected-env | uniq > $TEST_ROOT/expected-env.sorted + "$TEST_ROOT"/expected-env "$TEST_ROOT"/actual-env +sort "$TEST_ROOT"/expected-env | uniq > "$TEST_ROOT"/expected-env.sorted # nix run appears to clear _. I don't understand why. Is this ok? -echo "_=..." >> $TEST_ROOT/actual-env -sort $TEST_ROOT/actual-env | uniq > $TEST_ROOT/actual-env.sorted -diff $TEST_ROOT/expected-env.sorted $TEST_ROOT/actual-env.sorted +echo "_=..." >> "$TEST_ROOT"/actual-env +sort "$TEST_ROOT"/actual-env | uniq > "$TEST_ROOT"/actual-env.sorted +diff "$TEST_ROOT"/expected-env.sorted "$TEST_ROOT"/actual-env.sorted clearStore diff --git a/tests/functional/flakes/show.sh b/tests/functional/flakes/show.sh index 7fcc6aca9b4..c8238bf82b4 100755 --- a/tests/functional/flakes/show.sh +++ b/tests/functional/flakes/show.sh @@ -12,6 +12,7 @@ pushd "$flakeDir" # By default: Only show the packages content for the current system and no # legacyPackages at all nix flake show --json > show-output.json +# shellcheck disable=SC2016 nix eval --impure --expr ' let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); in @@ -23,6 +24,7 @@ true # With `--all-systems`, show the packages for all systems nix flake show --json --all-systems > show-output.json +# shellcheck disable=SC2016 nix eval --impure --expr ' let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); in @@ -33,6 +35,7 @@ true # With `--legacy`, show the legacy packages nix flake show --json --legacy > show-output.json +# shellcheck disable=SC2016 nix eval --impure --expr ' let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); in @@ -59,13 +62,7 @@ cat >flake.nix < show-output.json -nix eval --impure --expr ' -let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); -in -assert show_output == { }; -true -' +[[ $(nix flake show --all-systems --legacy | wc -l) = 1 ]] # Test that attributes with errors are handled correctly. # nixpkgs.legacyPackages is a particularly prominent instance of this. @@ -80,6 +77,7 @@ cat >flake.nix < show-output.json +# shellcheck disable=SC2016 nix eval --impure --expr ' let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); in @@ -91,11 +89,12 @@ true # Test that nix flake show doesn't fail if one of the outputs contains # an IFD popd -writeIfdFlake $flakeDir -pushd $flakeDir +writeIfdFlake "$flakeDir" +pushd "$flakeDir" nix flake show --json > show-output.json +# shellcheck disable=SC2016 nix eval --impure --expr ' let show_output = builtins.fromJSON (builtins.readFile ./show-output.json); in diff --git a/tests/functional/flakes/source-paths.sh b/tests/functional/flakes/source-paths.sh index 4709bf2fcec..3aa3683c27c 100644 --- a/tests/functional/flakes/source-paths.sh +++ b/tests/functional/flakes/source-paths.sh @@ -12,6 +12,10 @@ cat > "$repo/flake.nix" < "$repo/foo" + +expectStderr 1 nix eval "$repo#z" | grepQuiet "error: Path 'foo' in the repository \"$repo\" is not tracked by Git." +expectStderr 1 nix eval "$repo#a" | grepQuiet "error: Path 'foo' in the repository \"$repo\" is not tracked by Git." + +git -C "$repo" add "$repo/foo" + +[[ $(nix eval --raw "$repo#z") = 123 ]] + +expectStderr 1 nix eval "$repo#b" | grepQuiet "error: Path 'dir' does not exist in Git repository \"$repo\"." + +mkdir -p "$repo/dir" +echo 456 > "$repo/dir/default.nix" + +expectStderr 1 nix eval "$repo#b" | grepQuiet "error: Path 'dir' in the repository \"$repo\" is not tracked by Git." + +git -C "$repo" add "$repo/dir/default.nix" + +[[ $(nix eval "$repo#b") = 456 ]] diff --git a/tests/functional/flakes/substitution.sh b/tests/functional/flakes/substitution.sh new file mode 100644 index 00000000000..f7ea6001ce3 --- /dev/null +++ b/tests/functional/flakes/substitution.sh @@ -0,0 +1,31 @@ +#! /usr/bin/env bash + +# Test that inputs are substituted if they cannot be fetched from their original location. + +source ./common.sh + +if [[ $(nix config show lazy-trees) = true ]]; then + exit 0 +fi + +TODO_NixOS + +createFlake1 +createFlake2 + +nix build --no-link "$flake2Dir#bar" + +path1="$(nix flake metadata --json "$flake1Dir" | jq -r .path)" + +# Building after an input disappeared should succeed, because it's still in the Nix store. +mv "$flake1Dir" "$flake1Dir-tmp" +nix build --no-link "$flake2Dir#bar" --no-eval-cache + +# Check that Nix will fall back to fetching the input from a substituter. +cache="file://$TEST_ROOT/binary-cache" +nix copy --to "$cache" "$path1" +clearStore +nix build --no-link "$flake2Dir#bar" --no-eval-cache --substitute --substituters "$cache" + +clearStore +expectStderr 1 nix build --no-link "$flake2Dir#bar" --no-eval-cache | grepQuiet "The path.*does not exist" diff --git a/tests/functional/flakes/unlocked-override.sh b/tests/functional/flakes/unlocked-override.sh index ed05440de03..ed4d131b7ad 100755 --- a/tests/functional/flakes/unlocked-override.sh +++ b/tests/functional/flakes/unlocked-override.sh @@ -36,6 +36,7 @@ expectStderr 1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/f grepQuiet "Not writing lock file.*because it has an unlocked input" nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" --allow-dirty-locks +_NIX_TEST_FAIL_ON_LARGE_PATH=1 nix flake lock "$flake2Dir" --override-input flake1 "$TEST_ROOT/flake1" --allow-dirty-locks --warn-large-path-threshold 1 --lazy-trees # Using a lock file with a dirty lock does not require --allow-dirty-locks, but should print a warning. expectStderr 0 nix eval "$flake2Dir#x" | diff --git a/tests/functional/formatter.sh b/tests/functional/formatter.sh index 6631dd6b87a..03b31708d67 100755 --- a/tests/functional/formatter.sh +++ b/tests/functional/formatter.sh @@ -16,6 +16,7 @@ nix fmt --help | grep "reformat your code" nix fmt run --help | grep "reformat your code" nix fmt build --help | grep "build" +# shellcheck disable=SC2154 cat << EOF > flake.nix { outputs = _: { diff --git a/tests/functional/gc-auto.sh b/tests/functional/gc-auto.sh index efe3e4b2bb5..ea877f27f1f 100755 --- a/tests/functional/gc-auto.sh +++ b/tests/functional/gc-auto.sh @@ -2,22 +2,26 @@ source common.sh +# shellcheck disable=SC1111 needLocalStore "“min-free” and “max-free” are daemon options" TODO_NixOS clearStore +# shellcheck disable=SC2034 garbage1=$(nix store add-path --name garbage1 ./nar-access.sh) +# shellcheck disable=SC2034 garbage2=$(nix store add-path --name garbage2 ./nar-access.sh) +# shellcheck disable=SC2034 garbage3=$(nix store add-path --name garbage3 ./nar-access.sh) -ls -l $garbage3 -POSIXLY_CORRECT=1 du $garbage3 +ls -l "$garbage3" +POSIXLY_CORRECT=1 du "$garbage3" fake_free=$TEST_ROOT/fake-free export _NIX_TEST_FREE_SPACE_FILE=$fake_free -echo 1100 > $fake_free +echo 1100 > "$fake_free" fifoLock=$TEST_ROOT/fifoLock mkfifo "$fifoLock" @@ -65,11 +69,11 @@ with import ${config_nix}; mkDerivation { EOF ) -nix build --impure -v -o $TEST_ROOT/result-A -L --expr "$expr" \ +nix build --impure -v -o "$TEST_ROOT"/result-A -L --expr "$expr" \ --min-free 1K --max-free 2K --min-free-check-interval 1 & pid1=$! -nix build --impure -v -o $TEST_ROOT/result-B -L --expr "$expr2" \ +nix build --impure -v -o "$TEST_ROOT"/result-B -L --expr "$expr2" \ --min-free 1K --max-free 2K --min-free-check-interval 1 & pid2=$! @@ -77,9 +81,9 @@ pid2=$! # If the first build fails, we need to postpone the failure to still allow # the second one to finish wait "$pid1" || FIRSTBUILDSTATUS=$? -echo "unlock" > $fifoLock -( exit ${FIRSTBUILDSTATUS:-0} ) +echo "unlock" > "$fifoLock" +( exit "${FIRSTBUILDSTATUS:-0}" ) wait "$pid2" -[[ foo = $(cat $TEST_ROOT/result-A/bar) ]] -[[ foo = $(cat $TEST_ROOT/result-B/bar) ]] +[[ foo = $(cat "$TEST_ROOT"/result-A/bar) ]] +[[ foo = $(cat "$TEST_ROOT"/result-B/bar) ]] diff --git a/tests/functional/gc-concurrent.builder.sh b/tests/functional/gc-concurrent.builder.sh index bb6dcd4cfb9..b3c7abeb1f6 100644 --- a/tests/functional/gc-concurrent.builder.sh +++ b/tests/functional/gc-concurrent.builder.sh @@ -1,16 +1,19 @@ +# shellcheck shell=bash +# shellcheck disable=SC2154 echo "Build started" > "$lockFifo" -mkdir $out -echo $(cat $input1/foo)$(cat $input2/bar) > $out/foobar +# shellcheck disable=SC2154 +mkdir "$out" +echo "$(cat "$input1"/foo)""$(cat "$input2"/bar)" > "$out"/foobar # Wait for someone to write on the fifo cat "$lockFifo" # $out should not have been GC'ed while we were sleeping, but just in # case... -mkdir -p $out +mkdir -p "$out" # Check that the GC hasn't deleted the lock on our output. test -e "$out.lock" -ln -s $input2 $out/input-2 +ln -s "$input2" "$out"/input-2 diff --git a/tests/functional/gc-concurrent.sh b/tests/functional/gc-concurrent.sh index df180b14fd7..dcfcea3e960 100755 --- a/tests/functional/gc-concurrent.sh +++ b/tests/functional/gc-concurrent.sh @@ -10,54 +10,58 @@ lockFifo1=$TEST_ROOT/test1.fifo mkfifo "$lockFifo1" drvPath1=$(nix-instantiate gc-concurrent.nix -A test1 --argstr lockFifo "$lockFifo1") -outPath1=$(nix-store -q $drvPath1) +outPath1=$(nix-store -q "$drvPath1") drvPath2=$(nix-instantiate gc-concurrent.nix -A test2) -outPath2=$(nix-store -q $drvPath2) +outPath2=$(nix-store -q "$drvPath2") drvPath3=$(nix-instantiate simple.nix) -outPath3=$(nix-store -r $drvPath3) +outPath3=$(nix-store -r "$drvPath3") -(! test -e $outPath3.lock) -touch $outPath3.lock +# shellcheck disable=SC2235 +(! test -e "$outPath3".lock) +touch "$outPath3".lock rm -f "$NIX_STATE_DIR"/gcroots/foo* -ln -s $drvPath2 "$NIX_STATE_DIR/gcroots/foo" -ln -s $outPath3 "$NIX_STATE_DIR/gcroots/foo2" +ln -s "$drvPath2" "$NIX_STATE_DIR/gcroots/foo" +ln -s "$outPath3" "$NIX_STATE_DIR/gcroots/foo2" # Start build #1 in the background. It starts immediately. nix-store -rvv "$drvPath1" & pid1=$! # Wait for the build of $drvPath1 to start -cat $lockFifo1 +cat "$lockFifo1" # Run the garbage collector while the build is running. nix-collect-garbage # Unlock the build of $drvPath1 -echo "" > $lockFifo1 +echo "" > "$lockFifo1" echo waiting for pid $pid1 to finish... wait $pid1 # Check that the root of build #1 and its dependencies haven't been # deleted. The should not be deleted by the GC because they were # being built during the GC. -cat $outPath1/foobar -cat $outPath1/input-2/bar +cat "$outPath1"/foobar +cat "$outPath1"/input-2/bar # Check that the build build $drvPath2 succeeds. # It should succeed because the derivation is a GC root. nix-store -rvv "$drvPath2" -cat $outPath2/foobar +cat "$outPath2"/foobar rm -f "$NIX_STATE_DIR"/gcroots/foo* # The collector should have deleted lock files for paths that have # been built previously. -(! test -e $outPath3.lock) +# shellcheck disable=SC2235 +(! test -e "$outPath3".lock) # If we run the collector now, it should delete outPath1/2. nix-collect-garbage -(! test -e $outPath1) -(! test -e $outPath2) +# shellcheck disable=SC2235 +(! test -e "$outPath1") +# shellcheck disable=SC2235 +(! test -e "$outPath2") diff --git a/tests/functional/gc-concurrent2.builder.sh b/tests/functional/gc-concurrent2.builder.sh index 4f6c58b96fe..4b1ad6f5e37 100644 --- a/tests/functional/gc-concurrent2.builder.sh +++ b/tests/functional/gc-concurrent2.builder.sh @@ -1,5 +1,8 @@ -mkdir $out -echo $(cat $input1/foo)$(cat $input2/bar)xyzzy > $out/foobar +# shellcheck shell=bash +# shellcheck disable=SC2154 +mkdir "$out" +# shellcheck disable=SC2154 +echo "$(cat "$input1"/foo)""$(cat "$input2"/bar)"xyzzy > "$out"/foobar # Check that the GC hasn't deleted the lock on our output. test -e "$out.lock" diff --git a/tests/functional/gc-non-blocking.sh b/tests/functional/gc-non-blocking.sh index 9cd5c0e1cd2..a85b8e5db41 100755 --- a/tests/functional/gc-non-blocking.sh +++ b/tests/functional/gc-non-blocking.sh @@ -23,17 +23,17 @@ mkfifo "$fifo2" dummy=$(nix store add-path ./simple.nix) running=$TEST_ROOT/running -touch $running +touch "$running" # Start GC. -(_NIX_TEST_GC_SYNC_1=$fifo1 _NIX_TEST_GC_SYNC_2=$fifo2 nix-store --gc -vvvvv; rm $running) & +(_NIX_TEST_GC_SYNC_1=$fifo1 _NIX_TEST_GC_SYNC_2=$fifo2 nix-store --gc -vvvvv; rm "$running") & pid=$! sleep 2 # Delay the start of the root server to check that the build below # correctly handles ENOENT when connecting to the root server. -(sleep 1; echo > $fifo1) & +(sleep 1; echo > "$fifo1") & pid2=$! # Start a build. This should not be blocked by the GC in progress. @@ -47,6 +47,8 @@ outPath=$(nix-build --max-silent-time 60 -o "$TEST_ROOT/result" -E " wait $pid wait $pid2 -(! test -e $running) -(! test -e $dummy) -test -e $outPath +# shellcheck disable=SC2235 +(! test -e "$running") +# shellcheck disable=SC2235 +(! test -e "$dummy") +test -e "$outPath" diff --git a/tests/functional/gc-runtime.nix b/tests/functional/gc-runtime.nix index ee5980bdff9..df7f8ad1647 100644 --- a/tests/functional/gc-runtime.nix +++ b/tests/functional/gc-runtime.nix @@ -9,6 +9,7 @@ mkDerivation { cat > $out/program < \$TEST_ROOT/fifo sleep 10000 EOF diff --git a/tests/functional/gc-runtime.sh b/tests/functional/gc-runtime.sh index 0cccaaf16ab..34e99415d5c 100755 --- a/tests/functional/gc-runtime.sh +++ b/tests/functional/gc-runtime.sh @@ -21,11 +21,16 @@ nix-env -p "$profiles/test" -f ./gc-runtime.nix -i gc-runtime outPath=$(nix-env -p "$profiles/test" -q --no-name --out-path gc-runtime) echo "$outPath" +fifo="$TEST_ROOT/fifo" +mkfifo "$fifo" + echo "backgrounding program..." -"$profiles"/test/program & -sleep 2 # hack - wait for the program to get started +"$profiles"/test/program "$fifo" & child=$! echo PID=$child +cat "$fifo" + +expectStderr 1 nix-store --delete "$outPath" | grepQuiet "Cannot delete path.*because it's referenced by the GC root '/proc/" nix-env -p "$profiles/test" -e gc-runtime nix-env -p "$profiles/test" --delete-generations old diff --git a/tests/functional/gc.sh b/tests/functional/gc.sh index c58f47021f8..3ade6e4f582 100755 --- a/tests/functional/gc.sh +++ b/tests/functional/gc.sh @@ -13,7 +13,7 @@ outPath=$(nix-store -rvv "$drvPath") rm -f "$NIX_STATE_DIR/gcroots/foo" ln -sf "$outPath" "$NIX_STATE_DIR/gcroots/foo" -[ "$(nix-store -q --roots "$outPath")" = "$NIX_STATE_DIR/gcroots/foo -> $outPath" ] +expectStderr 0 nix-store -q --roots "$outPath" | grepQuiet "$NIX_STATE_DIR/gcroots/foo -> $outPath" nix-store --gc --print-roots | grep "$outPath" nix-store --gc --print-live | grep "$outPath" @@ -23,10 +23,10 @@ if nix-store --gc --print-dead | grep -E "$outPath"$; then false; fi nix-store --gc --print-dead inUse=$(readLink "$outPath/reference-to-input-2") -if nix-store --delete "$inUse"; then false; fi +expectStderr 1 nix-store --delete "$inUse" | grepQuiet "Cannot delete path.*because it's referenced by path '" test -e "$inUse" -if nix-store --delete "$outPath"; then false; fi +expectStderr 1 nix-store --delete "$outPath" | grepQuiet "Cannot delete path.*because it's referenced by the GC root " test -e "$outPath" for i in "$NIX_STORE_DIR"/*; do diff --git a/tests/functional/hash-convert.sh b/tests/functional/hash-convert.sh index c40cb469c76..9ef4c189de4 100755 --- a/tests/functional/hash-convert.sh +++ b/tests/functional/hash-convert.sh @@ -99,7 +99,7 @@ try3() { expectStderr 1 nix hash convert --hash-algo "$1" --from nix32 "$4" | grepQuiet "input hash" # Base-16 hashes can be in uppercase. - nix hash convert --hash-algo "$1" --from base16 "$(echo $2 | tr [a-z] [A-Z])" + nix hash convert --hash-algo "$1" --from base16 "$(echo "$2" | tr '[:lower:]' '[:upper:]')" } try3 sha1 "800d59cfcd3c05e900cb4e214be48f6b886a08df" "vw46m23bizj4n8afrc0fj19wrp7mj3c0" "gA1Zz808BekAy04hS+SPa4hqCN8=" diff --git a/tests/functional/impure-derivations.sh b/tests/functional/impure-derivations.sh index 5dea220fec7..1c942dc52d6 100755 --- a/tests/functional/impure-derivations.sh +++ b/tests/functional/impure-derivations.sh @@ -12,62 +12,62 @@ restartDaemon clearStoreIfPossible # Basic test of impure derivations: building one a second time should not use the previous result. -printf 0 > $TEST_ROOT/counter +printf 0 > "$TEST_ROOT"/counter # `nix derivation add` with impure derivations work drvPath=$(nix-instantiate ./impure-derivations.nix -A impure) -nix derivation show $drvPath | jq .[] > $TEST_HOME/impure-drv.json -drvPath2=$(nix derivation add < $TEST_HOME/impure-drv.json) +nix derivation show "$drvPath" | jq .[] > "$TEST_HOME"/impure-drv.json +drvPath2=$(nix derivation add < "$TEST_HOME"/impure-drv.json) [[ "$drvPath" = "$drvPath2" ]] # But only with the experimental feature! -expectStderr 1 nix derivation add < $TEST_HOME/impure-drv.json --experimental-features nix-command | grepQuiet "experimental Nix feature 'impure-derivations' is disabled" +expectStderr 1 nix derivation add < "$TEST_HOME"/impure-drv.json --experimental-features '' | grepQuiet "experimental Nix feature 'impure-derivations' is disabled" nix build --dry-run --json --file ./impure-derivations.nix impure.all json=$(nix build -L --no-link --json --file ./impure-derivations.nix impure.all) -path1=$(echo $json | jq -r .[].outputs.out) -path1_stuff=$(echo $json | jq -r .[].outputs.stuff) -[[ $(< $path1/n) = 0 ]] -[[ $(< $path1_stuff/bla) = 0 ]] +path1=$(echo "$json" | jq -r .[].outputs.out) +path1_stuff=$(echo "$json" | jq -r .[].outputs.stuff) +[[ $(< "$path1"/n) = 0 ]] +[[ $(< "$path1_stuff"/bla) = 0 ]] -[[ $(nix path-info --json $path1 | jq .[].ca) =~ fixed:r:sha256: ]] +[[ $(nix path-info --json "$path1" | jq .[].ca) =~ fixed:r:sha256: ]] path2=$(nix build -L --no-link --json --file ./impure-derivations.nix impure | jq -r .[].outputs.out) -[[ $(< $path2/n) = 1 ]] +[[ $(< "$path2"/n) = 1 ]] # Test impure derivations that depend on impure derivations. path3=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out) -[[ $(< $path3/n) = X2 ]] +[[ $(< "$path3"/n) = X2 ]] path4=$(nix build -L --no-link --json --file ./impure-derivations.nix impureOnImpure | jq -r .[].outputs.out) -[[ $(< $path4/n) = X3 ]] +[[ $(< "$path4"/n) = X3 ]] # Test that (self-)references work. -[[ $(< $path4/symlink/bla) = 3 ]] -[[ $(< $path4/self/n) = X3 ]] +[[ $(< "$path4"/symlink/bla) = 3 ]] +[[ $(< "$path4"/self/n) = X3 ]] # Input-addressed derivations cannot depend on impure derivations directly. (! nix build -L --no-link --json --file ./impure-derivations.nix inputAddressed 2>&1) | grep 'depends on impure derivation' drvPath=$(nix eval --json --file ./impure-derivations.nix impure.drvPath | jq -r .) -[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.out.impure") = true ]] -[[ $(nix derivation show $drvPath | jq ".[\"$drvPath\"].outputs.stuff.impure") = true ]] +[[ $(nix derivation show "$drvPath" | jq ".[\"$(basename "$drvPath")\"].outputs.out.impure") = true ]] +[[ $(nix derivation show "$drvPath" | jq ".[\"$(basename "$drvPath")\"].outputs.stuff.impure") = true ]] # Fixed-output derivations *can* depend on impure derivations. path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) -[[ $(< $path5) = X ]] -[[ $(< $TEST_ROOT/counter) = 5 ]] +[[ $(< "$path5") = X ]] +[[ $(< "$TEST_ROOT"/counter) = 5 ]] # And they should not be rebuilt. path5=$(nix build -L --no-link --json --file ./impure-derivations.nix contentAddressed | jq -r .[].outputs.out) -[[ $(< $path5) = X ]] -[[ $(< $TEST_ROOT/counter) = 5 ]] +[[ $(< "$path5") = X ]] +[[ $(< "$TEST_ROOT"/counter) = 5 ]] # Input-addressed derivations can depend on fixed-output derivations that depend on impure derivations. path6=$(nix build -L --no-link --json --file ./impure-derivations.nix inputAddressedAfterCA | jq -r .[].outputs.out) -[[ $(< $path6) = X ]] -[[ $(< $TEST_ROOT/counter) = 5 ]] +[[ $(< "$path6") = X ]] +[[ $(< "$TEST_ROOT"/counter) = 5 ]] # Test nix/fetchurl.nix. path7=$(nix build -L --no-link --print-out-paths --expr "import { impure = true; url = file://$PWD/impure-derivations.sh; }") -cmp $path7 $PWD/impure-derivations.sh +cmp "$path7" "$PWD"/impure-derivations.sh diff --git a/tests/functional/install-darwin.sh b/tests/functional/install-darwin.sh index ea2b753239b..0070e9dcee6 100755 --- a/tests/functional/install-darwin.sh +++ b/tests/functional/install-darwin.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash set -eux @@ -21,12 +21,13 @@ cleanup() { for file in ~/.bash_profile ~/.bash_login ~/.profile ~/.zshenv ~/.zprofile ~/.zshrc ~/.zlogin; do if [ -e "$file" ]; then + # shellcheck disable=SC2002 cat "$file" | grep -v nix-profile > "$file.next" mv "$file.next" "$file" fi done - for i in $(seq 1 $(sysctl -n hw.ncpu)); do + for i in $(seq 1 "$(sysctl -n hw.ncpu)"); do sudo /usr/bin/dscl . -delete "/Users/nixbld$i" || true done sudo /usr/bin/dscl . -delete "/Groups/nixbld" || true @@ -65,11 +66,11 @@ verify echo nix-build ./release.nix -A binaryTarball.x86_64-darwin ) | bash -l set -e - cp ./result/nix-*.tar.bz2 $scratch/nix.tar.bz2 + cp ./result/nix-*.tar.bz2 "$scratch"/nix.tar.bz2 ) ( - cd $scratch + cd "$scratch" tar -xf ./nix.tar.bz2 cd nix-* diff --git a/tests/functional/lang/eval-fail-blackhole.err.exp b/tests/functional/lang/eval-fail-blackhole.err.exp index 95e33a5fe45..d11eb338f9a 100644 --- a/tests/functional/lang/eval-fail-blackhole.err.exp +++ b/tests/functional/lang/eval-fail-blackhole.err.exp @@ -7,8 +7,8 @@ error: 3| x = y; error: infinite recursion encountered - at /pwd/lang/eval-fail-blackhole.nix:3:7: + at /pwd/lang/eval-fail-blackhole.nix:2:3: + 1| let { 2| body = x; + | ^ 3| x = y; - | ^ - 4| y = x; diff --git a/tests/functional/lang/eval-fail-derivation-name.err.exp b/tests/functional/lang/eval-fail-derivation-name.err.exp index 017326c3490..ba5ff2d002a 100644 --- a/tests/functional/lang/eval-fail-derivation-name.err.exp +++ b/tests/functional/lang/eval-fail-derivation-name.err.exp @@ -1,20 +1,20 @@ error: … while evaluating the attribute 'outPath' - at ::: + at «nix-internal»/derivation-internal.nix::: | value = commonAttrs // { | outPath = builtins.getAttr outputName strict; | ^ | drvPath = strict.drvPath; … while calling the 'getAttr' builtin - at ::: + at «nix-internal»/derivation-internal.nix::: | value = commonAttrs // { | outPath = builtins.getAttr outputName strict; | ^ | drvPath = strict.drvPath; … while calling the 'derivationStrict' builtin - at ::: + at «nix-internal»/derivation-internal.nix::: | | strict = derivationStrict drvAttrs; | ^ diff --git a/tests/functional/lang/eval-fail-fromTOML-overflow.err.exp b/tests/functional/lang/eval-fail-fromTOML-overflow.err.exp new file mode 100644 index 00000000000..14b0e31c18a --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-overflow.err.exp @@ -0,0 +1,13 @@ +error: + … while calling the 'fromTOML' builtin + at /pwd/lang/eval-fail-fromTOML-overflow.nix:1:1: + 1| builtins.fromTOML ''attr = 9223372036854775808'' + | ^ + 2| + + error: while parsing TOML: [error] toml::parse_dec_integer: too large integer: current max digits = 2^63 + --> fromTOML + | + 1 | attr = 9223372036854775808 + | ^-- must be < 2^63 + diff --git a/tests/functional/lang/eval-fail-fromTOML-overflow.nix b/tests/functional/lang/eval-fail-fromTOML-overflow.nix new file mode 100644 index 00000000000..17f0448b3df --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-overflow.nix @@ -0,0 +1 @@ +builtins.fromTOML ''attr = 9223372036854775808'' diff --git a/tests/functional/lang/eval-fail-fromTOML-underflow.err.exp b/tests/functional/lang/eval-fail-fromTOML-underflow.err.exp new file mode 100644 index 00000000000..28f1079dc5a --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-underflow.err.exp @@ -0,0 +1,13 @@ +error: + … while calling the 'fromTOML' builtin + at /pwd/lang/eval-fail-fromTOML-underflow.nix:1:1: + 1| builtins.fromTOML ''attr = -9223372036854775809'' + | ^ + 2| + + error: while parsing TOML: [error] toml::parse_dec_integer: too large integer: current max digits = 2^63 + --> fromTOML + | + 1 | attr = -9223372036854775809 + | ^-- must be < 2^63 + diff --git a/tests/functional/lang/eval-fail-fromTOML-underflow.nix b/tests/functional/lang/eval-fail-fromTOML-underflow.nix new file mode 100644 index 00000000000..923fdf3545f --- /dev/null +++ b/tests/functional/lang/eval-fail-fromTOML-underflow.nix @@ -0,0 +1 @@ +builtins.fromTOML ''attr = -9223372036854775809'' diff --git a/tests/functional/lang/eval-fail-hashfile-missing.err.exp b/tests/functional/lang/eval-fail-hashfile-missing.err.exp index 0d3747a6d57..901dea2b544 100644 --- a/tests/functional/lang/eval-fail-hashfile-missing.err.exp +++ b/tests/functional/lang/eval-fail-hashfile-missing.err.exp @@ -10,4 +10,4 @@ error: … while calling the 'hashFile' builtin - error: opening file '/pwd/lang/this-file-is-definitely-not-there-7392097': No such file or directory + error: path '/pwd/lang/this-file-is-definitely-not-there-7392097' does not exist diff --git a/tests/functional/lang/eval-fail-recursion.err.exp b/tests/functional/lang/eval-fail-recursion.err.exp index 8bfb4e12e47..21bf7a695bd 100644 --- a/tests/functional/lang/eval-fail-recursion.err.exp +++ b/tests/functional/lang/eval-fail-recursion.err.exp @@ -7,8 +7,8 @@ error: 3| in error: infinite recursion encountered - at /pwd/lang/eval-fail-recursion.nix:2:14: - 1| let - 2| a = { } // a; - | ^ + at /pwd/lang/eval-fail-recursion.nix:4:1: 3| in + 4| a.foo + | ^ + 5| diff --git a/tests/functional/lang/eval-fail-scope-5.err.exp b/tests/functional/lang/eval-fail-scope-5.err.exp index 6edc85f4f16..557054b5354 100644 --- a/tests/functional/lang/eval-fail-scope-5.err.exp +++ b/tests/functional/lang/eval-fail-scope-5.err.exp @@ -21,8 +21,8 @@ error: 8| x ? y, error: infinite recursion encountered - at /pwd/lang/eval-fail-scope-5.nix:8:11: - 7| { - 8| x ? y, - | ^ - 9| y ? x, + at /pwd/lang/eval-fail-scope-5.nix:13:3: + 12| + 13| body = f { }; + | ^ + 14| diff --git a/tests/functional/linux-sandbox.sh b/tests/functional/linux-sandbox.sh index abb635f1195..484ad1d2b68 100755 --- a/tests/functional/linux-sandbox.sh +++ b/tests/functional/linux-sandbox.sh @@ -19,8 +19,8 @@ if [[ ! $SHELL =~ /nix/store ]]; then skipTest "Shell is not from Nix store"; fi # An alias to automatically bind-mount the $SHELL on nix-build invocations nix-sandbox-build () { nix-build --no-out-link --sandbox-paths /nix/store "$@"; } -chmod -R u+w $TEST_ROOT/store0 || true -rm -rf $TEST_ROOT/store0 +chmod -R u+w "$TEST_ROOT"/store0 || true +rm -rf "$TEST_ROOT"/store0 export NIX_STORE_DIR=/my/store export NIX_REMOTE=$TEST_ROOT/store0 @@ -29,11 +29,11 @@ outPath=$(nix-sandbox-build dependencies.nix) [[ $outPath =~ /my/store/.*-dependencies ]] -nix path-info -r $outPath | grep input-2 +nix path-info -r "$outPath" | grep input-2 -nix store ls -R -l $outPath | grep foobar +nix store ls -R -l "$outPath" | grep foobar -nix store cat $outPath/foobar | grep FOOBAR +nix store cat "$outPath"/foobar | grep FOOBAR # Test --check without hash rewriting. nix-sandbox-build dependencies.nix --check @@ -42,9 +42,9 @@ nix-sandbox-build dependencies.nix --check nix-sandbox-build check.nix -A nondeterministic # `100 + 4` means non-determinstic, see doc/manual/source/command-ref/status-build-failure.md -expectStderr 104 nix-sandbox-build check.nix -A nondeterministic --check -K > $TEST_ROOT/log -grepQuietInverse 'error: renaming' $TEST_ROOT/log -grepQuiet 'may not be deterministic' $TEST_ROOT/log +expectStderr 104 nix-sandbox-build check.nix -A nondeterministic --check -K > "$TEST_ROOT"/log +grepQuietInverse 'error: renaming' "$TEST_ROOT"/log +grepQuiet 'may not be deterministic' "$TEST_ROOT"/log # Test that sandboxed builds cannot write to /etc easily # `100` means build failure without extra info, see doc/manual/source/command-ref/status-build-failure.md @@ -59,7 +59,7 @@ testCert () { certFile=$3 # a string that can be the path to a cert file # `100` means build failure without extra info, see doc/manual/source/command-ref/status-build-failure.md [ "$mode" == fixed-output ] && ret=1 || ret=100 - expectStderr $ret nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" | + expectStderr "$ret" nix-sandbox-build linux-sandbox-cert-test.nix --argstr mode "$mode" --option ssl-cert-file "$certFile" | grepQuiet "CERT_${expectation}_IN_SANDBOX" } @@ -68,10 +68,10 @@ cert=$TEST_ROOT/some-cert-file.pem symlinkcert=$TEST_ROOT/symlink-cert-file.pem transitivesymlinkcert=$TEST_ROOT/transitive-symlink-cert-file.pem symlinkDir=$TEST_ROOT/symlink-dir -echo -n "CERT_CONTENT" > $cert -ln -s $cert $symlinkcert -ln -s $symlinkcert $transitivesymlinkcert -ln -s $TEST_ROOT $symlinkDir +echo -n "CERT_CONTENT" > "$cert" +ln -s "$cert" "$symlinkcert" +ln -s "$symlinkcert" "$transitivesymlinkcert" +ln -s "$TEST_ROOT" "$symlinkDir" # No cert in sandbox when not a fixed-output derivation testCert missing normal "$cert" @@ -96,3 +96,9 @@ nix-sandbox-build symlink-derivation.nix -A test_sandbox_paths \ --option extra-sandbox-paths "/dir=$TEST_ROOT" \ --option extra-sandbox-paths "/symlinkDir=$symlinkDir" \ --option extra-sandbox-paths "/symlink=$symlinkcert" + +# Nonexistent sandbox paths should error early in the build process +# shellcheck disable=SC2016 +expectStderr 1 nix-sandbox-build --option extra-sandbox-paths '/does-not-exist' \ + -E 'with import '"${config_nix}"'; mkDerivation { name = "trivial"; buildCommand = "echo > $out"; }' | + grepQuiet "path '/does-not-exist' is configured as part of the \`sandbox-paths\` option, but is inaccessible" diff --git a/tests/functional/local-overlay-store/add-lower.sh b/tests/functional/local-overlay-store/add-lower.sh index 33bf20ebdd3..87cdb4f59ec 100755 --- a/tests/functional/local-overlay-store/add-lower.sh +++ b/tests/functional/local-overlay-store/add-lower.sh @@ -1,3 +1,4 @@ +# shellcheck shell=bash source common.sh source ../common/init.sh diff --git a/tests/functional/local-overlay-store/bad-uris.sh b/tests/functional/local-overlay-store/bad-uris.sh index f0c6a151c35..1b5b7fc54a4 100644 --- a/tests/functional/local-overlay-store/bad-uris.sh +++ b/tests/functional/local-overlay-store/bad-uris.sh @@ -1,3 +1,4 @@ +# shellcheck shell=bash source common.sh source ../common/init.sh @@ -5,7 +6,7 @@ requireEnvironment setupConfig setupStoreDirs -mkdir -p $TEST_ROOT/bad_test +mkdir -p "$TEST_ROOT"/bad_test badTestRoot=$TEST_ROOT/bad_test storeBadRoot="local-overlay://?root=$badTestRoot&lower-store=$storeA&upper-layer=$storeBTop" storeBadLower="local-overlay://?root=$storeBRoot&lower-store=$badTestRoot&upper-layer=$storeBTop" @@ -18,7 +19,8 @@ declare -a storesBad=( TODO_NixOS for i in "${storesBad[@]}"; do - echo $i + echo "$i" + # shellcheck disable=SC2119 execUnshare <&1 | grepQuiet 'building.*dependencies-top.drv' jq < "$TEST_ROOT/log.json" grep '{"action":"start","fields":\[".*-dependencies-top.drv","",1,1\],"id":.*,"level":3,"parent":0' "$TEST_ROOT/log.json" >&2 - (( $(grep '{"action":"msg","level":5,"msg":"executing builder .*"}' "$TEST_ROOT/log.json" | wc -l) == 5 )) + (( $(grep -c '{"action":"msg","level":5,"msg":"executing builder .*"}' "$TEST_ROOT/log.json" ) == 5 )) fi diff --git a/tests/functional/meson.build b/tests/functional/meson.build index 54e13b26d4b..6f649c8360b 100644 --- a/tests/functional/meson.build +++ b/tests/functional/meson.build @@ -174,6 +174,7 @@ suites = [ 'extra-sandbox-profile.sh', 'help.sh', 'symlinks.sh', + 'external-builders.sh', ], 'workdir' : meson.current_source_dir(), }, @@ -239,6 +240,12 @@ foreach suite : suites # Turns, e.g., `tests/functional/flakes/show.sh` into a Meson test target called # `functional-flakes-show`. name = fs.replace_suffix(script, '') + asan_options = 'abort_on_error=1:print_summary=1:detect_leaks=0' + # Otherwise ASAN dumps warnings into stderr that make some tests fail on stderr output + # comparisons. + asan_options += ':log_path=@0@'.format( + meson.current_build_dir() / 'asan-log', + ) test( name, @@ -253,6 +260,7 @@ foreach suite : suites ], suite : suite_name, env : { + 'ASAN_OPTIONS' : asan_options, '_NIX_TEST_SOURCE_DIR' : meson.current_source_dir(), '_NIX_TEST_BUILD_DIR' : meson.current_build_dir(), 'TEST_NAME' : suite_name / name, diff --git a/tests/functional/misc.sh b/tests/functional/misc.sh index b94a5fc578c..b8bbb74dddd 100755 --- a/tests/functional/misc.sh +++ b/tests/functional/misc.sh @@ -14,6 +14,7 @@ source common.sh nix-env --version | grep -F "${_NIX_TEST_CLIENT_VERSION:-$version}" nix_env=$(type -P nix-env) +# shellcheck disable=SC2123 (PATH=""; ! $nix_env --help 2>&1 ) | grepQuiet -F "The 'man' command was not found, but it is needed for 'nix-env' and some other 'nix-*' commands' help text. Perhaps you could install the 'man' command?" # Usage errors. @@ -22,12 +23,12 @@ expect 1 nix-env -q --foo 2>&1 | grep "unknown flag" # Eval Errors. eval_arg_res=$(nix-instantiate --eval -E 'let a = {} // a; in a.foo' 2>&1 || true) -echo $eval_arg_res | grep "at «string»:1:15:" -echo $eval_arg_res | grep "infinite recursion encountered" +echo "$eval_arg_res" | grep "at «string»:1:12:" +echo "$eval_arg_res" | grep "infinite recursion encountered" eval_stdin_res=$(echo 'let a = {} // a; in a.foo' | nix-instantiate --eval -E - 2>&1 || true) -echo $eval_stdin_res | grep "at «stdin»:1:15:" -echo $eval_stdin_res | grep "infinite recursion encountered" +echo "$eval_stdin_res" | grep "at «stdin»:1:12:" +echo "$eval_stdin_res" | grep "infinite recursion encountered" # Attribute path errors expectStderr 1 nix-instantiate --eval -E '{}' -A '"x' | grepQuiet "missing closing quote in selection path" @@ -40,10 +41,10 @@ expectStderr 1 nix-instantiate --eval -E '[]' -A '1' | grepQuiet "out of range" # NOTE(cole-h): behavior is different depending on the order, which is why we test an unknown option # before and after the `'{}'`! out="$(expectStderr 0 nix-instantiate --option foobar baz --expr '{}')" -[[ "$(echo "$out" | grep foobar | wc -l)" = 1 ]] +[[ "$(echo "$out" | grep -c foobar )" = 1 ]] out="$(expectStderr 0 nix-instantiate '{}' --option foobar baz --expr )" -[[ "$(echo "$out" | grep foobar | wc -l)" = 1 ]] +[[ "$(echo "$out" | grep -c foobar )" = 1 ]] if [[ $(uname) = Linux && $(uname -m) = i686 ]]; then [[ $(nix config show system) = i686-linux ]] diff --git a/tests/functional/multiple-outputs.sh b/tests/functional/multiple-outputs.sh index 35a78d152c7..f703fb02be6 100755 --- a/tests/functional/multiple-outputs.sh +++ b/tests/functional/multiple-outputs.sh @@ -6,13 +6,19 @@ TODO_NixOS clearStoreIfPossible -rm -f $TEST_ROOT/result* - -# Test whether the output names match our expectations -outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.out.outPath) -[ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a" ] -outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.dev.outPath) -[ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a-dev" ] +rm -f "$TEST_ROOT"/result* + +# Placeholder strings are opaque, so cannot do this check for floating +# content-addressing derivations. +if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + # Test whether the output names match our expectations + outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.out.outPath) + # shellcheck disable=SC2016 + [ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a" ] + outPath=$(nix-instantiate multiple-outputs.nix --eval -A nameCheck.dev.outPath) + # shellcheck disable=SC2016 + [ "$(echo "$outPath" | sed -E 's_^".*/[^-/]*-([^/]*)"$_\1_')" = "multiple-outputs-a-dev" ] +fi # Test whether read-only evaluation works when referring to the # ‘drvPath’ attribute. @@ -23,14 +29,19 @@ echo "evaluating c..." # outputs. drvPath=$(nix-instantiate multiple-outputs.nix -A c) #[ "$drvPath" = "$drvPath2" ] -grepQuiet 'multiple-outputs-a.drv",\["first","second"\]' $drvPath -grepQuiet 'multiple-outputs-b.drv",\["out"\]' $drvPath +grepQuiet 'multiple-outputs-a.drv",\["first","second"\]' "$drvPath" +grepQuiet 'multiple-outputs-b.drv",\["out"\]' "$drvPath" # While we're at it, test the ‘unsafeDiscardOutputDependency’ primop. outPath=$(nix-build multiple-outputs.nix -A d --no-out-link) -drvPath=$(cat $outPath/drv) -outPath=$(nix-store -q $drvPath) -(! [ -e "$outPath" ]) +drvPath=$(cat "$outPath"/drv) +if [[ -n "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + expectStderr 1 nix-store -q "$drvPath" | grepQuiet "Cannot use output path of floating content-addressing derivation until we know what it is (e.g. by building it)" +else + outPath=$(nix-store -q "$drvPath") + # shellcheck disable=SC2233 + (! [ -e "$outPath" ]) +fi # Do a build of something that depends on a derivation with multiple # outputs. @@ -40,34 +51,39 @@ echo "output path is $outPath" [ "$(cat "$outPath/file")" = "success" ] # Test nix-build on a derivation with multiple outputs. -outPath1=$(nix-build multiple-outputs.nix -A a -o $TEST_ROOT/result) -[ -e $TEST_ROOT/result-first ] -(! [ -e $TEST_ROOT/result-second ]) -nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result -[ "$(cat $TEST_ROOT/result-first/file)" = "first" ] -[ "$(cat $TEST_ROOT/result-second/file)" = "second" ] -[ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ] -hash1=$(nix-store -q --hash $TEST_ROOT/result-second) - -outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a) --no-out-link) -[[ $outPath1 = $outPath2 ]] - -outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.first) --no-out-link) -[[ $outPath1 = $outPath2 ]] - -outPath2=$(nix-build $(nix-instantiate multiple-outputs.nix -A a.second) --no-out-link) -[[ $(cat $outPath2/file) = second ]] - +outPath1=$(nix-build multiple-outputs.nix -A a -o "$TEST_ROOT"/result) +[ -e "$TEST_ROOT"/result-first ] +# shellcheck disable=SC2235 +(! [ -e "$TEST_ROOT"/result-second ]) +nix-build multiple-outputs.nix -A a.all -o "$TEST_ROOT"/result +[ "$(cat "$TEST_ROOT"/result-first/file)" = "first" ] +[ "$(cat "$TEST_ROOT"/result-second/file)" = "second" ] +[ "$(cat "$TEST_ROOT"/result-second/link/file)" = "first" ] +hash1=$(nix-store -q --hash "$TEST_ROOT"/result-second) + +outPath2=$(nix-build "$(nix-instantiate multiple-outputs.nix -A a)" --no-out-link) +[[ $outPath1 = "$outPath2" ]] + +outPath2=$(nix-build "$(nix-instantiate multiple-outputs.nix -A a.first)" --no-out-link) +[[ $outPath1 = "$outPath2" ]] + +outPath2=$(nix-build "$(nix-instantiate multiple-outputs.nix -A a.second)" --no-out-link) +[[ $(cat "$outPath2"/file) = second ]] + +# FIXME: Fixing this shellcheck causes the test to fail. +# shellcheck disable=SC2046 [[ $(nix-build $(nix-instantiate multiple-outputs.nix -A a.all) --no-out-link | wc -l) -eq 2 ]] -# Delete one of the outputs and rebuild it. This will cause a hash -# rewrite. -env -u NIX_REMOTE nix store delete $TEST_ROOT/result-second --ignore-liveness -nix-build multiple-outputs.nix -A a.all -o $TEST_ROOT/result -[ "$(cat $TEST_ROOT/result-second/file)" = "second" ] -[ "$(cat $TEST_ROOT/result-second/link/file)" = "first" ] -hash2=$(nix-store -q --hash $TEST_ROOT/result-second) -[ "$hash1" = "$hash2" ] +if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + # Delete one of the outputs and rebuild it. This will cause a hash + # rewrite. + env -u NIX_REMOTE nix store delete "$TEST_ROOT"/result-second --ignore-liveness + nix-build multiple-outputs.nix -A a.all -o "$TEST_ROOT"/result + [ "$(cat "$TEST_ROOT"/result-second/file)" = "second" ] + [ "$(cat "$TEST_ROOT"/result-second/link/file)" = "first" ] + hash2=$(nix-store -q --hash "$TEST_ROOT"/result-second) + [ "$hash1" = "$hash2" ] +fi # Make sure that nix-build works on derivations with multiple outputs. echo "building a.first..." @@ -82,11 +98,15 @@ fi # Do a GC. This should leave an empty store. echo "collecting garbage..." -rm $TEST_ROOT/result* +rm "$TEST_ROOT"/result* nix-store --gc --keep-derivations --keep-outputs nix-store --gc --print-roots -rm -rf $NIX_STORE_DIR/.links -rmdir $NIX_STORE_DIR - -expect 1 nix build -f multiple-outputs.nix invalid-output-name-1 2>&1 | grep 'contains illegal character' -expect 1 nix build -f multiple-outputs.nix invalid-output-name-2 2>&1 | grep 'contains illegal character' +rm -rf "$NIX_STORE_DIR"/.links +rmdir "$NIX_STORE_DIR" + +# TODO inspect why this doesn't work with floating content-addressing +# derivations. +if [[ -z "${NIX_TESTS_CA_BY_DEFAULT:-}" ]]; then + expect 1 nix build -f multiple-outputs.nix invalid-output-name-1 2>&1 | grep 'contains illegal character' + expect 1 nix build -f multiple-outputs.nix invalid-output-name-2 2>&1 | grep 'contains illegal character' +fi diff --git a/tests/functional/nested-sandboxing.sh b/tests/functional/nested-sandboxing.sh index 4d4cf125e83..8788c7d902c 100755 --- a/tests/functional/nested-sandboxing.sh +++ b/tests/functional/nested-sandboxing.sh @@ -11,7 +11,7 @@ requiresUnprivilegedUserNamespaces start="$TEST_ROOT/start" mkdir -p "$start" -cp -r common common.sh ${config_nix} ./nested-sandboxing "$start" +cp -r common common.sh "${config_nix}" ./nested-sandboxing "$start" cp "${_NIX_TEST_BUILD_DIR}/common/subst-vars.sh" "$start/common" # N.B. redefine _NIX_TEST_SOURCE_DIR="$start" @@ -20,6 +20,7 @@ cd "$start" source ./nested-sandboxing/command.sh +# shellcheck disable=SC2016 expectStderr 100 runNixBuild badStoreUrl 2 | grepQuiet '`sandbox-build-dir` must not contain' runNixBuild goodStoreUrl 5 diff --git a/tests/functional/nested-sandboxing/command.sh b/tests/functional/nested-sandboxing/command.sh index 7c04e82f5f6..c01133d9328 100644 --- a/tests/functional/nested-sandboxing/command.sh +++ b/tests/functional/nested-sandboxing/command.sh @@ -1,17 +1,20 @@ +# shellcheck shell=bash set -eu -o pipefail -export NIX_BIN_DIR=$(dirname $(type -p nix)) +NIX_BIN_DIR=$(dirname "$(type -p nix)") +export NIX_BIN_DIR # TODO Get Nix and its closure more flexibly -export EXTRA_SANDBOX="/nix/store $(dirname $NIX_BIN_DIR)" +EXTRA_SANDBOX="/nix/store $(dirname "$NIX_BIN_DIR")" +export EXTRA_SANDBOX badStoreUrl () { local altitude=$1 - echo $TEST_ROOT/store-$altitude + echo "$TEST_ROOT"/store-"$altitude" } goodStoreUrl () { local altitude=$1 - echo $("badStoreUrl" "$altitude")?store=/foo-$altitude + echo "$("badStoreUrl" "$altitude")"?store=/foo-"$altitude" } # The non-standard sandbox-build-dir helps ensure that we get the same behavior diff --git a/tests/functional/nix-build.sh b/tests/functional/nix-build.sh index 091e429e02d..33973c62852 100755 --- a/tests/functional/nix-build.sh +++ b/tests/functional/nix-build.sh @@ -6,30 +6,30 @@ TODO_NixOS clearStoreIfPossible -outPath=$(nix-build dependencies.nix -o $TEST_ROOT/result) -test "$(cat $TEST_ROOT/result/foobar)" = FOOBAR +outPath=$(nix-build dependencies.nix -o "$TEST_ROOT"/result) +test "$(cat "$TEST_ROOT"/result/foobar)" = FOOBAR # The result should be retained by a GC. echo A -target=$(readLink $TEST_ROOT/result) +target=$(readLink "$TEST_ROOT"/result) echo B -echo target is $target +echo target is "$target" nix-store --gc -test -e $target/foobar +test -e "$target"/foobar # But now it should be gone. -rm $TEST_ROOT/result +rm "$TEST_ROOT"/result nix-store --gc -if test -e $target/foobar; then false; fi +if test -e "$target"/foobar; then false; fi -outPath2=$(nix-build $(nix-instantiate dependencies.nix) --no-out-link) -[[ $outPath = $outPath2 ]] +outPath2=$(nix-build "$(nix-instantiate dependencies.nix)" --no-out-link) +[[ $outPath = "$outPath2" ]] -outPath2=$(nix-build $(nix-instantiate dependencies.nix)!out --no-out-link) -[[ $outPath = $outPath2 ]] +outPath2=$(nix-build "$(nix-instantiate dependencies.nix)"!out --no-out-link) +[[ $outPath = "$outPath2" ]] -outPath2=$(nix-store -r $(nix-instantiate --add-root $TEST_ROOT/indirect dependencies.nix)!out) -[[ $outPath = $outPath2 ]] +outPath2=$(nix-store -r "$(nix-instantiate --add-root "$TEST_ROOT"/indirect dependencies.nix)"!out) +[[ $outPath = "$outPath2" ]] # The order of the paths on stdout must correspond to the -A options # https://github.com/NixOS/nix/issues/4197 @@ -39,9 +39,11 @@ input1="$(nix-build nix-build-examples.nix -A input1 --no-out-link)" input2="$(nix-build nix-build-examples.nix -A input2 --no-out-link)" body="$(nix-build nix-build-examples.nix -A body --no-out-link)" +# shellcheck disable=SC2046,SC2005 outPathsA="$(echo $(nix-build nix-build-examples.nix -A input0 -A input1 -A input2 -A body --no-out-link))" [[ "$outPathsA" = "$input0 $input1 $input2 $body" ]] # test a different ordering to make sure it fails, not just in 23 out of 24 permutations +# shellcheck disable=SC2046,SC2005 outPathsB="$(echo $(nix-build nix-build-examples.nix -A body -A input1 -A input2 -A input0 --no-out-link))" [[ "$outPathsB" = "$body $input1 $input2 $input0" ]] diff --git a/tests/functional/nix-channel.sh b/tests/functional/nix-channel.sh index d0b772850dd..f23d4bbded0 100755 --- a/tests/functional/nix-channel.sh +++ b/tests/functional/nix-channel.sh @@ -4,7 +4,7 @@ source common.sh clearProfiles -rm -f $TEST_HOME/.nix-channels $TEST_HOME/.nix-profile +rm -f "$TEST_HOME"/.nix-channels "$TEST_HOME"/.nix-profile # Test add/list/remove. nix-channel --add http://foo/bar xyzzy @@ -12,8 +12,8 @@ nix-channel --list | grepQuiet http://foo/bar nix-channel --remove xyzzy [[ $(nix-channel --list-generations | wc -l) == 1 ]] -[ -e $TEST_HOME/.nix-channels ] -[ "$(cat $TEST_HOME/.nix-channels)" = '' ] +[ -e "$TEST_HOME"/.nix-channels ] +[ "$(cat "$TEST_HOME"/.nix-channels)" = '' ] # Test the XDG Base Directories support @@ -25,47 +25,47 @@ nix-channel --remove xyzzy unset NIX_CONFIG -[ -e $TEST_HOME/.local/state/nix/channels ] -[ "$(cat $TEST_HOME/.local/state/nix/channels)" = '' ] +[ -e "$TEST_HOME"/.local/state/nix/channels ] +[ "$(cat "$TEST_HOME"/.local/state/nix/channels)" = '' ] # Create a channel. -rm -rf $TEST_ROOT/foo -mkdir -p $TEST_ROOT/foo +rm -rf "$TEST_ROOT"/foo +mkdir -p "$TEST_ROOT"/foo drvPath=$(nix-instantiate dependencies.nix) -nix copy --to file://$TEST_ROOT/foo?compression="bzip2" $(nix-store -r "$drvPath") -rm -rf $TEST_ROOT/nixexprs -mkdir -p $TEST_ROOT/nixexprs -cp "${config_nix}" dependencies.nix dependencies.builder*.sh $TEST_ROOT/nixexprs/ -ln -s dependencies.nix $TEST_ROOT/nixexprs/default.nix -(cd $TEST_ROOT && tar cvf - nixexprs) | bzip2 > $TEST_ROOT/foo/nixexprs.tar.bz2 +nix copy --to file://"$TEST_ROOT"/foo?compression="bzip2" "$(nix-store -r "$drvPath")" +rm -rf "$TEST_ROOT"/nixexprs +mkdir -p "$TEST_ROOT"/nixexprs +cp "${config_nix}" dependencies.nix dependencies.builder*.sh "$TEST_ROOT"/nixexprs/ +ln -s dependencies.nix "$TEST_ROOT"/nixexprs/default.nix +(cd "$TEST_ROOT" && tar cvf - nixexprs) | bzip2 > "$TEST_ROOT"/foo/nixexprs.tar.bz2 # Test the update action. -nix-channel --add file://$TEST_ROOT/foo +nix-channel --add file://"$TEST_ROOT"/foo nix-channel --update [[ $(nix-channel --list-generations | wc -l) == 2 ]] # Do a query. -nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml -grepQuiet 'meta.*description.*Random test package' $TEST_ROOT/meta.xml -grepQuiet 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml +nix-env -qa \* --meta --xml --out-path > "$TEST_ROOT"/meta.xml +grepQuiet 'meta.*description.*Random test package' "$TEST_ROOT"/meta.xml +grepQuiet 'item.*attrPath="foo".*name="dependencies-top"' "$TEST_ROOT"/meta.xml # Do an install. nix-env -i dependencies-top -[ -e $TEST_HOME/.nix-profile/foobar ] +[ -e "$TEST_HOME"/.nix-profile/foobar ] # Test updating from a tarball -nix-channel --add file://$TEST_ROOT/foo/nixexprs.tar.bz2 bar +nix-channel --add file://"$TEST_ROOT"/foo/nixexprs.tar.bz2 bar nix-channel --update # Do a query. -nix-env -qa \* --meta --xml --out-path > $TEST_ROOT/meta.xml -grepQuiet 'meta.*description.*Random test package' $TEST_ROOT/meta.xml -grepQuiet 'item.*attrPath="bar".*name="dependencies-top"' $TEST_ROOT/meta.xml -grepQuiet 'item.*attrPath="foo".*name="dependencies-top"' $TEST_ROOT/meta.xml +nix-env -qa \* --meta --xml --out-path > "$TEST_ROOT"/meta.xml +grepQuiet 'meta.*description.*Random test package' "$TEST_ROOT"/meta.xml +grepQuiet 'item.*attrPath="bar".*name="dependencies-top"' "$TEST_ROOT"/meta.xml +grepQuiet 'item.*attrPath="foo".*name="dependencies-top"' "$TEST_ROOT"/meta.xml # Do an install. nix-env -i dependencies-top -[ -e $TEST_HOME/.nix-profile/foobar ] +[ -e "$TEST_HOME"/.nix-profile/foobar ] # Test evaluation through a channel symlink (#9882). drvPath=$(nix-instantiate '') @@ -73,9 +73,9 @@ drvPath=$(nix-instantiate '') # Add a test for the special case behaviour of 'nixpkgs' in the # channels for root (see EvalSettings::getDefaultNixPath()). if ! isTestOnNixOS; then - nix-channel --add file://$TEST_ROOT/foo nixpkgs + nix-channel --add file://"$TEST_ROOT"/foo nixpkgs nix-channel --update - mv $TEST_HOME/.local/state/nix/profiles $TEST_ROOT/var/nix/profiles/per-user/root + mv "$TEST_HOME"/.local/state/nix/profiles "$TEST_ROOT"/var/nix/profiles/per-user/root drvPath2=$(nix-instantiate '') [[ "$drvPath" = "$drvPath2" ]] fi diff --git a/tests/functional/nix-collect-garbage-d.sh b/tests/functional/nix-collect-garbage-d.sh index 119efe62925..44de90711a4 100755 --- a/tests/functional/nix-collect-garbage-d.sh +++ b/tests/functional/nix-collect-garbage-d.sh @@ -29,7 +29,7 @@ testCollectGarbageD # Run the same test, but forcing the profiles an arbitrary location. rm ~/.nix-profile -ln -s $TEST_ROOT/blah ~/.nix-profile +ln -s "$TEST_ROOT"/blah ~/.nix-profile testCollectGarbageD # Run the same test, but forcing the profiles at their legacy location under diff --git a/tests/functional/nix-copy-ssh-common.sh b/tests/functional/nix-copy-ssh-common.sh index 5eea9612d0d..8154585af5c 100644 --- a/tests/functional/nix-copy-ssh-common.sh +++ b/tests/functional/nix-copy-ssh-common.sh @@ -1,3 +1,4 @@ +# shellcheck shell=bash proto=$1 shift (( $# == 0 )) @@ -7,7 +8,7 @@ TODO_NixOS clearStore clearCache -mkdir -p $TEST_ROOT/stores +mkdir -p "$TEST_ROOT"/stores # Create path to copy back and forth outPath=$(nix-build --no-out-link dependencies.nix) @@ -37,17 +38,17 @@ if [[ "$proto" == "ssh-ng" ]]; then args+=(--no-check-sigs) fi -[ ! -f ${remoteRoot}${outPath}/foobar ] -nix copy "${args[@]}" --to "$remoteStore" $outPath -[ -f ${remoteRoot}${outPath}/foobar ] +[ ! -f "${remoteRoot}""${outPath}"/foobar ] +nix copy "${args[@]}" --to "$remoteStore" "$outPath" +[ -f "${remoteRoot}""${outPath}"/foobar ] # Copy back from store clearStore -[ ! -f $outPath/foobar ] -nix copy --no-check-sigs --from "$remoteStore" $outPath -[ -f $outPath/foobar ] +[ ! -f "$outPath"/foobar ] +nix copy --no-check-sigs --from "$remoteStore" "$outPath" +[ -f "$outPath"/foobar ] # Check --substitute-on-destination, avoid corrupted store diff --git a/tests/functional/nix-copy-ssh-ng.sh b/tests/functional/nix-copy-ssh-ng.sh index 41958c2c3c3..f74f3bb86c3 100755 --- a/tests/functional/nix-copy-ssh-ng.sh +++ b/tests/functional/nix-copy-ssh-ng.sh @@ -14,5 +14,5 @@ outPath=$(nix-build --no-out-link dependencies.nix) nix store info --store "$remoteStore" # Regression test for https://github.com/NixOS/nix/issues/6253 -nix copy --to "$remoteStore" $outPath --no-check-sigs & -nix copy --to "$remoteStore" $outPath --no-check-sigs +nix copy --to "$remoteStore" "$outPath" --no-check-sigs & +nix copy --to "$remoteStore" "$outPath" --no-check-sigs diff --git a/tests/functional/nix-profile.sh b/tests/functional/nix-profile.sh index b1cfef6b0b2..66fec6c1fe8 100755 --- a/tests/functional/nix-profile.sh +++ b/tests/functional/nix-profile.sh @@ -12,9 +12,10 @@ restartDaemon # Make a flake. flake1Dir=$TEST_ROOT/flake1 -mkdir -p $flake1Dir +mkdir -p "$flake1Dir" -cat > $flake1Dir/flake.nix < "$flake1Dir"/flake.nix < $flake1Dir/flake.nix < $flake1Dir/who -printf 1.0 > $flake1Dir/version -printf false > $flake1Dir/ca.nix +printf World > "$flake1Dir"/who +printf 1.0 > "$flake1Dir"/version +printf false > "$flake1Dir"/ca.nix -cp "${config_nix}" $flake1Dir/ +cp "${config_nix}" "$flake1Dir"/ # Test upgrading from nix-env. nix-env -f ./user-envs.nix -i foo-1.0 nix profile list | grep -A2 'Name:.*foo' | grep 'Store paths:.*foo-1.0' -nix profile add $flake1Dir -L +nix profile add "$flake1Dir" -L nix profile list | grep -A4 'Name:.*flake1' | grep 'Locked flake URL:.*narHash' -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] -[ -e $TEST_HOME/.nix-profile/share/man ] -(! [ -e $TEST_HOME/.nix-profile/include ]) +[[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello World" ]] +[ -e "$TEST_HOME"/.nix-profile/share/man ] +# shellcheck disable=SC2235 +(! [ -e "$TEST_HOME"/.nix-profile/include ]) nix profile history -nix profile history | grep "packages.$system.default: ∅ -> 1.0" -nix profile diff-closures | grep 'env-manifest.nix: ε → ∅' +nix profile history | grep "packages.$system.default: 1.0, 1.0-man added" +nix profile diff-closures | grep 'env-manifest.nix: (no version) removed' # Test XDG Base Directories support export NIX_CONFIG="use-xdg-base-directories = true" nix profile remove flake1 2>&1 | grep 'removed 1 packages' -nix profile add $flake1Dir -[[ $($TEST_HOME/.local/state/nix/profile/bin/hello) = "Hello World" ]] +nix profile add "$flake1Dir" +[[ $("$TEST_HOME"/.local/state/nix/profile/bin/hello) = "Hello World" ]] unset NIX_CONFIG # Test conflicting package add. -nix profile add $flake1Dir 2>&1 | grep "warning: 'flake1' is already added" +nix profile add "$flake1Dir" 2>&1 | grep "warning: 'flake1' is already added" # Test upgrading a package. -printf NixOS > $flake1Dir/who -printf 2.0 > $flake1Dir/version +printf NixOS > "$flake1Dir"/who +printf 2.0 > "$flake1Dir"/version nix profile upgrade flake1 -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello NixOS" ]] +[[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello NixOS" ]] nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 2.0, 2.0-man" # Test upgrading package using regular expression. -printf 2.1 > $flake1Dir/version +printf 2.1 > "$flake1Dir"/version nix profile upgrade --regex '.*' -[[ $(readlink $TEST_HOME/.nix-profile/bin/hello) =~ .*-profile-test-2\.1/bin/hello ]] +[[ $(readlink "$TEST_HOME"/.nix-profile/bin/hello) =~ .*-profile-test-2\.1/bin/hello ]] nix profile rollback # Test upgrading all packages -printf 2.2 > $flake1Dir/version +printf 2.2 > "$flake1Dir"/version nix profile upgrade --all -[[ $(readlink $TEST_HOME/.nix-profile/bin/hello) =~ .*-profile-test-2\.2/bin/hello ]] +[[ $(readlink "$TEST_HOME"/.nix-profile/bin/hello) =~ .*-profile-test-2\.2/bin/hello ]] nix profile rollback -printf 1.0 > $flake1Dir/version +printf 1.0 > "$flake1Dir"/version # Test --all exclusivity. assertStderr nix --offline profile upgrade --all foo << EOF error: --all cannot be used with package names or regular expressions. + Try 'nix --help' for more information. EOF @@ -117,98 +120,101 @@ nix profile rollback nix profile diff-closures # Test rollback. -printf World > $flake1Dir/who +printf World > "$flake1Dir"/who nix profile upgrade flake1 -printf NixOS > $flake1Dir/who +printf NixOS > "$flake1Dir"/who nix profile upgrade flake1 nix profile rollback -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] +[[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello World" ]] # Test uninstall. -[ -e $TEST_HOME/.nix-profile/bin/foo ] +[ -e "$TEST_HOME"/.nix-profile/bin/foo ] +# shellcheck disable=SC2235 nix profile remove foo 2>&1 | grep 'removed 1 packages' -(! [ -e $TEST_HOME/.nix-profile/bin/foo ]) -nix profile history | grep 'foo: 1.0 -> ∅' +[[ ! -e "$TEST_HOME"/.nix-profile/bin/foo ]] +nix profile history | grep 'foo: 1.0 removed' nix profile diff-closures | grep 'Version 3 -> 4' # Test installing a non-flake package. nix profile add --file ./simple.nix '' -[[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]] +[[ $(cat "$TEST_HOME"/.nix-profile/hello) = "Hello World!" ]] nix profile remove simple 2>&1 | grep 'removed 1 packages' -nix profile add $(nix-build --no-out-link ./simple.nix) -[[ $(cat $TEST_HOME/.nix-profile/hello) = "Hello World!" ]] +nix profile add "$(nix-build --no-out-link ./simple.nix)" +[[ $(cat "$TEST_HOME"/.nix-profile/hello) = "Hello World!" ]] # Test packages with same name from different sources -mkdir $TEST_ROOT/simple-too -cp ./simple.nix "${config_nix}" simple.builder.sh $TEST_ROOT/simple-too -nix profile add --file $TEST_ROOT/simple-too/simple.nix '' +mkdir "$TEST_ROOT"/simple-too +cp ./simple.nix "${config_nix}" simple.builder.sh "$TEST_ROOT"/simple-too +nix profile add --file "$TEST_ROOT"/simple-too/simple.nix '' nix profile list | grep -A4 'Name:.*simple' | grep 'Name:.*simple-1' nix profile remove simple 2>&1 | grep 'removed 1 packages' nix profile remove simple-1 2>&1 | grep 'removed 1 packages' # Test wipe-history. nix profile wipe-history -[[ $(nix profile history | grep Version | wc -l) -eq 1 ]] +[[ $(nix profile history | grep -c Version) -eq 1 ]] # Test upgrade to CA package. -printf true > $flake1Dir/ca.nix -printf 3.0 > $flake1Dir/version +printf true > "$flake1Dir"/ca.nix +printf 3.0 > "$flake1Dir"/version nix profile upgrade flake1 nix profile history | grep "packages.$system.default: 1.0, 1.0-man -> 3.0, 3.0-man" # Test new install of CA package. nix profile remove flake1 2>&1 | grep 'removed 1 packages' -printf 4.0 > $flake1Dir/version -printf Utrecht > $flake1Dir/who -nix profile add $flake1Dir -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]] -[[ $(nix path-info --json $(realpath $TEST_HOME/.nix-profile/bin/hello) | jq -r .[].ca) =~ fixed:r:sha256: ]] +printf 4.0 > "$flake1Dir"/version +printf Utrecht > "$flake1Dir"/who +nix profile add "$flake1Dir" +[[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello Utrecht" ]] +[[ $(nix path-info --json "$(realpath "$TEST_HOME"/.nix-profile/bin/hello)" | jq -r .[].ca) =~ fixed:r:sha256: ]] # Override the outputs. nix profile remove simple flake1 nix profile add "$flake1Dir^*" -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Utrecht" ]] -[ -e $TEST_HOME/.nix-profile/share/man ] -[ -e $TEST_HOME/.nix-profile/include ] +[[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello Utrecht" ]] +[ -e "$TEST_HOME"/.nix-profile/share/man ] +[ -e "$TEST_HOME"/.nix-profile/include ] -printf Nix > $flake1Dir/who +printf Nix > "$flake1Dir"/who nix profile list nix profile upgrade flake1 -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello Nix" ]] -[ -e $TEST_HOME/.nix-profile/share/man ] -[ -e $TEST_HOME/.nix-profile/include ] +[[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello Nix" ]] +[ -e "$TEST_HOME"/.nix-profile/share/man ] +[ -e "$TEST_HOME"/.nix-profile/include ] nix profile remove flake1 2>&1 | grep 'removed 1 packages' nix profile add "$flake1Dir^man" -(! [ -e $TEST_HOME/.nix-profile/bin/hello ]) -[ -e $TEST_HOME/.nix-profile/share/man ] -(! [ -e $TEST_HOME/.nix-profile/include ]) +# shellcheck disable=SC2235 +(! [ -e "$TEST_HOME"/.nix-profile/bin/hello ]) +[ -e "$TEST_HOME"/.nix-profile/share/man ] +# shellcheck disable=SC2235 +(! [ -e "$TEST_HOME"/.nix-profile/include ]) # test priority nix profile remove flake1 2>&1 | grep 'removed 1 packages' # Make another flake. flake2Dir=$TEST_ROOT/flake2 -printf World > $flake1Dir/who -cp -r $flake1Dir $flake2Dir -printf World2 > $flake2Dir/who +printf World > "$flake1Dir"/who +cp -r "$flake1Dir" "$flake2Dir" +printf World2 > "$flake2Dir"/who -nix profile add $flake1Dir -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] -expect 1 nix profile add $flake2Dir +nix profile add "$flake1Dir" +[[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello World" ]] +expect 1 nix profile add "$flake2Dir" diff -u <( - nix --offline profile install $flake2Dir 2>&1 1> /dev/null \ + nix --offline profile install "$flake2Dir" 2>&1 1> /dev/null \ | grep -vE "^warning: " \ | grep -vE "^error \(ignored\): " \ || true ) <(cat << EOF error: An existing package already provides the following file: - $(nix build --no-link --print-out-paths ${flake1Dir}"#default.out")/bin/hello + $(nix build --no-link --print-out-paths "${flake1Dir}""#default.out")/bin/hello This is the conflicting file from the new package: - $(nix build --no-link --print-out-paths ${flake2Dir}"#default.out")/bin/hello + $(nix build --no-link --print-out-paths "${flake2Dir}""#default.out")/bin/hello To remove the existing package: @@ -225,11 +231,11 @@ error: An existing package already provides the following file: nix profile add path:${flake2Dir}#packages.${system}.default --priority 6 EOF ) -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] -nix profile add $flake2Dir --priority 100 -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] -nix profile add $flake2Dir --priority 0 -[[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World2" ]] +[[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello World" ]] +nix profile add "$flake2Dir" --priority 100 +[[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello World" ]] +nix profile add "$flake2Dir" --priority 0 +[[ $("$TEST_HOME"/.nix-profile/bin/hello) = "Hello World2" ]] # nix profile add $flake1Dir --priority 100 # [[ $($TEST_HOME/.nix-profile/bin/hello) = "Hello World" ]] @@ -237,14 +243,15 @@ nix profile add $flake2Dir --priority 0 # flake references. # Regression test for https://github.com/NixOS/nix/issues/8284 clearProfiles -nix profile add $(nix build $flake1Dir --no-link --print-out-paths) +# shellcheck disable=SC2046 +nix profile add $(nix build "$flake1Dir" --no-link --print-out-paths) expect 1 nix profile add --impure --expr "(builtins.getFlake ''$flake2Dir'').packages.$system.default" # Test upgrading from profile version 2. clearProfiles -mkdir -p $TEST_ROOT/import-profile -outPath=$(nix build --no-link --print-out-paths $flake1Dir/flake.nix^out) -printf '{ "version": 2, "elements": [ { "active": true, "attrPath": "legacyPackages.x86_64-linux.hello", "originalUrl": "flake:nixpkgs", "outputs": null, "priority": 5, "storePaths": [ "%s" ], "url": "github:NixOS/nixpkgs/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } ] }' "$outPath" > $TEST_ROOT/import-profile/manifest.json -nix build --profile $TEST_HOME/.nix-profile $(nix store add-path $TEST_ROOT/import-profile) --no-link +mkdir -p "$TEST_ROOT"/import-profile +outPath=$(nix build --no-link --print-out-paths "$flake1Dir"/flake.nix^out) +printf '{ "version": 2, "elements": [ { "active": true, "attrPath": "legacyPackages.x86_64-linux.hello", "originalUrl": "flake:nixpkgs", "outputs": null, "priority": 5, "storePaths": [ "%s" ], "url": "github:NixOS/nixpkgs/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" } ] }' "$outPath" > "$TEST_ROOT"/import-profile/manifest.json +nix build --profile "$TEST_HOME"/.nix-profile "$(nix store add-path "$TEST_ROOT"/import-profile)" --no-link nix profile list | grep -A4 'Name:.*hello' | grep "Store paths:.*$outPath" nix profile remove hello 2>&1 | grep 'removed 1 packages, kept 0 packages' diff --git a/tests/functional/nix-shell.sh b/tests/functional/nix-shell.sh index bc49333b505..cf650e2c36c 100755 --- a/tests/functional/nix-shell.sh +++ b/tests/functional/nix-shell.sh @@ -16,16 +16,19 @@ export NIX_PATH=nixpkgs="$shellDotNix" export IMPURE_VAR=foo export SELECTED_IMPURE_VAR=baz +# shellcheck disable=SC2016 output=$(nix-shell --pure "$shellDotNix" -A shellDrv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') [ "$output" = " - foo - bar - true" ] +# shellcheck disable=SC2016 output=$(nix-shell --pure "$shellDotNix" -A shellDrv --option nix-shell-always-looks-for-shell-nix false --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') [ "$output" = " - foo - bar - true" ] # Test --keep +# shellcheck disable=SC2016 output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR "$shellDotNix" -A shellDrv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $SELECTED_IMPURE_VAR"') @@ -34,6 +37,7 @@ output=$(nix-shell --pure --keep SELECTED_IMPURE_VAR "$shellDotNix" -A shellDrv # test NIX_BUILD_TOP testTmpDir=$(pwd)/nix-shell mkdir -p "$testTmpDir" +# shellcheck disable=SC2016 output=$(TMPDIR="$testTmpDir" nix-shell --pure "$shellDotNix" -A shellDrv --run 'echo $NIX_BUILD_TOP') [[ "$output" =~ ${testTmpDir}.* ]] || { echo "expected $output =~ ${testTmpDir}.*" >&2 @@ -41,105 +45,111 @@ output=$(TMPDIR="$testTmpDir" nix-shell --pure "$shellDotNix" -A shellDrv --run } # Test nix-shell on a .drv -[[ $(nix-shell --pure $(nix-instantiate "$shellDotNix" -A shellDrv) --run \ +# shellcheck disable=SC2016 +[[ $(nix-shell --pure "$(nix-instantiate "$shellDotNix" -A shellDrv)" --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] - -[[ $(nix-shell --pure $(nix-instantiate "$shellDotNix" -A shellDrv) --run \ +# shellcheck disable=SC2016 +[[ $(nix-shell --pure "$(nix-instantiate "$shellDotNix" -A shellDrv)" --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX - $TEST_inNixShell"') = " - foo - bar - false" ]] # Test nix-shell on a .drv symlink # Legacy: absolute path and .drv extension required -nix-instantiate "$shellDotNix" -A shellDrv --add-root $TEST_ROOT/shell.drv -[[ $(nix-shell --pure $TEST_ROOT/shell.drv --run \ +nix-instantiate "$shellDotNix" -A shellDrv --add-root "$TEST_ROOT"/shell.drv +# shellcheck disable=SC2016 +[[ $(nix-shell --pure "$TEST_ROOT"/shell.drv --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # New behaviour: just needs to resolve to a derivation in the store -nix-instantiate "$shellDotNix" -A shellDrv --add-root $TEST_ROOT/shell -[[ $(nix-shell --pure $TEST_ROOT/shell --run \ +nix-instantiate "$shellDotNix" -A shellDrv --add-root "$TEST_ROOT"/shell +# shellcheck disable=SC2016 +[[ $(nix-shell --pure "$TEST_ROOT"/shell --run \ 'echo "$IMPURE_VAR - $VAR_FROM_STDENV_SETUP - $VAR_FROM_NIX"') = " - foo - bar" ]] # Test nix-shell -p +# shellcheck disable=SC2016 output=$(NIX_PATH=nixpkgs="$shellDotNix" nix-shell --pure -p foo bar --run 'echo "$(foo) $(bar)"') [ "$output" = "foo bar" ] # Test nix-shell -p --arg x y +# shellcheck disable=SC2016 output=$(NIX_PATH=nixpkgs="$shellDotNix" nix-shell --pure -p foo --argstr fooContents baz --run 'echo "$(foo)"') [ "$output" = "baz" ] # Test nix-shell shebang mode -sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/shell.shebang.sh -chmod a+rx $TEST_ROOT/shell.shebang.sh +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > "$TEST_ROOT"/shell.shebang.sh +chmod a+rx "$TEST_ROOT"/shell.shebang.sh -output=$($TEST_ROOT/shell.shebang.sh abc def) +output=$("$TEST_ROOT"/shell.shebang.sh abc def) [ "$output" = "foo bar abc def" ] # Test nix-shell shebang mode with an alternate working directory -sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.expr > $TEST_ROOT/shell.shebang.expr -chmod a+rx $TEST_ROOT/shell.shebang.expr +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.expr > "$TEST_ROOT"/shell.shebang.expr +chmod a+rx "$TEST_ROOT"/shell.shebang.expr # Should fail due to expressions using relative path -! $TEST_ROOT/shell.shebang.expr bar -cp shell.nix "${config_nix}" $TEST_ROOT + "$TEST_ROOT"/shell.shebang.expr bar && exit 1 +cp shell.nix "${config_nix}" "$TEST_ROOT" # Should succeed echo "cwd: $PWD" -output=$($TEST_ROOT/shell.shebang.expr bar) +output=$("$TEST_ROOT"/shell.shebang.expr bar) [ "$output" = foo ] # Test nix-shell shebang mode with an alternate working directory -sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.legacy.expr > $TEST_ROOT/shell.shebang.legacy.expr -chmod a+rx $TEST_ROOT/shell.shebang.legacy.expr +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.legacy.expr > "$TEST_ROOT"/shell.shebang.legacy.expr +chmod a+rx "$TEST_ROOT"/shell.shebang.legacy.expr # Should fail due to expressions using relative path mkdir -p "$TEST_ROOT/somewhere-unrelated" -output="$(cd "$TEST_ROOT/somewhere-unrelated"; $TEST_ROOT/shell.shebang.legacy.expr bar;)" +output="$(cd "$TEST_ROOT/somewhere-unrelated"; "$TEST_ROOT"/shell.shebang.legacy.expr bar;)" [[ $(realpath "$output") = $(realpath "$TEST_ROOT/somewhere-unrelated") ]] # Test nix-shell shebang mode again with metacharacters in the filename. # First word of filename is chosen to not match any file in the test root. -sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh -chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.sh +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.sh > "$TEST_ROOT"/spaced\ \\\'\"shell.shebang.sh +chmod a+rx "$TEST_ROOT"/spaced\ \\\'\"shell.shebang.sh -output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.sh abc def) +output=$("$TEST_ROOT"/spaced\ \\\'\"shell.shebang.sh abc def) [ "$output" = "foo bar abc def" ] # Test nix-shell shebang mode for ruby # This uses a fake interpreter that returns the arguments passed # This, in turn, verifies the `rc` script is valid and the `load()` script (given using `-e`) is as expected. -sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/shell.shebang.rb -chmod a+rx $TEST_ROOT/shell.shebang.rb +sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > "$TEST_ROOT"/shell.shebang.rb +chmod a+rx "$TEST_ROOT"/shell.shebang.rb -output=$($TEST_ROOT/shell.shebang.rb abc ruby) +output=$("$TEST_ROOT"/shell.shebang.rb abc ruby) [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/shell.shebang.rb abc ruby' ] # Test nix-shell shebang mode for ruby again with metacharacters in the filename. # Note: fake interpreter only space-separates args without adding escapes to its output. -sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb -chmod a+rx $TEST_ROOT/spaced\ \\\'\"shell.shebang.rb +sed -e "s|@SHELL_PROG@|$(type -P nix-shell)|" shell.shebang.rb > "$TEST_ROOT"/spaced\ \\\'\"shell.shebang.rb +chmod a+rx "$TEST_ROOT"/spaced\ \\\'\"shell.shebang.rb -output=$($TEST_ROOT/spaced\ \\\'\"shell.shebang.rb abc ruby) +output=$("$TEST_ROOT"/spaced\ \\\'\"shell.shebang.rb abc ruby) +# shellcheck disable=SC1003 [ "$output" = '-e load(ARGV.shift) -- '"$TEST_ROOT"'/spaced \'\''"shell.shebang.rb abc ruby' ] # Test nix-shell shebang quoting -sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.nix > $TEST_ROOT/shell.shebang.nix -chmod a+rx $TEST_ROOT/shell.shebang.nix -$TEST_ROOT/shell.shebang.nix +sed -e "s|@ENV_PROG@|$(type -P env)|" shell.shebang.nix > "$TEST_ROOT"/shell.shebang.nix +chmod a+rx "$TEST_ROOT"/shell.shebang.nix +"$TEST_ROOT"/shell.shebang.nix -mkdir $TEST_ROOT/lookup-test $TEST_ROOT/empty +mkdir "$TEST_ROOT"/lookup-test "$TEST_ROOT"/empty -echo "import $shellDotNix" > $TEST_ROOT/lookup-test/shell.nix -cp "${config_nix}" $TEST_ROOT/lookup-test/ -echo 'abort "do not load default.nix!"' > $TEST_ROOT/lookup-test/default.nix +echo "import $shellDotNix" > "$TEST_ROOT"/lookup-test/shell.nix +cp "${config_nix}" "$TEST_ROOT"/lookup-test/ +echo 'abort "do not load default.nix!"' > "$TEST_ROOT"/lookup-test/default.nix -nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it works"' | grepQuiet "it works" +nix-shell "$TEST_ROOT"/lookup-test -A shellDrv --run 'echo "it works"' | grepQuiet "it works" # https://github.com/NixOS/nix/issues/4529 nix-shell -I "testRoot=$TEST_ROOT" '' -A shellDrv --run 'echo "it works"' | grepQuiet "it works" -expectStderr 1 nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it works"' --option nix-shell-always-looks-for-shell-nix false \ +expectStderr 1 nix-shell "$TEST_ROOT"/lookup-test -A shellDrv --run 'echo "it works"' --option nix-shell-always-looks-for-shell-nix false \ | grepQuiet -F "do not load default.nix!" # we did, because we chose to enable legacy behavior -expectStderr 1 nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it works"' --option nix-shell-always-looks-for-shell-nix false \ +expectStderr 1 nix-shell "$TEST_ROOT"/lookup-test -A shellDrv --run 'echo "it works"' --option nix-shell-always-looks-for-shell-nix false \ | grepQuiet "Skipping .*lookup-test/shell\.nix.*, because the setting .*nix-shell-always-looks-for-shell-nix.* is disabled. This is a deprecated behavior\. Consider enabling .*nix-shell-always-looks-for-shell-nix.*" ( - cd $TEST_ROOT/empty; + cd "$TEST_ROOT"/empty; expectStderr 1 nix-shell | \ grepQuiet "error.*no argument specified and no .*shell\.nix.* or .*default\.nix.* file found in the working directory" ) @@ -147,29 +157,29 @@ expectStderr 1 nix-shell $TEST_ROOT/lookup-test -A shellDrv --run 'echo "it work expectStderr 1 nix-shell -I "testRoot=$TEST_ROOT" '' | grepQuiet "error.*neither .*shell\.nix.* nor .*default\.nix.* found in .*/empty" -cat >$TEST_ROOT/lookup-test/shebangscript <"$TEST_ROOT"/lookup-test/shebangscript < $TEST_ROOT/marco/shell.nix -cat >$TEST_ROOT/marco/polo/default.nix < "$TEST_ROOT"/marco/shell.nix +cat >"$TEST_ROOT"/marco/polo/default.nix <$TEST_ROOT/issue-11892/shebangscript <"$TEST_ROOT"/issue-11892/shebangscript <$TEST_ROOT/issue-11892/shebangscript <$TEST_ROOT/issue-11892/my_package.nix <"$TEST_ROOT"/issue-11892/my_package.nix < $TEST_ROOT/dev-env.sh -nix print-dev-env -f "$shellDotNix" shellDrv --json > $TEST_ROOT/dev-env.json +nix print-dev-env -f "$shellDotNix" shellDrv > "$TEST_ROOT"/dev-env.sh +nix print-dev-env -f "$shellDotNix" shellDrv --json > "$TEST_ROOT"/dev-env.json # Test with raw drv shellDrv=$(nix-instantiate "$shellDotNix" -A shellDrv.out) -nix develop $shellDrv -c bash -c '[[ -n $stdenv ]]' +# shellcheck disable=SC2016 +nix develop "$shellDrv" -c bash -c '[[ -n $stdenv ]]' -nix print-dev-env $shellDrv > $TEST_ROOT/dev-env2.sh -nix print-dev-env $shellDrv --json > $TEST_ROOT/dev-env2.json +nix print-dev-env "$shellDrv" > "$TEST_ROOT"/dev-env2.sh +nix print-dev-env "$shellDrv" --json > "$TEST_ROOT"/dev-env2.json -diff $TEST_ROOT/dev-env{,2}.sh -diff $TEST_ROOT/dev-env{,2}.json +diff "$TEST_ROOT"/dev-env{,2}.sh +diff "$TEST_ROOT"/dev-env{,2}.json # Ensure `nix print-dev-env --json` contains variable assignments. -[[ $(jq -r .variables.arr1.value[2] $TEST_ROOT/dev-env.json) = '3 4' ]] +[[ $(jq -r .variables.arr1.value[2] "$TEST_ROOT"/dev-env.json) = '3 4' ]] # Run tests involving `source <(nix print-dev-env)` in subshells to avoid modifying the current # environment. @@ -238,27 +250,32 @@ set -u # Ensure `source <(nix print-dev-env)` modifies the environment. ( path=$PATH - source $TEST_ROOT/dev-env.sh + # shellcheck disable=SC1091 + source "$TEST_ROOT"/dev-env.sh [[ -n $stdenv ]] + # shellcheck disable=SC2154 [[ ${arr1[2]} = "3 4" ]] + # shellcheck disable=SC2154 [[ ${arr2[1]} = $'\n' ]] [[ ${arr2[2]} = $'x\ny' ]] [[ $(fun) = blabla ]] - [[ $PATH = $(jq -r .variables.PATH.value $TEST_ROOT/dev-env.json):$path ]] + [[ $PATH = $(jq -r .variables.PATH.value "$TEST_ROOT"/dev-env.json):$path ]] ) # Ensure `source <(nix print-dev-env)` handles the case when PATH is empty. ( path=$PATH + # shellcheck disable=SC2123 PATH= - source $TEST_ROOT/dev-env.sh - [[ $PATH = $(PATH=$path jq -r .variables.PATH.value $TEST_ROOT/dev-env.json) ]] + # shellcheck disable=SC1091 + source "$TEST_ROOT"/dev-env.sh + [[ $PATH = $(PATH=$path jq -r .variables.PATH.value "$TEST_ROOT"/dev-env.json) ]] ) # Test nix-shell with ellipsis and no `inNixShell` argument (for backwards compat with old nixpkgs) -cat >$TEST_ROOT/shell-ellipsis.nix <"$TEST_ROOT"/shell-ellipsis.nix <' --restrict-eval unset NIX_PATH -mkdir -p $TEST_ROOT/{from-nix-path-file,from-NIX_PATH,from-nix-path,from-extra-nix-path,from-I} +mkdir -p "$TEST_ROOT"/{from-nix-path-file,from-NIX_PATH,from-nix-path,from-extra-nix-path,from-I} for i in from-nix-path-file from-NIX_PATH from-nix-path from-extra-nix-path from-I; do - touch $TEST_ROOT/$i/only-$i.nix + touch "$TEST_ROOT"/$i/only-$i.nix done # finding something that's not in any of the default paths fails +# shellcheck disable=SC2091 ( ! $(nix-instantiate --find-file test) ) echo "nix-path = test=$TEST_ROOT/from-nix-path-file" >> "$test_nix_conf" @@ -53,36 +54,36 @@ echo "nix-path = test=$TEST_ROOT/from-nix-path-file" >> "$test_nix_conf" (! NIX_PATH=test=$TEST_ROOT nix-instantiate --find-file test/only-from-nix-path-file.nix) # -I extends nix.conf -[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +[[ $(nix-instantiate -I test="$TEST_ROOT"/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] # if -I does not have the desired entry, the value from nix.conf is used -[[ $(nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path-file.nix) = $TEST_ROOT/from-nix-path-file/only-from-nix-path-file.nix ]] +[[ $(nix-instantiate -I test="$TEST_ROOT"/from-I --find-file test/only-from-nix-path-file.nix) = $TEST_ROOT/from-nix-path-file/only-from-nix-path-file.nix ]] # -I extends NIX_PATH -[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test="$TEST_ROOT"/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] # -I takes precedence over NIX_PATH -[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test) = $TEST_ROOT/from-I ]] +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test="$TEST_ROOT"/from-I --find-file test) = $TEST_ROOT/from-I ]] # if -I does not have the desired entry, the value from NIX_PATH is used -[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test=$TEST_ROOT/from-I --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate -I test="$TEST_ROOT"/from-I --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] # --extra-nix-path extends NIX_PATH -[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test="$TEST_ROOT"/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] # if --extra-nix-path does not have the desired entry, the value from NIX_PATH is used -[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --extra-nix-path test="$TEST_ROOT"/from-extra-nix-path --find-file test/only-from-NIX_PATH.nix) = $TEST_ROOT/from-NIX_PATH/only-from-NIX_PATH.nix ]] # --nix-path overrides NIX_PATH -[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +[[ $(NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test="$TEST_ROOT"/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] # if --nix-path does not have the desired entry, it fails -(! NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-NIX_PATH.nix) +(! NIX_PATH=test=$TEST_ROOT/from-NIX_PATH nix-instantiate --nix-path test="$TEST_ROOT"/from-nix-path --find-file test/only-from-NIX_PATH.nix) # --nix-path overrides nix.conf -[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] -(! nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path --find-file test/only-from-nix-path-file.nix) +[[ $(nix-instantiate --nix-path test="$TEST_ROOT"/from-nix-path --find-file test) = $TEST_ROOT/from-nix-path ]] +(! nix-instantiate --nix-path test="$TEST_ROOT"/from-nix-path --find-file test/only-from-nix-path-file.nix) # --extra-nix-path extends nix.conf -[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] +[[ $(nix-instantiate --extra-nix-path test="$TEST_ROOT"/from-extra-nix-path --find-file test/only-from-extra-nix-path.nix) = $TEST_ROOT/from-extra-nix-path/only-from-extra-nix-path.nix ]] # if --extra-nix-path does not have the desired entry, it is taken from nix.conf -[[ $(nix-instantiate --extra-nix-path test=$TEST_ROOT/from-extra-nix-path --find-file test) = $TEST_ROOT/from-nix-path-file ]] +[[ $(nix-instantiate --extra-nix-path test="$TEST_ROOT"/from-extra-nix-path --find-file test) = $TEST_ROOT/from-nix-path-file ]] # -I extends --nix-path -[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] -[[ $(nix-instantiate --nix-path test=$TEST_ROOT/from-nix-path -I test=$TEST_ROOT/from-I --find-file test/only-from-nix-path.nix) = $TEST_ROOT/from-nix-path/only-from-nix-path.nix ]] +[[ $(nix-instantiate --nix-path test="$TEST_ROOT"/from-nix-path -I test="$TEST_ROOT"/from-I --find-file test/only-from-I.nix) = $TEST_ROOT/from-I/only-from-I.nix ]] +[[ $(nix-instantiate --nix-path test="$TEST_ROOT"/from-nix-path -I test="$TEST_ROOT"/from-I --find-file test/only-from-nix-path.nix) = $TEST_ROOT/from-nix-path/only-from-nix-path.nix ]] diff --git a/tests/functional/optimise-store.sh b/tests/functional/optimise-store.sh index 05c4c41e428..332a308c208 100755 --- a/tests/functional/optimise-store.sh +++ b/tests/functional/optimise-store.sh @@ -4,28 +4,31 @@ source common.sh clearStoreIfPossible +# shellcheck disable=SC2016 outPath1=$(echo 'with import '"${config_nix}"'; mkDerivation { name = "foo1"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) +# shellcheck disable=SC2016 outPath2=$(echo 'with import '"${config_nix}"'; mkDerivation { name = "foo2"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link --auto-optimise-store) TODO_NixOS # ignoring the client-specified setting 'auto-optimise-store', because it is a restricted setting and you are not a trusted user # TODO: only continue when trusted user or root -inode1="$(stat --format=%i $outPath1/foo)" -inode2="$(stat --format=%i $outPath2/foo)" +inode1="$(stat --format=%i "$outPath1"/foo)" +inode2="$(stat --format=%i "$outPath2"/foo)" if [ "$inode1" != "$inode2" ]; then echo "inodes do not match" exit 1 fi -nlink="$(stat --format=%h $outPath1/foo)" +nlink="$(stat --format=%h "$outPath1"/foo)" if [ "$nlink" != 3 ]; then echo "link count incorrect" exit 1 fi +# shellcheck disable=SC2016 outPath3=$(echo 'with import '"${config_nix}"'; mkDerivation { name = "foo3"; builder = builtins.toFile "builder" "mkdir $out; echo hello > $out/foo"; }' | nix-build - --no-out-link) -inode3="$(stat --format=%i $outPath3/foo)" +inode3="$(stat --format=%i "$outPath3"/foo)" if [ "$inode1" = "$inode3" ]; then echo "inodes match unexpectedly" exit 1 @@ -34,8 +37,8 @@ fi # XXX: This should work through the daemon too NIX_REMOTE="" nix-store --optimise -inode1="$(stat --format=%i $outPath1/foo)" -inode3="$(stat --format=%i $outPath3/foo)" +inode1="$(stat --format=%i "$outPath1"/foo)" +inode3="$(stat --format=%i "$outPath3"/foo)" if [ "$inode1" != "$inode3" ]; then echo "inodes do not match" exit 1 @@ -43,7 +46,7 @@ fi nix-store --gc -if [ -n "$(ls $NIX_STORE_DIR/.links)" ]; then +if [ -n "$(ls "$NIX_STORE_DIR"/.links)" ]; then echo ".links directory not empty after GC" exit 1 fi diff --git a/tests/functional/output-normalization.sh b/tests/functional/output-normalization.sh index c55f1b1d148..bd1668db9ad 100755 --- a/tests/functional/output-normalization.sh +++ b/tests/functional/output-normalization.sh @@ -6,7 +6,7 @@ testNormalization () { TODO_NixOS clearStore outPath=$(nix-build ./simple.nix --no-out-link) - test "$(stat -c %Y $outPath)" -eq 1 + test "$(stat -c %Y "$outPath")" -eq 1 } testNormalization diff --git a/tests/functional/package.nix b/tests/functional/package.nix index 1f1d10ea85e..5f5e5886843 100644 --- a/tests/functional/package.nix +++ b/tests/functional/package.nix @@ -26,6 +26,9 @@ # For running the functional tests against a different pre-built Nix. test-daemon ? null, + + # Whether to run tests with lazy trees enabled. + lazyTrees ? false, }: let @@ -95,6 +98,8 @@ mkMesonDerivation ( mkdir $out ''; + _NIX_TEST_EXTRA_CONFIG = lib.optionalString lazyTrees "lazy-trees = true"; + meta = { platforms = lib.platforms.unix; }; diff --git a/tests/functional/parallel.builder.sh b/tests/functional/parallel.builder.sh index d092bc5a6bd..4362465713f 100644 --- a/tests/functional/parallel.builder.sh +++ b/tests/functional/parallel.builder.sh @@ -1,29 +1,31 @@ +# shellcheck shell=bash +# shellcheck disable=SC2154 echo "DOING $text" # increase counter -while ! ln -s x $shared.lock 2> /dev/null; do +while ! ln -s x "$shared".lock 2> /dev/null; do sleep 1 done -test -f $shared.cur || echo 0 > $shared.cur -test -f $shared.max || echo 0 > $shared.max -new=$(($(cat $shared.cur) + 1)) -if test $new -gt $(cat $shared.max); then - echo $new > $shared.max +test -f "$shared".cur || echo 0 > "$shared".cur +test -f "$shared".max || echo 0 > "$shared".max +new=$(($(cat "$shared".cur) + 1)) +if test $new -gt "$(cat "$shared".max)"; then + echo $new > "$shared".max fi -echo $new > $shared.cur -rm $shared.lock +echo $new > "$shared".cur +rm "$shared".lock -echo -n $(cat $inputs)$text > $out +echo -n "$(cat "$inputs")""$text" > "$out" -sleep $sleepTime +sleep "$sleepTime" # decrease counter -while ! ln -s x $shared.lock 2> /dev/null; do +while ! ln -s x "$shared".lock 2> /dev/null; do sleep 1 done -test -f $shared.cur || echo 0 > $shared.cur -echo $(($(cat $shared.cur) - 1)) > $shared.cur -rm $shared.lock +test -f "$shared".cur || echo 0 > "$shared".cur +echo $(($(cat "$shared".cur) - 1)) > "$shared".cur +rm "$shared".lock diff --git a/tests/functional/parallel.sh b/tests/functional/parallel.sh index 7e420688d4f..4d0bf0f1be0 100644 --- a/tests/functional/parallel.sh +++ b/tests/functional/parallel.sh @@ -1,3 +1,4 @@ +# shellcheck shell=bash source common.sh @@ -8,7 +9,7 @@ TODO_NixOS clearStore -rm -f $_NIX_TEST_SHARED.cur $_NIX_TEST_SHARED.max +rm -f "$_NIX_TEST_SHARED".cur "$_NIX_TEST_SHARED".max outPath=$(nix-build -j10000 parallel.nix --no-out-link) @@ -17,8 +18,8 @@ echo "output path is $outPath" text=$(cat "$outPath") if test "$text" != "abacade"; then exit 1; fi -if test "$(cat $_NIX_TEST_SHARED.cur)" != 0; then fail "wrong current process count"; fi -if test "$(cat $_NIX_TEST_SHARED.max)" != 3; then fail "not enough parallelism"; fi +if test "$(cat "$_NIX_TEST_SHARED".cur)" != 0; then fail "wrong current process count"; fi +if test "$(cat "$_NIX_TEST_SHARED".max)" != 3; then fail "not enough parallelism"; fi # Second, test that parallel invocations of nix-build perform builds @@ -27,7 +28,7 @@ echo "testing multiple nix-build -j1..." clearStore -rm -f $_NIX_TEST_SHARED.cur $_NIX_TEST_SHARED.max +rm -f "$_NIX_TEST_SHARED".cur "$_NIX_TEST_SHARED".max drvPath=$(nix-instantiate parallel.nix --argstr sleepTime 15) @@ -54,5 +55,5 @@ wait $pid2 || fail "instance 2 failed: $?" wait $pid3 || fail "instance 3 failed: $?" wait $pid4 || fail "instance 4 failed: $?" -if test "$(cat $_NIX_TEST_SHARED.cur)" != 0; then fail "wrong current process count"; fi -if test "$(cat $_NIX_TEST_SHARED.max)" != 3; then fail "not enough parallelism"; fi +if test "$(cat "$_NIX_TEST_SHARED".cur)" != 0; then fail "wrong current process count"; fi +if test "$(cat "$_NIX_TEST_SHARED".max)" != 3; then fail "not enough parallelism"; fi diff --git a/tests/functional/pass-as-file.sh b/tests/functional/pass-as-file.sh index 66a8e588ee1..68f68b8cf1f 100755 --- a/tests/functional/pass-as-file.sh +++ b/tests/functional/pass-as-file.sh @@ -4,6 +4,7 @@ source common.sh clearStoreIfPossible +# shellcheck disable=SC2034 outPath=$(nix-build --no-out-link -E " with import ${config_nix}; diff --git a/tests/functional/path-from-hash-part.sh b/tests/functional/path-from-hash-part.sh index 41d1b7410f6..0b258a6ea5e 100755 --- a/tests/functional/path-from-hash-part.sh +++ b/tests/functional/path-from-hash-part.sh @@ -4,9 +4,9 @@ source common.sh path=$(nix build --no-link --print-out-paths -f simple.nix) -hash_part=$(basename $path) +hash_part=$(basename "$path") hash_part=${hash_part:0:32} -path2=$(nix store path-from-hash-part $hash_part) +path2=$(nix store path-from-hash-part "$hash_part") -[[ $path = $path2 ]] +[[ $path = "$path2" ]] diff --git a/tests/functional/path-info.sh b/tests/functional/path-info.sh index 8597de68341..a44d94396ef 100755 --- a/tests/functional/path-info.sh +++ b/tests/functional/path-info.sh @@ -2,15 +2,15 @@ source common.sh -echo foo > $TEST_ROOT/foo -foo=$(nix store add-file $TEST_ROOT/foo) +echo foo > "$TEST_ROOT"/foo +foo=$(nix store add-file "$TEST_ROOT"/foo) -echo bar > $TEST_ROOT/bar -bar=$(nix store add-file $TEST_ROOT/bar) +echo bar > "$TEST_ROOT"/bar +bar=$(nix store add-file "$TEST_ROOT"/bar) -echo baz > $TEST_ROOT/baz -baz=$(nix store add-file $TEST_ROOT/baz) -nix-store --delete "$baz" +echo baz > "$TEST_ROOT"/baz +baz=$(nix store add-file "$TEST_ROOT"/baz) +nix-store --delete --ignore-liveness "$baz" diff --unified --color=always \ <(nix path-info --json "$foo" "$bar" "$baz" | diff --git a/tests/functional/placeholders.sh b/tests/functional/placeholders.sh index 374203af8d8..5791d8006ae 100755 --- a/tests/functional/placeholders.sh +++ b/tests/functional/placeholders.sh @@ -4,6 +4,7 @@ source common.sh clearStoreIfPossible +# shellcheck disable=SC2016 nix-build --no-out-link -E ' with import '"${config_nix}"'; diff --git a/tests/functional/post-hook.sh b/tests/functional/post-hook.sh index 94a6d0d6912..67bb46377a9 100755 --- a/tests/functional/post-hook.sh +++ b/tests/functional/post-hook.sh @@ -6,10 +6,10 @@ TODO_NixOS clearStore -rm -f $TEST_ROOT/result +rm -f "$TEST_ROOT"/result export REMOTE_STORE=file:$TEST_ROOT/remote_store -echo 'require-sigs = false' >> $test_nix_conf +echo 'require-sigs = false' >> "$test_nix_conf" restartDaemon @@ -20,11 +20,14 @@ else fi # Build the dependencies and push them to the remote store. -nix-build -o $TEST_ROOT/result dependencies.nix --post-build-hook "$pushToStore" +nix-build -o "$TEST_ROOT"/result dependencies.nix --post-build-hook "$pushToStore" # See if all outputs are passed to the post-build hook by only specifying one # We're not able to test CA tests this way -export BUILD_HOOK_ONLY_OUT_PATHS=$([ ! $NIX_TESTS_CA_BY_DEFAULT ]) -nix-build -o $TEST_ROOT/result-mult multiple-outputs.nix -A a.first --post-build-hook "$pushToStore" +# +# FIXME: This export is hiding error condition +# shellcheck disable=SC2155 +export BUILD_HOOK_ONLY_OUT_PATHS=$([ ! "$NIX_TESTS_CA_BY_DEFAULT" ]) +nix-build -o "$TEST_ROOT"/result-mult multiple-outputs.nix -A a.first --post-build-hook "$pushToStore" clearStore diff --git a/tests/functional/pure-eval.sh b/tests/functional/pure-eval.sh index 45a65f9ab8f..b769b2150f1 100755 --- a/tests/functional/pure-eval.sh +++ b/tests/functional/pure-eval.sh @@ -10,6 +10,7 @@ nix eval --expr 'assert 1 + 2 == 3; true' missingImpureErrorMsg=$(! nix eval --expr 'builtins.readFile ./pure-eval.sh' 2>&1) +# shellcheck disable=SC1111 echo "$missingImpureErrorMsg" | grepQuiet -- --impure || \ fail "The error message should mention the “--impure” flag to unblock users" @@ -25,14 +26,15 @@ echo "$missingImpureErrorMsg" | grepQuiet -- --impure || \ (! nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; })).x") nix eval --expr "(import (builtins.fetchurl { url = file://$(pwd)/pure-eval.nix; sha256 = \"$(nix hash file pure-eval.nix --type sha256)\"; })).x" -rm -rf $TEST_ROOT/eval-out -nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ x = "foo" + "bar"; y = { z = "bla"; }; }' -[[ $(cat $TEST_ROOT/eval-out/x) = foobar ]] -[[ $(cat $TEST_ROOT/eval-out/y/z) = bla ]] +rm -rf "$TEST_ROOT"/eval-out +nix eval --store dummy:// --write-to "$TEST_ROOT"/eval-out --expr '{ x = "foo" + "bar"; y = { z = "bla"; }; }' +[[ $(cat "$TEST_ROOT"/eval-out/x) = foobar ]] +[[ $(cat "$TEST_ROOT"/eval-out/y/z) = bla ]] -rm -rf $TEST_ROOT/eval-out -(! nix eval --store dummy:// --write-to $TEST_ROOT/eval-out --expr '{ "." = "bla"; }') +rm -rf "$TEST_ROOT"/eval-out +(! nix eval --store dummy:// --write-to "$TEST_ROOT"/eval-out --expr '{ "." = "bla"; }') +# shellcheck disable=SC2088 (! nix eval --expr '~/foo') expectStderr 0 nix eval --expr "/some/absolute/path" \ diff --git a/tests/functional/read-only-store.sh b/tests/functional/read-only-store.sh index ea96bba41b3..8ccca2192af 100755 --- a/tests/functional/read-only-store.sh +++ b/tests/functional/read-only-store.sh @@ -12,10 +12,10 @@ clearStore happy () { # We can do a read-only query just fine with a read-only store - nix --store local?read-only=true path-info $dummyPath + nix --store local?read-only=true path-info "$dummyPath" # `local://` also works. - nix --store local://?read-only=true path-info $dummyPath + nix --store local://?read-only=true path-info "$dummyPath" # We can "write" an already-present store-path a read-only store, because no IO is actually required nix-store --store local?read-only=true --add dummy @@ -37,8 +37,8 @@ happy ## Testing read-only mode with an underlying store that is actually read-only # Ensure store is actually read-only -chmod -R -w $TEST_ROOT/store -chmod -R -w $TEST_ROOT/var +chmod -R -w "$TEST_ROOT"/store +chmod -R -w "$TEST_ROOT"/var # Make sure we fail on add operations on the read-only store # This is only for adding files that are not *already* in the store diff --git a/tests/functional/readfile-context.sh b/tests/functional/readfile-context.sh index cb9ef62347e..effe483dc6e 100755 --- a/tests/functional/readfile-context.sh +++ b/tests/functional/readfile-context.sh @@ -9,12 +9,12 @@ clearStore outPath=$(nix-build --no-out-link readfile-context.nix) # Set a GC root. -ln -s $outPath "$NIX_STATE_DIR/gcroots/foo" +ln -s "$outPath" "$NIX_STATE_DIR/gcroots/foo" # Check that file exists. -[ "$(cat $(cat $outPath))" = "Hello World!" ] +[ "$(cat "$(cat "$outPath")")" = "Hello World!" ] nix-collect-garbage # Check that file still exists. -[ "$(cat $(cat $outPath))" = "Hello World!" ] +[ "$(cat "$(cat "$outPath")")" = "Hello World!" ] diff --git a/tests/functional/recursive.nix b/tests/functional/recursive.nix index be9e55da37e..aa2aa26c549 100644 --- a/tests/functional/recursive.nix +++ b/tests/functional/recursive.nix @@ -17,7 +17,7 @@ mkDerivation rec { buildCommand = '' mkdir $out - opts="--experimental-features nix-command ${ + opts="${ if (NIX_TESTS_CA_BY_DEFAULT == "1") then "--extra-experimental-features ca-derivations" else "" }" diff --git a/tests/functional/recursive.sh b/tests/functional/recursive.sh index 640fb92d2c5..16c3fdab4df 100755 --- a/tests/functional/recursive.sh +++ b/tests/functional/recursive.sh @@ -9,15 +9,16 @@ restartDaemon clearStore -rm -f $TEST_ROOT/result +rm -f "$TEST_ROOT"/result -export unreachable=$(nix store add-path ./recursive.sh) +unreachable=$(nix store add-path ./recursive.sh) +export unreachable -NIX_BIN_DIR=$(dirname $(type -p nix)) nix --extra-experimental-features 'nix-command recursive-nix' build -o $TEST_ROOT/result -L --impure --file ./recursive.nix +NIX_BIN_DIR=$(dirname "$(type -p nix)") nix --extra-experimental-features 'recursive-nix' build -o "$TEST_ROOT"/result -L --impure --file ./recursive.nix -[[ $(cat $TEST_ROOT/result/inner1) =~ blaat ]] +[[ $(cat "$TEST_ROOT"/result/inner1) =~ blaat ]] # Make sure the recursively created paths are in the closure. -nix path-info -r $TEST_ROOT/result | grep foobar -nix path-info -r $TEST_ROOT/result | grep fnord -nix path-info -r $TEST_ROOT/result | grep inner1 +nix path-info -r "$TEST_ROOT"/result | grep foobar +nix path-info -r "$TEST_ROOT"/result | grep fnord +nix path-info -r "$TEST_ROOT"/result | grep inner1 diff --git a/tests/functional/referrers.sh b/tests/functional/referrers.sh index 411cdb7c1fb..ae6b39ae115 100755 --- a/tests/functional/referrers.sh +++ b/tests/functional/referrers.sh @@ -11,32 +11,34 @@ clearStore max=500 reference=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bla -touch $reference -(echo $reference && echo && echo 0) | nix-store --register-validity +touch "$reference" +(echo "$reference" && echo && echo 0) | nix-store --register-validity echo "making registration..." set +x +# shellcheck disable=SC2004 for ((n = 0; n < $max; n++)); do storePath=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-$n - echo -n > $storePath + echo -n > "$storePath" ref2=$NIX_STORE_DIR/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-$((n+1)) if test $((n+1)) = $max; then ref2=$reference fi - echo $storePath; echo; echo 2; echo $reference; echo $ref2 -done > $TEST_ROOT/reg_info + echo "$storePath"; echo; echo 2; echo "$reference"; echo "$ref2" +done > "$TEST_ROOT"/reg_info set -x echo "registering..." -nix-store --register-validity < $TEST_ROOT/reg_info +nix-store --register-validity < "$TEST_ROOT"/reg_info echo "collecting garbage..." -ln -sfn $reference "$NIX_STATE_DIR/gcroots/ref" +ln -sfn "$reference" "$NIX_STATE_DIR/gcroots/ref" nix-store --gc -if [ -n "$(type -p sqlite3)" -a "$(sqlite3 $NIX_STATE_DIR/db/db.sqlite 'select count(*) from Refs')" -ne 0 ]; then +# shellcheck disable=SC2166 +if [ -n "$(type -p sqlite3)" -a "$(sqlite3 "$NIX_STATE_DIR"/db/db.sqlite 'select count(*) from Refs')" -ne 0 ]; then echo "referrers not cleaned up" exit 1 fi diff --git a/tests/functional/remote-store.sh b/tests/functional/remote-store.sh index 841b6b27ae4..f125ae13759 100755 --- a/tests/functional/remote-store.sh +++ b/tests/functional/remote-store.sh @@ -7,10 +7,10 @@ TODO_NixOS clearStore # Ensure "fake ssh" remote store works just as legacy fake ssh would. -nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store doctor +nix --store ssh-ng://localhost?remote-store="$TEST_ROOT"/other-store doctor # Ensure that store info trusted works with ssh-ng:// -nix --store ssh-ng://localhost?remote-store=$TEST_ROOT/other-store store info --json | jq -e '.trusted' +nix --store ssh-ng://localhost?remote-store="$TEST_ROOT"/other-store store info --json | jq -e '.trusted' startDaemon @@ -31,8 +31,8 @@ NIX_REMOTE_=$NIX_REMOTE $SHELL ./user-envs-test-case.sh nix-store --gc --max-freed 1K -nix-store --dump-db > $TEST_ROOT/d1 -NIX_REMOTE= nix-store --dump-db > $TEST_ROOT/d2 -cmp $TEST_ROOT/d1 $TEST_ROOT/d2 +nix-store --dump-db > "$TEST_ROOT"/d1 +NIX_REMOTE='' nix-store --dump-db > "$TEST_ROOT"/d2 +cmp "$TEST_ROOT"/d1 "$TEST_ROOT"/d2 killDaemon diff --git a/tests/functional/repair.sh b/tests/functional/repair.sh index 1f6004b2c84..a90bdcfd5b4 100755 --- a/tests/functional/repair.sh +++ b/tests/functional/repair.sh @@ -8,39 +8,43 @@ TODO_NixOS clearStore -path=$(nix-build dependencies.nix -o $TEST_ROOT/result) -path2=$(nix-store -qR $path | grep input-2) +path=$(nix-build dependencies.nix -o "$TEST_ROOT"/result) +path2=$(nix-store -qR "$path" | grep input-2) nix-store --verify --check-contents -v -hash=$(nix-hash $path2) +hash=$(nix-hash "$path2") # Corrupt a path and check whether nix-build --repair can fix it. -chmod u+w $path2 -touch $path2/bad +chmod u+w "$path2" +touch "$path2"/bad (! nix-store --verify --check-contents -v) # The path can be repaired by rebuilding the derivation. nix-store --verify --check-contents --repair -(! [ -e $path2/bad ]) -(! [ -w $path2 ]) +# shellcheck disable=SC2235 +(! [ -e "$path2"/bad ]) +# shellcheck disable=SC2235 +(! [ -w "$path2" ]) -nix-store --verify-path $path2 +nix-store --verify-path "$path2" # Re-corrupt and delete the deriver. Now --verify --repair should # not work. -chmod u+w $path2 -touch $path2/bad +chmod u+w "$path2" +touch "$path2"/bad -nix-store --delete $(nix-store -q --referrers-closure $(nix-store -qd $path2)) +# shellcheck disable=SC2046 +nix-store --delete $(nix-store -q --referrers-closure "$(nix-store -qd "$path2")") (! nix-store --verify --check-contents --repair) -nix-build dependencies.nix -o $TEST_ROOT/result --repair +nix-build dependencies.nix -o "$TEST_ROOT"/result --repair -if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then +# shellcheck disable=SC2166 +if [ "$(nix-hash "$path2")" != "$hash" -o -e "$path2"/bad ]; then echo "path not repaired properly" >&2 exit 1 fi @@ -49,79 +53,83 @@ fi # --verify can fix it. clearCache -nix copy --to file://$cacheDir $path +nix copy --to file://"$cacheDir" "$path" -chmod u+w $path2 -rm -rf $path2 +chmod u+w "$path2" +rm -rf "$path2" nix-store --verify --check-contents --repair --substituters "file://$cacheDir" --no-require-sigs -if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then +# shellcheck disable=SC2166 +if [ "$(nix-hash "$path2")" != "$hash" -o -e "$path2"/bad ]; then echo "path not repaired properly" >&2 exit 1 fi # Check --verify-path and --repair-path. -nix-store --verify-path $path2 +nix-store --verify-path "$path2" -chmod u+w $path2 -rm -rf $path2 +chmod u+w "$path2" +rm -rf "$path2" -if nix-store --verify-path $path2; then +if nix-store --verify-path "$path2"; then echo "nix-store --verify-path succeeded unexpectedly" >&2 exit 1 fi -nix-store --repair-path $path2 --substituters "file://$cacheDir" --no-require-sigs +nix-store --repair-path "$path2" --substituters "file://$cacheDir" --no-require-sigs -if [ "$(nix-hash $path2)" != "$hash" -o -e $path2/bad ]; then +# shellcheck disable=SC2166 +if [ "$(nix-hash "$path2")" != "$hash" -o -e "$path2"/bad ]; then echo "path not repaired properly" >&2 exit 1 fi # Check that --repair-path also checks content of optimised symlinks (1/2) -nix-store --verify-path $path2 +nix-store --verify-path "$path2" if (! nix-store --optimize); then echo "nix-store --optimize failed to optimize the store" >&2 exit 1 fi -chmod u+w $path2/bar -echo 'rabrab' > $path2/bar # different length +chmod u+w "$path2"/bar +echo 'rabrab' > "$path2"/bar # different length -if nix-store --verify-path $path2; then +if nix-store --verify-path "$path2"; then echo "nix-store --verify-path did not detect .links file corruption" >&2 exit 1 fi -nix-store --repair-path $path2 --option auto-optimise-store true +nix-store --repair-path "$path2" --option auto-optimise-store true -if [ "$(nix-hash $path2)" != "$hash" -o "BAR" != "$(< $path2/bar)" ]; then +# shellcheck disable=SC2166 +if [ "$(nix-hash "$path2")" != "$hash" -o "BAR" != "$(< "$path2"/bar)" ]; then echo "path not repaired properly" >&2 exit 1 fi # Check that --repair-path also checks content of optimised symlinks (2/2) -nix-store --verify-path $path2 +nix-store --verify-path "$path2" if (! nix-store --optimize); then echo "nix-store --optimize failed to optimize the store" >&2 exit 1 fi -chmod u+w $path2 -chmod u+w $path2/bar -sed -e 's/./X/g' < $path2/bar > $path2/tmp # same length, different content. -cp $path2/tmp $path2/bar -rm $path2/tmp +chmod u+w "$path2" +chmod u+w "$path2"/bar +sed -e 's/./X/g' < "$path2"/bar > "$path2"/tmp # same length, different content. +cp "$path2"/tmp "$path2"/bar +rm "$path2"/tmp -if nix-store --verify-path $path2; then +if nix-store --verify-path "$path2"; then echo "nix-store --verify-path did not detect .links file corruption" >&2 exit 1 fi -nix-store --repair-path $path2 --substituters "file://$cacheDir" --no-require-sigs --option auto-optimise-store true +nix-store --repair-path "$path2" --substituters "file://$cacheDir" --no-require-sigs --option auto-optimise-store true -if [ "$(nix-hash $path2)" != "$hash" -o "BAR" != "$(< $path2/bar)" ]; then +# shellcheck disable=SC2166 +if [ "$(nix-hash "$path2")" != "$hash" -o "BAR" != "$(< "$path2"/bar)" ]; then echo "path not repaired properly" >&2 exit 1 fi diff --git a/tests/functional/repl.sh b/tests/functional/repl.sh index bfe18c9e586..d75b80bb0b0 100755 --- a/tests/functional/repl.sh +++ b/tests/functional/repl.sh @@ -155,7 +155,7 @@ EOF testReplResponse ' foo + baz ' "3" \ - ./flake ./flake\#bar --experimental-features 'flakes' + ./flake ./flake\#bar testReplResponse $' :a { a = 1; b = 2; longerName = 3; "with spaces" = 4; } @@ -190,7 +190,7 @@ testReplResponseNoRegex $' # - Check that the result has changed mkfifo repl_fifo touch repl_output -nix repl ./flake --experimental-features 'flakes' < repl_fifo >> repl_output 2>&1 & +nix repl ./flake < repl_fifo >> repl_output 2>&1 & repl_pid=$! exec 3>repl_fifo # Open fifo for writing echo "changingThing" >&3 @@ -314,7 +314,7 @@ import $testDir/lang/parse-fail-eof-pos.nix badDiff=0 badExitCode=0 -nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion' --extra-experimental-features nix-command)" +nixVersion="$(nix eval --impure --raw --expr 'builtins.nixVersion')" # TODO: write a repl interacter for testing. Papering over the differences between readline / editline and between platforms is a pain. diff --git a/tests/functional/restricted.sh b/tests/functional/restricted.sh index 00ee4ddc8c2..2f65f15fe5d 100755 --- a/tests/functional/restricted.sh +++ b/tests/functional/restricted.sh @@ -40,30 +40,32 @@ nix eval --raw --expr "builtins.fetchurl file://${_NIX_TEST_SOURCE_DIR}/restrict (! nix eval --raw --expr "fetchGit git://github.com/NixOS/patchelf.git" --impure --restrict-eval) ln -sfn "${_NIX_TEST_SOURCE_DIR}/restricted.nix" "$TEST_ROOT/restricted.nix" -[[ $(nix-instantiate --eval $TEST_ROOT/restricted.nix) == 3 ]] -(! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix) -(! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I $TEST_ROOT) -(! nix-instantiate --eval --restrict-eval $TEST_ROOT/restricted.nix -I .) +[[ $(nix-instantiate --eval "$TEST_ROOT"/restricted.nix) == 3 ]] +(! nix-instantiate --eval --restrict-eval "$TEST_ROOT"/restricted.nix) +(! nix-instantiate --eval --restrict-eval "$TEST_ROOT"/restricted.nix -I "$TEST_ROOT") +(! nix-instantiate --eval --restrict-eval "$TEST_ROOT"/restricted.nix -I .) nix-instantiate --eval --restrict-eval "$TEST_ROOT/restricted.nix" -I "$TEST_ROOT" -I "${_NIX_TEST_SOURCE_DIR}" +# shellcheck disable=SC2016 [[ $(nix eval --raw --impure --restrict-eval -I . --expr 'builtins.readFile "${import ./simple.nix}/hello"') == 'Hello World!' ]] # Check that we can't follow a symlink outside of the allowed paths. -mkdir -p $TEST_ROOT/tunnel.d $TEST_ROOT/foo2 -ln -sfn .. $TEST_ROOT/tunnel.d/tunnel -echo foo > $TEST_ROOT/bar +mkdir -p "$TEST_ROOT"/tunnel.d "$TEST_ROOT"/foo2 +ln -sfn .. "$TEST_ROOT"/tunnel.d/tunnel +echo foo > "$TEST_ROOT"/bar -expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readFile " -I "$TEST_ROOT"/tunnel.d | grepQuiet "forbidden in restricted mode" -expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d | grepQuiet "forbidden in restricted mode" +expectStderr 1 nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I "$TEST_ROOT"/tunnel.d | grepQuiet "forbidden in restricted mode" # Reading the parents of allowed paths should show only the ancestors of the allowed paths. -[[ $(nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I $TEST_ROOT/tunnel.d) == '{ "tunnel.d" = "directory"; }' ]] +[[ $(nix-instantiate --restrict-eval --eval -E "let __nixPath = [ { prefix = \"foo\"; path = $TEST_ROOT/tunnel.d; } ]; in builtins.readDir " -I "$TEST_ROOT"/tunnel.d) == '{ "tunnel.d" = "directory"; }' ]] # Check whether we can leak symlink information through directory traversal. traverseDir="${_NIX_TEST_SOURCE_DIR}/restricted-traverse-me" ln -sfn "${_NIX_TEST_SOURCE_DIR}/restricted-secret" "${_NIX_TEST_SOURCE_DIR}/restricted-innocent" mkdir -p "$traverseDir" +# shellcheck disable=SC2001 goUp="..$(echo "$traverseDir" | sed -e 's,[^/]\+,..,g')" output="$(nix eval --raw --restrict-eval -I "$traverseDir" \ --expr "builtins.readFile \"$traverseDir/$goUp${_NIX_TEST_SOURCE_DIR}/restricted-innocent\"" \ diff --git a/tests/functional/secure-drv-outputs.sh b/tests/functional/secure-drv-outputs.sh index 5cc4af43521..876d3c817e7 100755 --- a/tests/functional/secure-drv-outputs.sh +++ b/tests/functional/secure-drv-outputs.sh @@ -13,20 +13,20 @@ clearStore startDaemon # Determine the output path of the "good" derivation. -goodOut=$(nix-store -q $(nix-instantiate ./secure-drv-outputs.nix -A good)) +goodOut=$(nix-store -q "$(nix-instantiate ./secure-drv-outputs.nix -A good)") # Instantiate the "bad" derivation. badDrv=$(nix-instantiate ./secure-drv-outputs.nix -A bad) -badOut=$(nix-store -q $badDrv) +badOut=$(nix-store -q "$badDrv") # Rewrite the bad derivation to produce the output path of the good # derivation. -rm -f $TEST_ROOT/bad.drv -sed -e "s|$badOut|$goodOut|g" < $badDrv > $TEST_ROOT/bad.drv +rm -f "$TEST_ROOT"/bad.drv +sed -e "s|$badOut|$goodOut|g" < "$badDrv" > "$TEST_ROOT"/bad.drv # Add the manipulated derivation to the store and build it. This # should fail. -if badDrv2=$(nix-store --add $TEST_ROOT/bad.drv); then +if badDrv2=$(nix-store --add "$TEST_ROOT"/bad.drv); then nix-store -r "$badDrv2" fi diff --git a/tests/functional/selfref-gc.sh b/tests/functional/selfref-gc.sh index dc4f14cc190..de202a09dd0 100755 --- a/tests/functional/selfref-gc.sh +++ b/tests/functional/selfref-gc.sh @@ -6,6 +6,7 @@ requireDaemonNewerThan "2.6.0pre20211215" clearStoreIfPossible +# shellcheck disable=SC2016 nix-build --no-out-link -E ' with import '"${config_nix}"'; diff --git a/tests/functional/shell.shebang.sh b/tests/functional/shell.shebang.sh index f7132043de4..b6e4ee28693 100755 --- a/tests/functional/shell.shebang.sh +++ b/tests/functional/shell.shebang.sh @@ -1,4 +1,5 @@ #! @ENV_PROG@ nix-shell #! nix-shell -I nixpkgs=shell.nix --no-substitute #! nix-shell --pure -i bash -p foo bar -echo "$(foo) $(bar) $@" +# shellcheck shell=bash +echo "$(foo) $(bar)" "$@" diff --git a/tests/functional/simple.builder.sh b/tests/functional/simple.builder.sh index 97abf06763d..27cdfe6843a 100644 --- a/tests/functional/simple.builder.sh +++ b/tests/functional/simple.builder.sh @@ -6,7 +6,9 @@ echo "PATH=$PATH" if mkdir foo 2> /dev/null; then exit 1; fi # Set a PATH (!!! impure). +# shellcheck disable=SC2154 export PATH=$goodPath +# shellcheck disable=SC2154 mkdir "$out" echo "Hello World!" > "$out"/hello diff --git a/tests/functional/simple.sh b/tests/functional/simple.sh index c1f2eef411e..e54ad860ca9 100755 --- a/tests/functional/simple.sh +++ b/tests/functional/simple.sh @@ -21,7 +21,7 @@ TODO_NixOS # Directed delete: $outPath is not reachable from a root, so it should # be deleteable. -nix-store --delete "$outPath" +nix-store --delete "$outPath" --ignore-liveness [[ ! -e $outPath/hello ]] outPath="$(NIX_REMOTE='local?store=/foo&real='"$TEST_ROOT"'/real-store' nix-instantiate --readonly-mode hash-check.nix)" diff --git a/tests/functional/store-info.sh b/tests/functional/store-info.sh index 7c9257215bf..ee896929ae3 100755 --- a/tests/functional/store-info.sh +++ b/tests/functional/store-info.sh @@ -13,14 +13,20 @@ normalize_nix_store_url () { # Need to actually ask Nix in this case echo "$defaultStore" ;; + local | 'local://' ) + echo 'local' + ;; + daemon | 'unix://' ) + echo 'daemon' + ;; 'local://'* ) # To not be captured by next pattern echo "$url" ;; - local | 'local?'* ) + 'local?'* ) echo "local://${url#local}" ;; - daemon | 'daemon?'* ) + 'daemon?'* ) echo "unix://${url#daemon}" ;; * ) @@ -38,13 +44,13 @@ defaultStore="$(normalize_nix_store_url "$(echo "$STORE_INFO_JSON" | jq -r ".url # Test cases for `normalize_nix_store_url` itself # Normalize local store -[[ "$(normalize_nix_store_url "local://")" = "local://" ]] -[[ "$(normalize_nix_store_url "local")" = "local://" ]] +[[ "$(normalize_nix_store_url "local://")" = "local" ]] +[[ "$(normalize_nix_store_url "local")" = "local" ]] [[ "$(normalize_nix_store_url "local?foo=bar")" = "local://?foo=bar" ]] # Normalize unix domain socket remote store -[[ "$(normalize_nix_store_url "unix://")" = "unix://" ]] -[[ "$(normalize_nix_store_url "daemon")" = "unix://" ]] +[[ "$(normalize_nix_store_url "unix://")" = "daemon" ]] +[[ "$(normalize_nix_store_url "daemon")" = "daemon" ]] [[ "$(normalize_nix_store_url "daemon?x=y")" = "unix://?x=y" ]] # otherwise unchanged @@ -59,7 +65,7 @@ check_human_readable "$STORE_INFO" check_human_readable "$LEGACY_STORE_INFO" if [[ -v NIX_DAEMON_PACKAGE ]] && isDaemonNewer "2.7.0pre20220126"; then - DAEMON_VERSION=$("$NIX_DAEMON_PACKAGE"/bin/nix daemon --version | cut -d' ' -f3) + DAEMON_VERSION=$("$NIX_DAEMON_PACKAGE"/bin/nix daemon --version | sed 's/.*) //') echo "$STORE_INFO" | grep "Version: $DAEMON_VERSION" [[ "$(echo "$STORE_INFO_JSON" | jq -r ".version")" == "$DAEMON_VERSION" ]] fi diff --git a/tests/functional/structured-attrs.nix b/tests/functional/structured-attrs.nix index 4e19845176e..70ac807ab25 100644 --- a/tests/functional/structured-attrs.nix +++ b/tests/functional/structured-attrs.nix @@ -82,4 +82,8 @@ mkDerivation { "foo$" = "BAD"; exportReferencesGraph.refs = [ dep ]; + exportReferencesGraph.refs2 = [ + dep + [ dep ] + ]; # regression test for heterogeneous arrays } diff --git a/tests/functional/structured-attrs.sh b/tests/functional/structured-attrs.sh index 2bd9b4aaf1b..473a037f9f3 100755 --- a/tests/functional/structured-attrs.sh +++ b/tests/functional/structured-attrs.sh @@ -2,9 +2,8 @@ source common.sh -# 27ce722638 required some incompatible changes to the nix file, so skip this -# tests for the older versions -requireDaemonNewerThan "2.4pre20210712" +# https://github.com/NixOS/nix/pull/14189 +requireDaemonNewerThan "2.33" clearStoreIfPossible @@ -50,4 +49,4 @@ expectStderr 0 nix-instantiate --expr "$hackyExpr" --eval --strict | grepQuiet " # Check it works with the expected structured attrs hacky=$(nix-instantiate --expr "$hackyExpr") -nix derivation show "$hacky" | jq --exit-status '."'"$hacky"'".structuredAttrs | . == {"a": 1}' +nix derivation show "$hacky" | jq --exit-status '."'"$(basename "$hacky")"'".structuredAttrs | . == {"a": 1}' diff --git a/tests/functional/supplementary-groups.sh b/tests/functional/supplementary-groups.sh index a667d3e998c..0f614a13093 100755 --- a/tests/functional/supplementary-groups.sh +++ b/tests/functional/supplementary-groups.sh @@ -9,6 +9,7 @@ needLocalStore "The test uses --store always so we would just be bypassing the d TODO_NixOS +# shellcheck disable=SC2119 execUnshare <buildPathsWithResults(paths, bmNormal, store); for (const auto & result : results) { - for (const auto & [outputName, realisation] : result.builtOutputs) { - std::cout << store->printStorePath(realisation.outPath) << "\n"; + if (auto * successP = result.tryGetSuccess()) { + for (const auto & [outputName, realisation] : successP->builtOutputs) { + std::cout << store->printStorePath(realisation.outPath) << "\n"; + } } } diff --git a/tests/functional/test-libstoreconsumer/meson.build b/tests/functional/test-libstoreconsumer/meson.build index e5a1cc18221..7f619d01baa 100644 --- a/tests/functional/test-libstoreconsumer/meson.build +++ b/tests/functional/test-libstoreconsumer/meson.build @@ -1,3 +1,7 @@ +cxx = meson.get_compiler('cpp') + +subdir('nix-meson-build-support/asan-options') + libstoreconsumer_tester = executable( 'test-libstoreconsumer', 'main.cc', diff --git a/tests/functional/test-libstoreconsumer/nix-meson-build-support b/tests/functional/test-libstoreconsumer/nix-meson-build-support new file mode 120000 index 00000000000..ac8a39762cb --- /dev/null +++ b/tests/functional/test-libstoreconsumer/nix-meson-build-support @@ -0,0 +1 @@ +../../../nix-meson-build-support \ No newline at end of file diff --git a/tests/functional/toString-path.sh b/tests/functional/toString-path.sh index d790109f41a..c425b61be0e 100755 --- a/tests/functional/toString-path.sh +++ b/tests/functional/toString-path.sh @@ -2,8 +2,8 @@ source common.sh -mkdir -p $TEST_ROOT/foo -echo bla > $TEST_ROOT/foo/bar +mkdir -p "$TEST_ROOT"/foo +echo bla > "$TEST_ROOT"/foo/bar [[ $(nix eval --raw --impure --expr "builtins.readFile (builtins.toString (builtins.fetchTree { type = \"path\"; path = \"$TEST_ROOT/foo\"; } + \"/bar\"))") = bla ]] diff --git a/tests/functional/user-envs-migration.sh b/tests/functional/user-envs-migration.sh index 0f33074e156..46337cdda55 100755 --- a/tests/functional/user-envs-migration.sh +++ b/tests/functional/user-envs-migration.sh @@ -29,6 +29,7 @@ nix-env -f user-envs.nix -i bar-0.1 # Migrate to the new profile dir, and ensure that everything’s there export PATH="$PATH_WITH_NEW_NIX" nix-env -q # Trigger the migration +# shellcheck disable=SC2235 ( [[ -L ~/.nix-profile ]] && \ [[ $(readlink ~/.nix-profile) == ~/.local/share/nix/profiles/profile ]] ) || \ fail "The nix profile should point to the new location" diff --git a/tests/functional/user-envs-test-case.sh b/tests/functional/user-envs-test-case.sh index 3483a4600d7..f6a8ab8c692 100644 --- a/tests/functional/user-envs-test-case.sh +++ b/tests/functional/user-envs-test-case.sh @@ -1,14 +1,17 @@ +# shellcheck shell=bash clearProfiles # Query installed: should be empty. -test "$(nix-env -p $profiles/test -q '*' | wc -l)" -eq 0 +# shellcheck disable=SC2154 +test "$(nix-env -p "$profiles"/test -q '*' | wc -l)" -eq 0 -nix-env --switch-profile $profiles/test +nix-env --switch-profile "$profiles"/test # Query available: should contain several. test "$(nix-env -f ./user-envs.nix -qa '*' | wc -l)" -eq 6 outPath10=$(nix-env -f ./user-envs.nix -qa --out-path --no-name '*' | grep foo-1.0) drvPath10=$(nix-env -f ./user-envs.nix -qa --drv-path --no-name '*' | grep foo-1.0) +# shellcheck disable=SC2166 [ -n "$outPath10" -a -n "$drvPath10" ] TODO_NixOS @@ -20,18 +23,19 @@ nix-env -f ./user-envs.nix -qa --json | jq -e '.[] | select(.name == "bar-0.1") ] | all' nix-env -f ./user-envs.nix -qa --json --out-path | jq -e '.[] | select(.name == "bar-0.1") | [ .outputName == "out", - (.outputs.out | test("'$NIX_STORE_DIR'.*-0\\.1")) + (.outputs.out | test("'"$NIX_STORE_DIR"'.*-0\\.1")) ] | all' -nix-env -f ./user-envs.nix -qa --json --drv-path | jq -e '.[] | select(.name == "bar-0.1") | (.drvPath | test("'$NIX_STORE_DIR'.*-0\\.1\\.drv"))' +nix-env -f ./user-envs.nix -qa --json --drv-path | jq -e '.[] | select(.name == "bar-0.1") | (.drvPath | test("'"$NIX_STORE_DIR"'.*-0\\.1\\.drv"))' # Query descriptions. nix-env -f ./user-envs.nix -qa '*' --description | grepQuiet silly -rm -rf $HOME/.nix-defexpr -ln -s $(pwd)/user-envs.nix $HOME/.nix-defexpr +rm -rf "$HOME"/.nix-defexpr +ln -s "$(pwd)"/user-envs.nix "$HOME"/.nix-defexpr nix-env -qa '*' --description | grepQuiet silly # Query the system. -nix-env -qa '*' --system | grepQuiet $system +# shellcheck disable=SC2154 +nix-env -qa '*' --system | grepQuiet "$system" # Install "foo-1.0". nix-env -i foo-1.0 @@ -40,7 +44,7 @@ nix-env -i foo-1.0 # executable). test "$(nix-env -q '*' | wc -l)" -eq 1 nix-env -q '*' | grepQuiet foo-1.0 -test "$($profiles/test/bin/foo)" = "foo-1.0" +test "$("$profiles"/test/bin/foo)" = "foo-1.0" # Test nix-env -qc to compare installed against available packages, and vice versa. nix-env -qc '*' | grepQuiet '< 2.0' @@ -55,6 +59,7 @@ nix-env -qas | grepQuiet -- '--- bar-0.1' # Disable foo. nix-env --set-flag active false foo +# shellcheck disable=SC2235 (! [ -e "$profiles/test/bin/foo" ]) # Enable foo. @@ -72,7 +77,7 @@ nix-env -i foo-2.0pre1 # Query installed: should contain foo-2.0pre1 now. test "$(nix-env -q '*' | wc -l)" -eq 1 nix-env -q '*' | grepQuiet foo-2.0pre1 -test "$($profiles/test/bin/foo)" = "foo-2.0pre1" +test "$("$profiles"/test/bin/foo)" = "foo-2.0pre1" # Upgrade "foo": should install foo-2.0. NIX_PATH=nixpkgs=./user-envs.nix:${NIX_PATH-} nix-env -f '' -u foo @@ -80,7 +85,7 @@ NIX_PATH=nixpkgs=./user-envs.nix:${NIX_PATH-} nix-env -f '' -u foo # Query installed: should contain foo-2.0 now. test "$(nix-env -q '*' | wc -l)" -eq 1 nix-env -q '*' | grepQuiet foo-2.0 -test "$($profiles/test/bin/foo)" = "foo-2.0" +test "$("$profiles"/test/bin/foo)" = "foo-2.0" # Store the path of foo-2.0. outPath20=$(nix-env -q --out-path --no-name '*' | grep foo-2.0) @@ -95,9 +100,9 @@ if nix-env -q '*' | grepQuiet foo; then false; fi nix-env -q '*' | grepQuiet bar # Rollback: should bring "foo" back. -oldGen="$(nix-store -q --resolve $profiles/test)" +oldGen="$(nix-store -q --resolve "$profiles"/test)" nix-env --rollback -[ "$(nix-store -q --resolve $profiles/test)" != "$oldGen" ] +[ "$(nix-store -q --resolve "$profiles"/test)" != "$oldGen" ] nix-env -q '*' | grepQuiet foo-2.0 nix-env -q '*' | grepQuiet bar @@ -122,23 +127,23 @@ test "$(nix-env --list-generations | wc -l)" -eq 8 # Switch to a specified generation. nix-env --switch-generation 7 -[ "$(nix-store -q --resolve $profiles/test)" = "$oldGen" ] +[ "$(nix-store -q --resolve "$profiles"/test)" = "$oldGen" ] # Install foo-1.0, now using its store path. nix-env -i "$outPath10" nix-env -q '*' | grepQuiet foo-1.0 -nix-store -qR $profiles/test | grep "$outPath10" -nix-store -q --referrers-closure $profiles/test | grep "$(nix-store -q --resolve $profiles/test)" -[ "$(nix-store -q --deriver "$outPath10")" = $drvPath10 ] +nix-store -qR "$profiles"/test | grep "$outPath10" +nix-store -q --referrers-closure "$profiles"/test | grep "$(nix-store -q --resolve "$profiles"/test)" +[ "$(nix-store -q --deriver "$outPath10")" = "$drvPath10" ] # Uninstall foo-1.0, using a symlink to its store path. -ln -sfn $outPath10/bin/foo $TEST_ROOT/symlink -nix-env -e $TEST_ROOT/symlink +ln -sfn "$outPath10"/bin/foo "$TEST_ROOT"/symlink +nix-env -e "$TEST_ROOT"/symlink if nix-env -q '*' | grepQuiet foo; then false; fi -nix-store -qR $profiles/test | grepInverse "$outPath10" +nix-store -qR "$profiles"/test | grepInverse "$outPath10" # Install foo-1.0, now using a symlink to its store path. -nix-env -i $TEST_ROOT/symlink +nix-env -i "$TEST_ROOT"/symlink nix-env -q '*' | grepQuiet foo # Delete all old generations. @@ -148,6 +153,7 @@ nix-env --delete-generations old # foo-1.0. nix-collect-garbage test -e "$outPath10" +# shellcheck disable=SC2235 (! [ -e "$outPath20" ]) # Uninstall everything @@ -156,7 +162,7 @@ test "$(nix-env -q '*' | wc -l)" -eq 0 # Installing "foo" should only install the newest foo. nix-env -i foo -test "$(nix-env -q '*' | grep foo- | wc -l)" -eq 1 +test "$(nix-env -q '*' | grep foo- -c)" -eq 1 nix-env -q '*' | grepQuiet foo-2.0 # On the other hand, this should install both (and should fail due to @@ -177,25 +183,25 @@ nix-env -q '*' | grepQuiet bar-0.1.1 # declared priorities. nix-env -e '*' nix-env -i foo-0.1 foo-1.0 -[ "$($profiles/test/bin/foo)" = "foo-1.0" ] +[ "$("$profiles"/test/bin/foo)" = "foo-1.0" ] nix-env --set-flag priority 1 foo-0.1 -[ "$($profiles/test/bin/foo)" = "foo-0.1" ] +[ "$("$profiles"/test/bin/foo)" = "foo-0.1" ] # Priorities can be overridden with the --priority flag nix-env -e '*' nix-env -i foo-1.0 -[ "$($profiles/test/bin/foo)" = "foo-1.0" ] +[ "$("$profiles"/test/bin/foo)" = "foo-1.0" ] nix-env -i --priority 1 foo-0.1 -[ "$($profiles/test/bin/foo)" = "foo-0.1" ] +[ "$("$profiles"/test/bin/foo)" = "foo-0.1" ] # Test nix-env --set. -nix-env --set $outPath10 -[ "$(nix-store -q --resolve $profiles/test)" = $outPath10 ] -nix-env --set $drvPath10 -[ "$(nix-store -q --resolve $profiles/test)" = $outPath10 ] +nix-env --set "$outPath10" +[ "$(nix-store -q --resolve "$profiles"/test)" = "$outPath10" ] +nix-env --set "$drvPath10" +[ "$(nix-store -q --resolve "$profiles"/test)" = "$outPath10" ] # Test the case where $HOME contains a symlink. -mkdir -p $TEST_ROOT/real-home/alice/.nix-defexpr/channels -ln -sfn $TEST_ROOT/real-home $TEST_ROOT/home -ln -sfn $(pwd)/user-envs.nix $TEST_ROOT/home/alice/.nix-defexpr/channels/foo +mkdir -p "$TEST_ROOT"/real-home/alice/.nix-defexpr/channels +ln -sfn "$TEST_ROOT"/real-home "$TEST_ROOT"/home +ln -sfn "$(pwd)"/user-envs.nix "$TEST_ROOT"/home/alice/.nix-defexpr/channels/foo HOME=$TEST_ROOT/home/alice nix-env -i foo-0.1 diff --git a/tests/functional/user-envs.builder.sh b/tests/functional/user-envs.builder.sh index 5fafa797f11..e875c2fe54d 100644 --- a/tests/functional/user-envs.builder.sh +++ b/tests/functional/user-envs.builder.sh @@ -1,5 +1,8 @@ -mkdir $out -mkdir $out/bin -echo "#! $shell" > $out/bin/$progName -echo "echo $name" >> $out/bin/$progName -chmod +x $out/bin/$progName +# shellcheck shell=bash +# shellcheck disable=SC2154 +mkdir "$out" +mkdir "$out"/bin +echo "#! $shell" > "$out"/bin/"$progName" +# shellcheck disable=SC2154 +echo "echo $name" >> "$out"/bin/"$progName" +chmod +x "$out"/bin/"$progName" diff --git a/tests/functional/why-depends.sh b/tests/functional/why-depends.sh index 45d1f2f0b4f..fe9ff9a6226 100755 --- a/tests/functional/why-depends.sh +++ b/tests/functional/why-depends.sh @@ -4,9 +4,9 @@ source common.sh clearStoreIfPossible -cp ./dependencies.nix ./dependencies.builder0.sh "${config_nix}" $TEST_HOME +cp ./dependencies.nix ./dependencies.builder0.sh "${config_nix}" "$TEST_HOME" -cd $TEST_HOME +cd "$TEST_HOME" nix why-depends --derivation --file ./dependencies.nix input2_drv input1_drv nix why-depends --file ./dependencies.nix input2_drv input1_drv diff --git a/tests/installer/default.nix b/tests/installer/default.nix index d48537dd0d0..dc831cc97b1 100644 --- a/tests/installer/default.nix +++ b/tests/installer/default.nix @@ -232,7 +232,7 @@ let source /etc/bashrc || true nix-env --version - nix --extra-experimental-features nix-command store info + nix store info out=\$(nix-build --no-substitute -E 'derivation { name = "foo"; system = "x86_64-linux"; builder = "/bin/sh"; args = ["-c" "echo foobar > \$out"]; }') [[ \$(cat \$out) = foobar ]] diff --git a/tests/nixos/authorization.nix b/tests/nixos/authorization.nix index ee3be7504bc..944e5925925 100644 --- a/tests/nixos/authorization.nix +++ b/tests/nixos/authorization.nix @@ -13,8 +13,6 @@ users.users.alice.isNormalUser = true; users.users.bob.isNormalUser = true; users.users.mallory.isNormalUser = true; - - nix.settings.experimental-features = "nix-command"; }; testScript = @@ -84,7 +82,7 @@ su --login mallory -c ' nix-store --generate-binary-cache-key cache1.example.org sk1 pk1 (! nix store sign --key-file sk1 ${pathFour} 2>&1)' | tee diag 1>&2 - grep -F "cannot open connection to remote store 'unix://'" diag + grep -F "cannot open connection to remote store 'daemon'" diag """) machine.succeed(""" diff --git a/tests/nixos/cgroups/default.nix b/tests/nixos/cgroups/default.nix index a6b4bca8c76..4161aba2ca2 100644 --- a/tests/nixos/cgroups/default.nix +++ b/tests/nixos/cgroups/default.nix @@ -9,7 +9,7 @@ { virtualisation.additionalPaths = [ pkgs.stdenvNoCC ]; nix.extraOptions = '' - extra-experimental-features = nix-command auto-allocate-uids cgroups + extra-experimental-features = auto-allocate-uids cgroups extra-system-features = uid-range ''; nix.settings.use-cgroups = true; diff --git a/tests/nixos/chroot-store.nix b/tests/nixos/chroot-store.nix index 0a4fff99222..ecac371e152 100644 --- a/tests/nixos/chroot-store.nix +++ b/tests/nixos/chroot-store.nix @@ -25,7 +25,6 @@ in virtualisation.writableStore = true; virtualisation.additionalPaths = [ pkgA ]; environment.systemPackages = [ pkgB ]; - nix.extraOptions = "experimental-features = nix-command"; }; }; diff --git a/tests/nixos/containers/containers.nix b/tests/nixos/containers/containers.nix index b590dc8498f..8d07c80b6a3 100644 --- a/tests/nixos/containers/containers.nix +++ b/tests/nixos/containers/containers.nix @@ -23,7 +23,7 @@ virtualisation.memorySize = 4096; nix.settings.substituters = lib.mkForce [ ]; nix.extraOptions = '' - extra-experimental-features = nix-command auto-allocate-uids cgroups + extra-experimental-features = auto-allocate-uids cgroups extra-system-features = uid-range ''; nix.nixPath = [ "nixpkgs=${nixpkgs}" ]; diff --git a/tests/nixos/content-encoding.nix b/tests/nixos/content-encoding.nix new file mode 100644 index 00000000000..debee377bdf --- /dev/null +++ b/tests/nixos/content-encoding.nix @@ -0,0 +1,190 @@ +# Test content encoding support in Nix: +# 1. Fetching compressed files from servers with Content-Encoding headers +# (e.g., fetching a zstd archive from a server using gzip Content-Encoding +# should preserve the zstd format, not double-decompress) +# 2. HTTP binary cache store upload/download with compression support + +{ lib, config, ... }: + +let + pkgs = config.nodes.machine.nixpkgs.pkgs; + + ztdCompressedFile = pkgs.stdenv.mkDerivation { + name = "dummy-zstd-compressed-archive"; + dontUnpack = true; + nativeBuildInputs = with pkgs; [ zstd ]; + buildPhase = '' + mkdir archive + for _ in {1..100}; do echo "lorem" > archive/file1; done + for _ in {1..100}; do echo "ipsum" > archive/file2; done + tar --zstd -cf archive.tar.zst archive + ''; + installPhase = '' + install -Dm 644 -T archive.tar.zst $out/share/archive + ''; + }; + + # Bare derivation for testing binary cache with logs + testDrv = builtins.toFile "test.nix" '' + derivation { + name = "test-package"; + builder = "/bin/sh"; + args = [ "-c" "echo 'Building test package...' >&2; echo 'hello from test package' > $out; echo 'Build complete!' >&2" ]; + system = builtins.currentSystem; + } + ''; +in + +{ + name = "content-encoding"; + + nodes = { + machine = + { pkgs, ... }: + { + networking.firewall.allowedTCPPorts = [ 80 ]; + + services.nginx.enable = true; + services.nginx.virtualHosts."localhost" = { + root = "${ztdCompressedFile}/share/"; + # Make sure that nginx really tries to compress the + # file on the fly with no regard to size/mime. + # http://nginx.org/en/docs/http/ngx_http_gzip_module.html + extraConfig = '' + gzip on; + gzip_types *; + gzip_proxied any; + gzip_min_length 0; + ''; + + # Upload endpoint with WebDAV + locations."/cache-upload" = { + root = "/var/lib/nginx-cache"; + extraConfig = '' + client_body_temp_path /var/lib/nginx-cache/tmp; + create_full_put_path on; + dav_methods PUT DELETE; + dav_access user:rw group:rw all:r; + + # Don't try to compress already compressed files + gzip off; + + # Rewrite to remove -upload suffix when writing files + rewrite ^/cache-upload/(.*)$ /cache/$1 break; + ''; + }; + + # Download endpoint with Content-Encoding headers + locations."/cache" = { + root = "/var/lib/nginx-cache"; + extraConfig = '' + gzip off; + + # Serve .narinfo files with gzip encoding + location ~ \.narinfo$ { + add_header Content-Encoding gzip; + default_type "text/x-nix-narinfo"; + } + + # Serve .ls files with gzip encoding + location ~ \.ls$ { + add_header Content-Encoding gzip; + default_type "application/json"; + } + + # Serve log files with brotli encoding + location ~ ^/cache/log/ { + add_header Content-Encoding br; + default_type "text/plain"; + } + ''; + }; + }; + + systemd.services.nginx = { + serviceConfig = { + StateDirectory = "nginx-cache"; + StateDirectoryMode = "0755"; + }; + }; + + environment.systemPackages = with pkgs; [ + file + gzip + brotli + curl + ]; + + virtualisation.writableStore = true; + nix.settings.substituters = lib.mkForce [ ]; + nix.settings.experimental-features = [ + "nix-command" + "flakes" + ]; + }; + }; + + # Check that when nix-prefetch-url is used with a zst tarball it does not get decompressed. + # Also test HTTP binary cache store with compression support. + testScript = '' + # fmt: off + start_all() + + machine.wait_for_unit("nginx.service") + + # Original test: zstd archive with gzip content-encoding + # Make sure that the file is properly compressed as the test would be meaningless otherwise + curl_output = machine.succeed("curl --compressed -v http://localhost/archive 2>&1") + assert "content-encoding: gzip" in curl_output.lower(), f"Expected 'content-encoding: gzip' in curl output, but got: {curl_output}" + + archive_path = machine.succeed("nix-prefetch-url http://localhost/archive --print-path | tail -n1").strip() + mime_type = machine.succeed(f"file --brief --mime-type {archive_path}").strip() + assert mime_type == "application/zstd", f"Expected archive to be 'application/zstd', but got: {mime_type}" + machine.succeed(f"tar --zstd -xf {archive_path}") + + # Test HTTP binary cache store with compression + outPath = machine.succeed(""" + nix build --store /var/lib/build-store -f ${testDrv} --print-out-paths --print-build-logs + """).strip() + + drvPath = machine.succeed(f""" + nix path-info --store /var/lib/build-store --derivation {outPath} + """).strip() + + # Upload to cache with compression (use cache-upload endpoint) + machine.succeed(f""" + nix copy --store /var/lib/build-store --to 'http://localhost/cache-upload?narinfo-compression=gzip&ls-compression=gzip&write-nar-listing=1' {outPath} -vvvvv 2>&1 | tail -100 + """) + machine.succeed(f""" + nix store copy-log --store /var/lib/build-store --to 'http://localhost/cache-upload?log-compression=br' {drvPath} -vvvvv 2>&1 | tail -100 + """) + + # List cache contents + print(machine.succeed("find /var/lib/nginx-cache -type f")) + + narinfoHash = outPath.split('/')[3].split('-')[0] + drvName = drvPath.split('/')[3] + + # Verify compression + machine.succeed(f"gzip -t /var/lib/nginx-cache/cache/{narinfoHash}.narinfo") + machine.succeed(f"gzip -t /var/lib/nginx-cache/cache/{narinfoHash}.ls") + machine.succeed(f"brotli -t /var/lib/nginx-cache/cache/log/{drvName}") + + # Check Content-Encoding headers on the download endpoint + narinfo_headers = machine.succeed(f"curl -I http://localhost/cache/{narinfoHash}.narinfo 2>&1") + assert "content-encoding: gzip" in narinfo_headers.lower(), f"Expected 'content-encoding: gzip' for .narinfo file, but headers were: {narinfo_headers}" + + ls_headers = machine.succeed(f"curl -I http://localhost/cache/{narinfoHash}.ls 2>&1") + assert "content-encoding: gzip" in ls_headers.lower(), f"Expected 'content-encoding: gzip' for .ls file, but headers were: {ls_headers}" + + log_headers = machine.succeed(f"curl -I http://localhost/cache/log/{drvName} 2>&1") + assert "content-encoding: br" in log_headers.lower(), f"Expected 'content-encoding: br' for log file, but headers were: {log_headers}" + + # Test fetching from cache + machine.succeed(f"nix copy --from 'http://localhost/cache' --no-check-sigs {outPath}") + + # Test log retrieval + log_output = machine.succeed(f"nix log --store 'http://localhost/cache' {drvPath} 2>&1") + assert "Building test package" in log_output, f"Expected 'Building test package' in log output, but got: {log_output}" + ''; +} diff --git a/tests/nixos/default.nix b/tests/nixos/default.nix index 2031e02a437..edfa4124f3f 100644 --- a/tests/nixos/default.nix +++ b/tests/nixos/default.nix @@ -187,7 +187,7 @@ in ca-fd-leak = runNixOSTest ./ca-fd-leak; - gzip-content-encoding = runNixOSTest ./gzip-content-encoding.nix; + content-encoding = runNixOSTest ./content-encoding.nix; functional_user = runNixOSTest ./functional/as-user.nix; @@ -207,5 +207,7 @@ in fetchurl = runNixOSTest ./fetchurl.nix; + fetchersSubstitute = runNixOSTest ./fetchers-substitute.nix; + chrootStore = runNixOSTest ./chroot-store.nix; } diff --git a/tests/nixos/fetch-git/test-cases/build-time-fetch-tree/default.nix b/tests/nixos/fetch-git/test-cases/build-time-fetch-tree/default.nix new file mode 100644 index 00000000000..a241c877d21 --- /dev/null +++ b/tests/nixos/fetch-git/test-cases/build-time-fetch-tree/default.nix @@ -0,0 +1,49 @@ +{ config, ... }: +{ + description = "build-time fetching"; + script = '' + import json + + # add a file to the repo + client.succeed(f""" + echo ${config.name # to make the git tree and store path unique + } > {repo.path}/test-case \ + && echo chiang-mai > {repo.path}/thailand \ + && {repo.git} add test-case thailand \ + && {repo.git} commit -m 'commit1' \ + && {repo.git} push origin main + """) + + # get the NAR hash + nar_hash = json.loads(client.succeed(f""" + nix flake prefetch --flake-registry "" git+{repo.remote} --json + """))['hash'] + + # construct the derivation + expr = f""" + derivation {{ + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = {{ + type = "git"; + url = "{repo.remote}"; + ref = "main"; + }}; + outputHashMode = "recursive"; + outputHash = "{nar_hash}"; + }} + """ + + # do the build-time fetch + out_path = client.succeed(f""" + nix build --print-out-paths --store /run/store --flake-registry "" --extra-experimental-features build-time-fetch-tree --expr '{expr}' + """).strip() + + # check if the committed file is there + client.succeed(f""" + test -f /run/store/{out_path}/thailand + """) + ''; +} diff --git a/tests/nixos/fetch-git/testsupport/setup.nix b/tests/nixos/fetch-git/testsupport/setup.nix index c13386c7223..3c9f4bddea1 100644 --- a/tests/nixos/fetch-git/testsupport/setup.nix +++ b/tests/nixos/fetch-git/testsupport/setup.nix @@ -81,10 +81,6 @@ in environment.variables = { _NIX_FORCE_HTTP = "1"; }; - nix.settings.experimental-features = [ - "nix-command" - "flakes" - ]; }; setupScript = ''''; testScript = '' diff --git a/tests/nixos/fetchers-substitute.nix b/tests/nixos/fetchers-substitute.nix new file mode 100644 index 00000000000..453982677be --- /dev/null +++ b/tests/nixos/fetchers-substitute.nix @@ -0,0 +1,176 @@ +{ + name = "fetchers-substitute"; + + nodes.substituter = + { pkgs, ... }: + { + virtualisation.writableStore = true; + + nix.settings.extra-experimental-features = [ + "nix-command" + "fetch-tree" + ]; + + networking.firewall.allowedTCPPorts = [ 5000 ]; + + services.nix-serve = { + enable = true; + secretKeyFile = + let + key = pkgs.writeTextFile { + name = "secret-key"; + text = '' + substituter:SerxxAca5NEsYY0DwVo+subokk+OoHcD9m6JwuctzHgSQVfGHe6nCc+NReDjV3QdFYPMGix4FMg0+K/TM1B3aA== + ''; + }; + in + "${key}"; + }; + }; + + nodes.importer = + { lib, ... }: + { + virtualisation.writableStore = true; + + nix.settings = { + extra-experimental-features = [ + "nix-command" + "fetch-tree" + ]; + substituters = lib.mkForce [ "http://substituter:5000" ]; + trusted-public-keys = lib.mkForce [ "substituter:EkFXxh3upwnPjUXg41d0HRWDzBoseBTINPiv0zNQd2g=" ]; + }; + }; + + testScript = + { nodes }: # python + '' + import json + + start_all() + + substituter.wait_for_unit("multi-user.target") + + ########################################## + # Test 1: builtins.fetchurl with substitution + ########################################## + + missing_file = "/only-on-substituter.txt" + + substituter.succeed(f"echo 'this should only exist on the substituter' > {missing_file}") + + file_hash = substituter.succeed(f"nix hash file {missing_file}").strip() + + file_store_path_json = substituter.succeed(f""" + nix-instantiate --eval --json --read-write-mode --expr ' + builtins.fetchurl {{ + url = "file://{missing_file}"; + sha256 = "{file_hash}"; + }} + ' + """) + + file_store_path = json.loads(file_store_path_json) + + substituter.succeed(f"nix store sign --key-file ${nodes.substituter.services.nix-serve.secretKeyFile} {file_store_path}") + + importer.wait_for_unit("multi-user.target") + + print("Testing fetchurl with substitution...") + importer.succeed(f""" + nix-instantiate -vvvvv --eval --json --read-write-mode --expr ' + builtins.fetchurl {{ + url = "file://{missing_file}"; + sha256 = "{file_hash}"; + }} + ' + """) + print("✓ fetchurl substitution works!") + + ########################################## + # Test 2: builtins.fetchTarball with substitution + ########################################## + + missing_tarball = "/only-on-substituter.tar.gz" + + # Create a directory with some content + substituter.succeed(""" + mkdir -p /tmp/test-tarball + echo 'Hello from tarball!' > /tmp/test-tarball/hello.txt + echo 'Another file' > /tmp/test-tarball/file2.txt + """) + + # Create a tarball + substituter.succeed(f"tar czf {missing_tarball} -C /tmp test-tarball") + + # For fetchTarball, we need to first fetch it without hash to get the store path, + # then compute the NAR hash of that path + tarball_store_path_json = substituter.succeed(f""" + nix-instantiate --eval --json --read-write-mode --expr ' + builtins.fetchTarball {{ + url = "file://{missing_tarball}"; + }} + ' + """) + + tarball_store_path = json.loads(tarball_store_path_json) + + # Get the NAR hash of the unpacked tarball in SRI format + path_info_json = substituter.succeed(f"nix path-info --json {tarball_store_path}").strip() + path_info_dict = json.loads(path_info_json) + # nix path-info returns a dict with store paths as keys + tarball_hash_sri = path_info_dict[tarball_store_path]["narHash"] + print(f"Tarball NAR hash (SRI): {tarball_hash_sri}") + + # Also get the old format hash for fetchTarball (which uses sha256 parameter) + tarball_hash = substituter.succeed(f"nix-store --query --hash {tarball_store_path}").strip() + + # Sign the tarball's store path + substituter.succeed(f"nix store sign --recursive --key-file ${nodes.substituter.services.nix-serve.secretKeyFile} {tarball_store_path}") + + # Now try to fetch the same tarball on the importer + # The file doesn't exist locally, so it should be substituted + print("Testing fetchTarball with substitution...") + result = importer.succeed(f""" + nix-instantiate -vvvvv --eval --json --read-write-mode --expr ' + builtins.fetchTarball {{ + url = "file://{missing_tarball}"; + sha256 = "{tarball_hash}"; + }} + ' + """) + + result_path = json.loads(result) + print(f"✓ fetchTarball substitution works! Result: {result_path}") + + # Verify the content is correct + # fetchTarball strips the top-level directory if there's only one + content = importer.succeed(f"cat {result_path}/hello.txt").strip() + assert content == "Hello from tarball!", f"Content mismatch: {content}" + print("✓ fetchTarball content verified!") + + ########################################## + # Test 3: Verify fetchTree does NOT substitute (preserves metadata) + ########################################## + + print("Testing that fetchTree without __final does NOT use substitution...") + + # fetchTree with just narHash (not __final) should try to download, which will fail + # since the file doesn't exist on the importer + exit_code = importer.fail(f""" + nix-instantiate --eval --json --read-write-mode --expr ' + builtins.fetchTree {{ + type = "tarball"; + url = "file:///only-on-substituter.tar.gz"; + narHash = "{tarball_hash_sri}"; + }} + ' 2>&1 + """) + + # Should fail with "does not exist" since it tries to download instead of substituting + assert "does not exist" in exit_code or "Couldn't open file" in exit_code, f"Expected download failure, got: {exit_code}" + print("✓ fetchTree correctly does NOT substitute non-final inputs!") + print(" (This preserves metadata like lastModified from the actual fetch)") + ''; +} diff --git a/tests/nixos/fetchurl.nix b/tests/nixos/fetchurl.nix index e8663debbcd..d75cc2017de 100644 --- a/tests/nixos/fetchurl.nix +++ b/tests/nixos/fetchurl.nix @@ -64,8 +64,6 @@ in ]; virtualisation.writableStore = true; - - nix.settings.experimental-features = "nix-command"; }; }; diff --git a/tests/nixos/fsync.nix b/tests/nixos/fsync.nix index e215e5b3c25..50105f1ccd9 100644 --- a/tests/nixos/fsync.nix +++ b/tests/nixos/fsync.nix @@ -23,7 +23,6 @@ in { virtualisation.emptyDiskImages = [ 1024 ]; environment.systemPackages = [ pkg1 ]; - nix.settings.experimental-features = [ "nix-command" ]; nix.settings.fsync-store-paths = true; nix.settings.require-sigs = false; boot.supportedFilesystems = [ diff --git a/tests/nixos/functional/common.nix b/tests/nixos/functional/common.nix index 4d32b757324..72b7b61d12c 100644 --- a/tests/nixos/functional/common.nix +++ b/tests/nixos/functional/common.nix @@ -24,6 +24,7 @@ in ]; nix.settings.substituters = lib.mkForce [ ]; + systemd.services.nix-daemon.environment._NIX_IN_TEST = "1"; environment.systemPackages = let diff --git a/tests/nixos/git-submodules.nix b/tests/nixos/git-submodules.nix index c6f53ada2dc..9105eb79bd7 100644 --- a/tests/nixos/git-submodules.nix +++ b/tests/nixos/git-submodules.nix @@ -24,7 +24,6 @@ { programs.ssh.extraConfig = "ConnectTimeout 30"; environment.systemPackages = [ pkgs.git ]; - nix.extraOptions = "experimental-features = nix-command flakes"; }; }; diff --git a/tests/nixos/github-flakes.nix b/tests/nixos/github-flakes.nix index 91fd6b06234..64fc8e27787 100644 --- a/tests/nixos/github-flakes.nix +++ b/tests/nixos/github-flakes.nix @@ -163,7 +163,6 @@ in ]; virtualisation.memorySize = 4096; nix.settings.substituters = lib.mkForce [ ]; - nix.extraOptions = "experimental-features = nix-command flakes"; networking.hosts.${(builtins.head nodes.github.networking.interfaces.eth1.ipv4.addresses).address} = [ "channels.nixos.org" @@ -204,14 +203,53 @@ in assert info["revision"] == "${nixpkgs.rev}", f"revision mismatch: {info['revision']} != ${nixpkgs.rev}" cat_log() + out = client.succeed("nix flake prefetch nixpkgs --json") + nar_hash = json.loads(out)['hash'] + + # Test build-time fetching of public flakes. + expr = f""" + derivation {{ + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = {{ + type = "github"; + owner = "NixOS"; + repo = "nixpkgs"; + }}; + outputHashMode = "recursive"; + outputHash = "{nar_hash}"; + }} + """ + client.succeed(f"nix build --store /run/store --extra-experimental-features build-time-fetch-tree -L --expr '{expr}'") + # ... otherwise it should use the API - out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0") + out = client.succeed("nix flake metadata private-flake --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0 --no-trust-tarballs-from-git-forges") print(out) info = json.loads(out) assert info["revision"] == "${private-flake-rev}", f"revision mismatch: {info['revision']} != ${private-flake-rev}" assert info["fingerprint"] cat_log() + # Test build-time fetching of private flakes. + expr = f""" + derivation {{ + name = "source"; + builder = "builtin:fetch-tree"; + system = "builtin"; + __structuredAttrs = true; + input = {{ + type = "github"; + owner = "fancy-enterprise"; + repo = "private-flake"; + }}; + outputHashMode = "recursive"; + outputHash = "{info['locked']['narHash']}"; + }} + """ + client.succeed(f"nix build --store /run/store --extra-experimental-features build-time-fetch-tree --access-tokens github.com=ghp_000000000000000000000000000000000000 -L --expr '{expr}'") + # Fetching with the resolved URL should produce the same result. info2 = json.loads(client.succeed(f"nix flake metadata {info['url']} --json --access-tokens github.com=ghp_000000000000000000000000000000000000 --tarball-ttl 0")) print(info["fingerprint"], info2["fingerprint"]) @@ -219,11 +257,16 @@ in client.succeed("nix registry pin nixpkgs") client.succeed("nix flake metadata nixpkgs --tarball-ttl 0 >&2") + client.succeed("nix eval nixpkgs#hello --eval-store dummy://?read-only=false >&2") # Test fetchTree on a github URL. hash = client.succeed(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr '(fetchTree {info['url']}).narHash'") assert hash == info['locked']['narHash'] + # Fetching with an incorrect NAR hash should fail. + out = client.fail(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr '(fetchTree \"github:fancy-enterprise/private-flake/{info['revision']}?narHash=sha256-HsrRFZYg69qaVe/wDyWBYLeS6ca7ACEJg2Z%2BGpEFw4A%3D\").narHash' 2>&1") + assert "NAR hash mismatch in input" in out, "NAR hash check did not fail with the expected error" + # Fetching without a narHash should succeed if trust-github is set and fail otherwise. client.succeed(f"nix eval --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}'") out = client.fail(f"nix eval --no-trust-tarballs-from-git-forges --raw --expr 'builtins.fetchTree github:github:fancy-enterprise/private-flake/{info['revision']}' 2>&1") diff --git a/tests/nixos/gzip-content-encoding.nix b/tests/nixos/gzip-content-encoding.nix deleted file mode 100644 index 22d196c6186..00000000000 --- a/tests/nixos/gzip-content-encoding.nix +++ /dev/null @@ -1,74 +0,0 @@ -# Test that compressed files fetched from server with compressed responses -# do not get excessively decompressed. -# E.g. fetching a zstd compressed tarball from a server, -# which compresses the response with `Content-Encoding: gzip`. -# The expected result is that the fetched file is a zstd archive. - -{ lib, config, ... }: - -let - pkgs = config.nodes.machine.nixpkgs.pkgs; - - ztdCompressedFile = pkgs.stdenv.mkDerivation { - name = "dummy-zstd-compressed-archive"; - dontUnpack = true; - nativeBuildInputs = with pkgs; [ zstd ]; - buildPhase = '' - mkdir archive - for _ in {1..100}; do echo "lorem" > archive/file1; done - for _ in {1..100}; do echo "ipsum" > archive/file2; done - tar --zstd -cf archive.tar.zst archive - ''; - installPhase = '' - install -Dm 644 -T archive.tar.zst $out/share/archive - ''; - }; - - fileCmd = "${pkgs.file}/bin/file"; -in - -{ - name = "gzip-content-encoding"; - - nodes = { - machine = - { config, pkgs, ... }: - { - networking.firewall.allowedTCPPorts = [ 80 ]; - - services.nginx.enable = true; - services.nginx.virtualHosts."localhost" = { - root = "${ztdCompressedFile}/share/"; - # Make sure that nginx really tries to compress the - # file on the fly with no regard to size/mime. - # http://nginx.org/en/docs/http/ngx_http_gzip_module.html - extraConfig = '' - gzip on; - gzip_types *; - gzip_proxied any; - gzip_min_length 0; - ''; - }; - virtualisation.writableStore = true; - virtualisation.additionalPaths = with pkgs; [ file ]; - nix.settings.substituters = lib.mkForce [ ]; - }; - }; - - # Check that when nix-prefetch-url is used with a zst tarball it does not get decompressed. - testScript = - { nodes }: - '' - # fmt: off - start_all() - - machine.wait_for_unit("nginx.service") - machine.succeed(""" - # Make sure that the file is properly compressed as the test would be meaningless otherwise - curl --compressed -v http://localhost/archive |& tr -s ' ' |& grep --ignore-case 'content-encoding: gzip' - archive_path=$(nix-prefetch-url http://localhost/archive --print-path | tail -n1) - [[ $(${fileCmd} --brief --mime-type $archive_path) == "application/zstd" ]] - tar --zstd -xf $archive_path - """) - ''; -} diff --git a/tests/nixos/nix-copy.nix b/tests/nixos/nix-copy.nix index 64de622de76..a7f0a6a326f 100644 --- a/tests/nixos/nix-copy.nix +++ b/tests/nixos/nix-copy.nix @@ -39,7 +39,6 @@ in pkgD.drvPath ]; nix.settings.substituters = lib.mkForce [ ]; - nix.settings.experimental-features = [ "nix-command" ]; services.getty.autologinUser = "root"; programs.ssh.extraConfig = '' Host * diff --git a/tests/nixos/s3-binary-cache-store.nix b/tests/nixos/s3-binary-cache-store.nix index a22e4c2c28f..1f79e8cf969 100644 --- a/tests/nixos/s3-binary-cache-store.nix +++ b/tests/nixos/s3-binary-cache-store.nix @@ -34,7 +34,6 @@ in virtualisation.additionalPaths = [ pkgA ]; environment.systemPackages = [ pkgs.minio-client ]; nix.extraOptions = '' - experimental-features = nix-command substituters = ''; services.minio = { @@ -53,7 +52,6 @@ in { virtualisation.writableStore = true; nix.extraOptions = '' - experimental-features = nix-command substituters = ''; }; diff --git a/tests/nixos/sourcehut-flakes.nix b/tests/nixos/sourcehut-flakes.nix index 3f05130d6aa..5b40866d1fa 100644 --- a/tests/nixos/sourcehut-flakes.nix +++ b/tests/nixos/sourcehut-flakes.nix @@ -119,7 +119,6 @@ in virtualisation.memorySize = 4096; nix.settings.substituters = lib.mkForce [ ]; nix.extraOptions = '' - experimental-features = nix-command flakes flake-registry = https://git.sr.ht/~NixOS/flake-registry/blob/master/flake-registry.json ''; environment.systemPackages = [ pkgs.jq ]; diff --git a/tests/nixos/tarball-flakes.nix b/tests/nixos/tarball-flakes.nix index 26c20cb1aef..a68d64bfdd8 100644 --- a/tests/nixos/tarball-flakes.nix +++ b/tests/nixos/tarball-flakes.nix @@ -61,7 +61,6 @@ in ]; virtualisation.memorySize = 4096; nix.settings.substituters = lib.mkForce [ ]; - nix.extraOptions = "experimental-features = nix-command flakes"; }; }; diff --git a/tests/nixos/user-sandboxing/default.nix b/tests/nixos/user-sandboxing/default.nix index 3f6b575b035..d6899140ad0 100644 --- a/tests/nixos/user-sandboxing/default.nix +++ b/tests/nixos/user-sandboxing/default.nix @@ -104,8 +104,8 @@ in # Wait for the build to be ready # This is OK because it runs as root, so we can access everything - machine.wait_until_succeeds("stat /nix/var/nix/builds/nix-build-open-build-dir.drv-*/build/syncPoint") - dir = machine.succeed("ls -d /nix/var/nix/builds/nix-build-open-build-dir.drv-*").strip() + machine.wait_until_succeeds("stat /nix/var/nix/builds/nix-*/build/syncPoint") + dir = machine.succeed("ls -d /nix/var/nix/builds/nix-*").strip() # But Alice shouldn't be able to access the build directory machine.fail(f"su alice -c 'ls {dir}/build'") @@ -125,8 +125,8 @@ in args = [ (builtins.storePath "${create-hello-world}") ]; }' >&2 & """.strip()) - machine.wait_until_succeeds("stat /nix/var/nix/builds/nix-build-innocent.drv-*/build/syncPoint") - dir = machine.succeed("ls -d /nix/var/nix/builds/nix-build-innocent.drv-*").strip() + machine.wait_until_succeeds("stat /nix/var/nix/builds/nix-*/build/syncPoint") + dir = machine.succeed("ls -d /nix/var/nix/builds/nix-*").strip() # The build ran as `nixbld1` (which is the only build user on the # machine), but a process running as `nixbld1` outside the sandbox diff --git a/tests/repl-completion.nix b/tests/repl-completion.nix index 07406e969cd..9ae37796bf5 100644 --- a/tests/repl-completion.nix +++ b/tests/repl-completion.nix @@ -15,7 +15,7 @@ runCommand "repl-completion" ]; expectScript = '' # Regression https://github.com/NixOS/nix/pull/10778 - spawn nix repl --offline --extra-experimental-features nix-command + spawn nix repl --offline expect "nix-repl>" send "foo = import ./does-not-exist.nix\n" expect "nix-repl>"