diff --git a/.agents/skills/openwrt-package-update/SKILL.md b/.agents/skills/openwrt-package-update/SKILL.md new file mode 100644 index 000000000..9ecbdb217 --- /dev/null +++ b/.agents/skills/openwrt-package-update/SKILL.md @@ -0,0 +1,211 @@ +--- +name: openwrt-package-update +description: Update a forked OpenWrt upstream package to a newer upstream version while preserving local customizations. Use when updating packages like adblock, mwan3, or other non-ns- prefixed upstream packages to newer releases, and when comparing local forks against the canonical OpenWrt packages feed to identify and merge upstream improvements. +license: GPL-2.0-only +compatibility: Requires git, curl, and bash. Works with OpenWrt release tags (v24.10.x, v24.11.x, etc.) and HEAD of the packages feed. +metadata: + domain: nethsecurity-packages + type: upstream-package-update +--- + +## What I do + +I help you update forked OpenWrt upstream packages (non-`ns-*` packages) to newer upstream versions while carefully preserving intentional local customizations. I: + +1. Fetch the canonical OpenWrt `feeds.conf.default` for a given release tag or the latest HEAD +2. Extract the pinned commit hash for the packages feed +3. Clone or update a local copy of the `openwrt/packages` GitHub mirror +4. Locate the package in the upstream feed +5. Generate a detailed diff showing all differences between upstream and your local fork +6. Guide you through analyzing the diff to identify: + - Pure upstream improvements to apply + - Your intentional customizations to preserve + - Conflicting changes requiring manual adaptation + +## When to use me + +Use this skill when you are: +- **Updating an existing forked package** to a newer upstream release (e.g., adblock 4.1.5 → 4.5.5) +- **Merging upstream improvements** while keeping local patches and configuration +- **Comparing against multiple OpenWrt releases** to understand what changed between versions +- **Evaluating the scope of local customization** before deciding whether to continue forking or contribute upstream +- **Refreshing patches and adapting them** to newer upstream versions + +Do **not** use this skill for: +- Creating entirely new `ns-*` packages (use `openwrt-package` skill instead) +- Patches that should be contributed upstream (this skill is for local forks only) +- Non-OpenWrt packages or custom software + +## Workflow + +### Step 1: Identify the target package and version + +Ask the user: +- **Which package are you updating?** (e.g., `adblock`, `mwan3`, `keepalived`) +- **Which OpenWrt version?** Suggest either: + - A specific release tag (e.g., `v24.10.5`) — extracts the pinned hash from that release's `feeds.conf.default` + - `HEAD` — uses the latest upstream version + +### Step 2: Run the diff script + +From the skill root directory, invoke: + +```bash +scripts/fetch-upstream.sh +``` + +Examples: +```bash +scripts/fetch-upstream.sh v24.10.5 adblock ../../../packages/adblock +scripts/fetch-upstream.sh HEAD mwan3 /absolute/path/to/packages/mwan3 +``` + +The script will: +- Fetch the feeds.conf.default for the target tag +- Clone the packages feed to `assets/packages/` (gitignored) +- Output a detailed `diff -ru` comparing upstream vs local + +### Step 3: Analyze the diff + +Review the diff output and classify each change: + +#### Category A: Pure Upstream Changes (Safe to apply) +These are bug fixes or features from upstream with no local modification: +- Upstream script improvements, new functions, or bugfixes +- Version bumps in dependencies +- Documentation updates + +**Action:** Copy the upstream version directly into the local package. + +#### Category B: Intentional Local Customizations (Always preserve) +These are deliberate modifications specific to NethSecurity: +- UCI configuration defaults hardcoded for NethSecurity +- Removal of upstream features not needed in NethSecurity +- Integration with ns-api or other NethSecurity components +- Build system tweaks (PKG_RELEASE bumps for local patches) + +**Action:** Keep these changes as-is. They should remain in the diff even after updating. + +#### Category C: Upstream Changes That Conflict With Local Customizations (Needs manual work) +These require careful adaptation: +- Upstream modified the same function that you customized +- New upstream feature overlaps with your local workaround +- Upstream changed configuration options you've modified + +**Action:** +- Read both versions carefully +- Decide whether upstream's approach is better (merge it) +- Or adapt upstream's changes to work with your customization +- Test thoroughly after merging + +### Step 4: Update the Makefile + +Update version information in the package Makefile: + +- **`PKG_VERSION`**: Set to the upstream version (e.g., `4.1.5` → `4.5.5`) +- **`PKG_RELEASE`**: Reset to `1` if the upstream version changed, or increment if only applying new local patches + +Example transitions: +```makefile +# Before: 4.1.5 with 9 local releases +PKG_VERSION:=4.1.5 +PKG_RELEASE:=9 + +# After updating to 4.5.5: +PKG_VERSION:=4.5.5 +PKG_RELEASE:=1 + +# Or if staying at 4.1.5 but adding a new local patch: +PKG_VERSION:=4.1.5 +PKG_RELEASE:=10 +``` + +### Step 5: Apply the changes + +For each file in the diff: + +1. **If it's Category A (safe upstream change)**: Copy the upstream version directly +2. **If it's Category B (local customization)**: Leave unchanged +3. **If it's Category C (conflict)**: Manually edit to merge both approaches + +Common files to check: +- `Makefile` — version and release +- `files/*.sh` — main scripts (often have conflicts) +- `files/*.conf` — configuration templates +- `files/*.init` — OpenWrt init scripts +- `files/README.md` — documentation + +### Step 6: Re-run the diff for verification + +After making changes, re-run: + +```bash +scripts/fetch-upstream.sh +``` + +The diff should now only show your **intentional** customizations. Review it one more time: +- Do all remaining differences make sense? +- Are there any accidental changes that slipped in? +- Should any new upstream code be adopted? + +### Step 7: Update documentation (if significant changes) + +If the package's behavior changes, update the package's `README.md` or comments to document: +- The version it's based on (e.g., "Based on upstream adblock 4.5.5") +- A summary of local customizations (e.g., "Adds automatic DNS reload on UCI changes") +- Any compatibility notes with the controller or UI + +## Common Scenarios + +### Scenario: Small upstream bugfix, large local customization + +**Situation:** adblock 4.1.5→4.1.6 (just a bugfix), but you've heavily customized the script for NethSecurity. + +**Steps:** +1. Run diff with `v24.10.5` tag +2. See the bugfix hunk in `files/adblock.sh` +3. Merge the bugfix into your customized version +4. Update Makefile: `PKG_VERSION:=4.1.6 PKG_RELEASE:=1` +5. Re-run diff — should show only your customizations remain + +### Scenario: Multiple version jumps (e.g., 4.1.5 → 4.5.5) + +**Situation:** You skipped several releases and want to jump to the latest. + +**Steps:** +1. Run diff with `HEAD` (latest packages feed) +2. Review all changes — likely many new features and bugfixes +3. Categorize each: adopt, preserve, or merge +4. Update Makefile: `PKG_VERSION:=4.5.5 PKG_RELEASE:=1` +5. Test extensively — more changes = higher risk +6. Consider running against intermediate versions to ease the transition + +### Scenario: Package has local patches in `patches/feeds/packages/` + +**Situation:** You have patches for the upstream package (in `patches/feeds/packages/`), and the upstream has evolved. + +**Steps:** +1. Run diff to see what's different +2. Review whether your patches are still needed (upstream might have fixed the issue) +3. If needed, regenerate the patches from the upstream source: + ```bash + cd assets/packages/ + git diff HEAD^ HEAD > /tmp/new-patch.diff + ``` +4. Compare with your current patch; update if necessary +5. Test the patched version in a build + +## Edge Cases + +- **Package renamed upstream**: The script won't find it. Manually verify it still exists in the feed under a new name. +- **Package removed from upstream**: You're now the sole maintainer. Update documentation and consider contributing upstream. +- **Multiple packages with same name**: The script finds the first one. Use `find` manually to disambiguate. +- **Large binary files**: The diff may be very large. Use `diff -r --exclude="*.bin" ...` if needed. + +## Reference Files + +See [REFERENCE.md](references/REFERENCE.md) for: +- Shell script explanation of `fetch-upstream.sh` +- OpenWrt feeds.conf.default format +- Git workflow for managing local patches +- Diff interpretation guide diff --git a/.agents/skills/openwrt-package-update/assets/.gitignore b/.agents/skills/openwrt-package-update/assets/.gitignore new file mode 100644 index 000000000..c96a04f00 --- /dev/null +++ b/.agents/skills/openwrt-package-update/assets/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore \ No newline at end of file diff --git a/.agents/skills/openwrt-package-update/references/REFERENCE.md b/.agents/skills/openwrt-package-update/references/REFERENCE.md new file mode 100644 index 000000000..277ac78aa --- /dev/null +++ b/.agents/skills/openwrt-package-update/references/REFERENCE.md @@ -0,0 +1,219 @@ +# Reference: openwrt-package-update Skill + +## fetch-upstream.sh Script Details + +### Purpose +The `fetch-upstream.sh` script automates fetching upstream package code and generating a diff for manual review. + +### Workflow + +1. **Argument parsing**: Takes ``, ``, `` +2. **Fetch feeds.conf.default**: If a tag is provided, curl the feeds.conf.default from that OpenWrt release +3. **Extract hash**: Parse the `src-git packages` line to find the pinned commit hash after `^` +4. **Clone/update feed**: Clone the GitHub mirror of openwrt/packages or update an existing clone +5. **Checkout hash**: `git checkout` the extracted hash (or HEAD if no hash provided) +6. **Locate package**: `find` the package directory (up to 3 levels deep) +7. **Generate diff**: Run `diff -ru` between upstream and local + +### Output Format + +``` +=== Step 1: Fetching feeds.conf.default from OpenWrt === +... +=== Step 2: Cloning/updating packages feed to === +... +=== Step 3: Locating package '' in upstream feed === +... +=== Step 4: Comparing upstream vs local package === +... +--- Diff (upstream vs local) --- +[diff output] +... +=== Diff Summary === +... +``` + +### Exit Codes +- `0`: Success +- `1`: Missing arguments, network error, or package not found + +### Asset Management + +The script clones the packages feed into `.agents/skills/openwrt-package-update/assets/packages/` which is added to `.gitignore`. This folder: +- Grows over time as more packages are cloned +- Can be manually deleted to save space (script will re-clone on next run) +- Is not committed to git +- Is safe to delete between sessions + +### Network Requirements + +The script needs: +- `curl` to fetch feeds.conf.default from GitHub +- `git` to clone and checkout the packages repository +- Network access to github.com (both GitHub CDN and Git protocol) + +Firewall/corporate proxy rules should allow: +- `https://raw.githubusercontent.com/openwrt/openwrt/...` +- `https://github.com/openwrt/packages.git` (or git:// protocol) + +## OpenWrt feeds.conf.default Format + +The feeds.conf.default file lists the official feeds: + +``` +src-git packages https://git.openwrt.org/feed/packages.git^ +src-git luci https://git.openwrt.org/project/luci.git^ +... +``` + +Each line format: +- `src-git` — source control type (Git) +- `` — feed name (e.g., `packages`, `luci`) +- `` — Git repository URL +- `^` — (optional) pinned commit hash + +The hash ensures reproducible builds by locking to a specific point in time. + +## Diff Interpretation Guide + +When reviewing the script's diff output: + +### Common diff symbols: +- `---` — upstream file +- `+++` — local file +- `@@` — hunk header showing line numbers +- `-` (red) — line removed in local (upstream has it, you removed it) +- `+` (green) — line added in local (you added this, upstream doesn't have it) + +### Example: Local customization + +```diff +--- a/files/adblock.sh ++++ b/files/adblock.sh +@@ -10,6 +10,8 @@ + UPSTREAM_VERSION="4.1.5" + + # Initialize DNS backend ++# NethSecurity: Force dnsmasq backend ++DNS_BACKEND="dnsmasq" + init_dns_backend() { +``` + +Here: +- Upstream has no hardcoded `DNS_BACKEND` +- Local (your fork) forces `dnsmasq` +- This is a **local customization** to preserve + +### Example: Pure upstream change + +```diff +--- a/files/adblock.sh ++++ b/files/adblock.sh +@@ -234,6 +234,8 @@ + log "Error: Invalid source URL" + return 1 + fi ++ # Upstream fix: sanitize URL before processing ++ sanitize_url "$source_url" + process_source "$source_url" +``` + +Here: +- Upstream added a sanitization step +- Neither the old nor new version is in your local file +- This is a **pure upstream improvement** to adopt + +## Git Workflow for Patches + +If your package has patches in `patches/feeds/packages/`: + +### Understand the current patch +```bash +cd assets/packages/ +git log --oneline -5 # See recent changes +git show HEAD # See the latest change +cat /path/to/your/patches/feeds/packages/*-.patch # Your patch +``` + +### Check if patch still applies +```bash +cd assets/packages/ +git apply --check /path/to/your/patch.patch +``` + +### Regenerate patch if needed +```bash +cd assets/packages/ +# Make your changes to files +git diff > /tmp/regenerated.patch +# Copy to repo +cp /tmp/regenerated.patch /path/to/patches/feeds/packages/ +``` + +## Decision Tree for Categorizing Diffs + +``` +For each hunk in the diff: + +1. Is this hunk in a file you modified locally? + NO → Category A (pure upstream, safe to apply) + YES → go to 2 + +2. Does this hunk match a specific customization documented in README.md or comments? + YES → Category B (intentional customization, preserve) + NO → go to 3 + +3. Does this hunk conflict with your customization (same lines touched)? + YES → Category C (conflict, needs manual merge) + NO → Category B (you customized different parts, keep both) +``` + +## Example: Updating adblock 4.1.5 → 4.5.5 + +**Preparation:** +```bash +scripts/fetch-upstream.sh HEAD adblock ../../../packages/adblock +``` + +**Review output:** +- Makefile version bumped from 4.1.5 to 4.5.5 (upstream) +- New categories in `adblock.categories` (upstream) +- New sources in `adblock.sources` (upstream) +- Custom DNS backend forcing in `adblock.sh` (local) +- Custom email notification logic in `adblock.sh` (local) + +**Categorization:** +- Makefile version: Category A (pure upstream) → adopt +- New categories: Category A → adopt +- New sources: Category A → adopt +- Custom DNS backend: Category B → preserve +- Custom email: Category B → preserve + +**Actions:** +1. Copy upstream Makefile, update locally to preserve DNS backend forcing +2. Copy upstream categories file +3. Copy upstream sources file +4. Merge adblock.sh: take upstream as base, reapply the two local customizations +5. Update Makefile PKG_VERSION=4.5.5, PKG_RELEASE=1 +6. Re-run script to verify remaining diffs are just your customizations + +## Troubleshooting + +### Script fails to clone the feed +**Cause:** Network connectivity or git configuration issue +**Solution:** Verify git works: `git clone https://github.com/openwrt/packages.git /tmp/test` + +### Script can't find the package +**Cause:** Package name doesn't match directory name, or doesn't exist in that release +**Solution:** Manually browse the assets folder: `ls assets/packages/net/ | grep ` + +### Diff is huge (thousands of lines) +**Cause:** You're comparing very different versions +**Solution:** +- Try an intermediate version first +- Use `diff -r --exclude-from=.diffignore` to skip binary files +- Split review into categories (Makefile, scripts, configs) + +### Cloned repo is getting too large +**Cause:** Multiple runs accumulate git history +**Solution:** Delete the assets folder: `rm -rf assets/packages/`, script will re-clone on next run diff --git a/.agents/skills/openwrt-package-update/scripts/fetch-upstream.sh b/.agents/skills/openwrt-package-update/scripts/fetch-upstream.sh new file mode 100755 index 000000000..ab7cb2761 --- /dev/null +++ b/.agents/skills/openwrt-package-update/scripts/fetch-upstream.sh @@ -0,0 +1,142 @@ +#!/bin/bash +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# +# Fetch upstream OpenWrt package and compare with local fork. +# +# Usage: ./fetch-upstream.sh +# +# Arguments: +# openwrt-tag - OpenWrt release tag (e.g., v24.10.5) or literal 'HEAD' for latest +# package-name - Name of the package (e.g., 'adblock', 'mwan3') +# local-package-dir - Path to the local forked package (relative or absolute) +# +# Examples: +# ./fetch-upstream.sh v24.10.5 adblock ../../../packages/adblock +# ./fetch-upstream.sh HEAD mwan3 /absolute/path/to/packages/mwan3 + +set -o pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +ASSETS_DIR="$SCRIPT_DIR/assets/packages" + +OPENWRT_TAG="$1" +PACKAGE_NAME="$2" +LOCAL_PKG_DIR="$3" + +# Validate arguments +if [[ -z "$OPENWRT_TAG" || -z "$PACKAGE_NAME" || -z "$LOCAL_PKG_DIR" ]]; then + echo "ERROR: Missing arguments" + echo "Usage: $0 " + exit 1 +fi + +# Resolve local package directory to absolute path +if [[ ! "$LOCAL_PKG_DIR" = /* ]]; then + LOCAL_PKG_DIR="$(cd "$LOCAL_PKG_DIR" 2>/dev/null && pwd)" || { + echo "ERROR: Local package directory does not exist: $LOCAL_PKG_DIR" + exit 1 + } +else + if [[ ! -d "$LOCAL_PKG_DIR" ]]; then + echo "ERROR: Local package directory does not exist: $LOCAL_PKG_DIR" + exit 1 + fi +fi + +FEEDS_URL="https://raw.githubusercontent.com/openwrt/openwrt" +PACKAGES_REPO="https://github.com/openwrt/packages.git" +TARGET_HASH="" + +# Step 1: Determine the packages feed hash +echo "=== Step 1: Fetching feeds.conf.default from OpenWrt $OPENWRT_TAG ===" + +if [[ "$OPENWRT_TAG" == "HEAD" ]]; then + echo "Using HEAD of packages feed (no pinned commit hash)" + TARGET_HASH="HEAD" +else + FEEDS_CONF_URL="$FEEDS_URL/$OPENWRT_TAG/feeds.conf.default" + echo "Fetching: $FEEDS_CONF_URL" + + FEEDS_CONF=$(curl -sS "$FEEDS_CONF_URL" 2>&1) || { + echo "ERROR: Failed to fetch feeds.conf.default from $FEEDS_CONF_URL" + echo "$FEEDS_CONF" + exit 1 + } + + # Extract the packages feed line and parse the hash after '^' + PACKAGES_LINE=$(echo "$FEEDS_CONF" | grep "^src-git packages") + if [[ -z "$PACKAGES_LINE" ]]; then + echo "ERROR: Could not find 'src-git packages' line in feeds.conf.default" + exit 1 + fi + + # Hash is after the '^' character if present + TARGET_HASH=$(echo "$PACKAGES_LINE" | sed -n 's/.*\^\([a-f0-9]*\).*/\1/p') + + if [[ -z "$TARGET_HASH" ]]; then + echo "WARNING: No pinned commit hash found in feeds.conf.default" + echo " Line: $PACKAGES_LINE" + echo " Using HEAD instead" + TARGET_HASH="HEAD" + else + echo "Found packages feed hash: $TARGET_HASH" + fi +fi + +# Step 2: Clone or update the packages feed +echo "" +echo "=== Step 2: Cloning/updating packages feed to $ASSETS_DIR ===" + +if [[ -d "$ASSETS_DIR/.git" ]]; then + echo "Repository already exists, fetching latest..." + cd "$ASSETS_DIR" + git fetch origin "$TARGET_HASH" 2>&1 | grep -v "^From https" | grep -v "^ " || true +else + echo "Cloning packages feed..." + mkdir -p "$ASSETS_DIR" + git clone --depth 1 "$PACKAGES_REPO" "$ASSETS_DIR" 2>&1 | grep -E "(Cloning|fatal)" || true + cd "$ASSETS_DIR" +fi + +# Checkout the target hash/branch +echo "Checking out $TARGET_HASH..." +git checkout "$TARGET_HASH" 2>&1 | grep -v "^Already on" || true + +# Step 3: Find the package directory in the upstream feed +echo "" +echo "=== Step 3: Locating package '$PACKAGE_NAME' in upstream feed ===" + +UPSTREAM_PKG_DIR=$(find "$ASSETS_DIR" -maxdepth 3 -type d -name "$PACKAGE_NAME" 2>/dev/null | head -1) + +if [[ -z "$UPSTREAM_PKG_DIR" ]]; then + echo "ERROR: Package '$PACKAGE_NAME' not found in upstream feed" + echo "Searched in: $ASSETS_DIR" + exit 1 +fi + +echo "Found upstream package at: $UPSTREAM_PKG_DIR" + +# Step 4: Compare the directories +echo "" +echo "=== Step 4: Comparing upstream vs local package ===" +echo "" +echo "Upstream: $UPSTREAM_PKG_DIR" +echo "Local: $LOCAL_PKG_DIR" +echo "" +echo "--- Diff (upstream vs local) ---" +echo "" + +diff -ru "$UPSTREAM_PKG_DIR" "$LOCAL_PKG_DIR" || true + +echo "" +echo "=== Diff Summary ===" +echo "If no changes are shown above, the local package matches upstream." +echo "" +echo "To update the local package:" +echo " 1. Review the diff above carefully" +echo " 2. Copy relevant files from: $UPSTREAM_PKG_DIR" +echo " 3. Preserve intentional local customizations" +echo " 4. Update PKG_VERSION and PKG_RELEASE in Makefile" +echo " 5. Re-run this script to verify changes" diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index eae1f39af..c52f863b8 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -11,10 +11,8 @@ on: - 'files/**' - 'packages/**' - 'patches/**' - - 'build.conf.example' + - 'build.conf.defaults' - 'build-nethsec.sh' - tags: - - '*' pull_request: paths: - 'builder/**' @@ -22,7 +20,7 @@ on: - 'files/**' - 'packages/**' - 'patches/**' - - 'build.conf.example' + - 'build.conf.defaults' - 'build-nethsec.sh' jobs: @@ -33,8 +31,8 @@ jobs: NETHSECURITY_VERSION: ${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} REPO_CHANNEL: ${{ steps.build_vars.outputs.REPO_CHANNEL }} env: - USIGN_PUB_KEY: ${{ secrets.USIGN_PUB_KEY }} - USIGN_PRIV_KEY: ${{ secrets.USIGN_PRIV_KEY }} + APK_PUB_KEY: ${{ secrets.APK_PUB_KEY }} + APK_PRIV_KEY: ${{ secrets.APK_PRIV_KEY }} GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} steps: @@ -42,32 +40,34 @@ jobs: - name: Generate build variables id: build_vars run: | - # export OWRT_VERSION from build.conf.example - echo "OWRT_VERSION=$(grep -oP 'OWRT_VERSION=\K.*' build.conf.example)" >> $GITHUB_OUTPUT + # export OWRT_VERSION from build.conf.defaults + echo "OWRT_VERSION=$(grep -oP 'OWRT_VERSION=\K.*' build.conf.defaults)" >> $GITHUB_OUTPUT - # export TARGET from build.conf.example - echo "TARGET=$(grep -oP 'TARGET=\K.*' build.conf.example)" >> $GITHUB_OUTPUT + # export TARGET from build.conf.defaults + echo "TARGET=$(grep -oP 'TARGET=\K.*' build.conf.defaults)" >> $GITHUB_OUTPUT - # export NETHSECURITY_VERSION from build - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)" >> $GITHUB_OUTPUT - - # When pushing a tag, set REPO_CHANNEL to stable - if [[ "${{ github.ref }}" == refs/tags/* ]]; then - echo "REPO_CHANNEL=stable" >> $GITHUB_OUTPUT - # save NETHSECURITY_VERSION to env - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)" >> $GITHUB_OUTPUT + # export base NETHSECURITY_VERSION from build.conf.defaults + BASE_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.defaults) + COMMIT_HASH=$(git rev-parse --short HEAD) + TIMESTAMP=$(date +'%Y%m%d%H%M%S') + echo "NETHSECURITY_VERSION=${BASE_VERSION}" >> $GITHUB_OUTPUT # When pushing to main branch, set REPO_CHANNEL to dev - elif [[ "${{ github.ref }}" == refs/heads/main ]]; then + if [[ "${{ github.ref }}" == refs/heads/main ]]; then echo "REPO_CHANNEL=dev" >> $GITHUB_OUTPUT - # save NETHSECURITY_VERSION to env and append -dev to it - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)-dev+$(git rev-parse --short HEAD).$(date +'%Y%m%d%H%M%S')" >> $GITHUB_OUTPUT + echo "BUILD_SEMVER_SUFFIX=-dev.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT + + # For pull requests and workflow_dispatch, add timestamp suffix + elif [[ "${{ github.event_name }}" == 'pull_request' ]]; then + BRANCH_NAME="${{ github.head_ref }}" + echo "REPO_CHANNEL=${BRANCH_NAME}" >> $GITHUB_OUTPUT + echo "BUILD_SEMVER_SUFFIX=-${BRANCH_NAME}.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT + + elif [[ "${{ github.event_name }}" == 'workflow_dispatch' ]]; then + BRANCH_NAME=$(echo "${{ github.ref }}" | sed 's|refs/heads/||') + echo "REPO_CHANNEL=${BRANCH_NAME}" >> $GITHUB_OUTPUT + echo "BUILD_SEMVER_SUFFIX=-.${TIMESTAMP}.${COMMIT_HASH}" >> $GITHUB_OUTPUT - # Otherwise, get the branch name of the PR pushing if REPO_CHANNEL is not set - elif [[ "${{ github.event_name }}" == 'pull_request' && ! -v REPO_CHANNEL ]]; then - echo "REPO_CHANNEL=${{ github.head_ref }}" >> $GITHUB_OUTPUT - # save NETHSECURITY_VERSION to env and append last commit hash to it - echo "NETHSECURITY_VERSION=$(grep -oP 'NETHSECURITY_VERSION=\K.*' build.conf.example)-${{ github.head_ref }}+$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT fi - name: Build the image env: @@ -75,12 +75,14 @@ jobs: NETHSECURITY_VERSION: ${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} REPO_CHANNEL: ${{ steps.build_vars.outputs.REPO_CHANNEL }} TARGET: ${{ steps.build_vars.outputs.TARGET }} + BUILD_SEMVER_SUFFIX: ${{ steps.build_vars.outputs.BUILD_SEMVER_SUFFIX }} run: ./build-nethsec.sh - name: Update latest_release file run: | - # Create release file pointing to 8-VERSION - echo "${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}" > latest_release - echo "::notice title='Image published':: ${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}" + # Create release file with the full display version (base + suffix) + FULL_VERSION="${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}${{ steps.build_vars.outputs.BUILD_SEMVER_SUFFIX }}" + echo "${FULL_VERSION}" > latest_release + echo "::notice title='Image published':: ${FULL_VERSION}" - uses: actions/upload-artifact@v7 name: Upload image with: @@ -117,8 +119,14 @@ jobs: RCLONE_CONFIG_REPO_ACCESS_KEY_ID: ${{ secrets.DO_SPACE_ACCESS_KEY }} RCLONE_CONFIG_REPO_SECRET_ACCESS_KEY: ${{ secrets.DO_SPACE_SECRET_KEY }} run: | + # Sync binaries to channel/version/ rclone sync bin/ repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.NETHSECURITY_VERSION }} --progress --create-empty-src-dirs - rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/ --progress --create-empty-src-dirs + + # Place latest_release inside the version directory + rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/${{ steps.build_vars.outputs.NETHSECURITY_VERSION }}/ --progress + + # Also place latest_release at channel root as fallback + rclone copy latest_release repo:nethsecurity/${{ steps.build_vars.outputs.REPO_CHANNEL }}/ --progress tools: name: 'Run tools' diff --git a/.gitignore b/.gitignore index f54bf6bc0..40665e2e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,8 @@ -key-build* -build.conf.override +private-key.pem +public-key.pem /bin build-logs build.conf netify-flow-actions netify-agent-stats-plugin +scripts/netifyd-apks diff --git a/AGENTS.md b/AGENTS.md index 6425615b1..6f3be5b70 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -12,6 +12,7 @@ This project has domain-specific skills available. You MUST activate the relevan - `ns-api` — **ACTIVATE** when writing, modifying, or reviewing Python RPCD API scripts. Triggers: creating or updating `ns.*` RPCD API endpoints, handling UCI configuration changes, managing pre/post-commit hooks, defining ACL permissions, documenting methods in OpenAPI 3.1.0, or when user mentions ns-api, API endpoints, hooks, or references `/usr/libexec/rpcd/ns.` files. Covers stdin/stdout JSON protocol, error handling, naming conventions, code style, and spec file updates. - `openwrt-package` — **ACTIVATE** when creating or modifying OpenWrt `ns-*` packages. Triggers: building new packages for NethSecurity, managing package dependencies, patching upstream feeds, modifying Makefiles, or when user mentions Makefile, package structure, config fragments, or upstream patches. Covers naming conventions, required Makefile fields, architecture selection, external version management, and patch workflows. +- `openwrt-package-update` — **ACTIVATE** when updating forked OpenWrt upstream packages to newer versions. Triggers: updating packages like adblock, mwan3, or other non-ns- prefixed upstream packages, comparing local forks against the canonical OpenWrt packages feed, or when user mentions upstream package updates and merging improvements. Covers version updates, preserving local customizations, and identifying upstream improvements. - `python-nethsecurity` — **ACTIVATE** when writing or modifying Python scripts for NethSecurity packages. Triggers: creating new Python scripts, configuring package build systems, writing utilities, or when user mentions Python code in packages/ns-* or references Python scripts in the NethSecurity package tree. Covers shebang, license headers, extension handling, ruff compliance, available modules, and UCI commit conventions. --- diff --git a/build-nethsec.sh b/build-nethsec.sh index 9e570ea92..000cb2100 100755 --- a/build-nethsec.sh +++ b/build-nethsec.sh @@ -7,10 +7,16 @@ set -e -# Source build files if it exists +# Source versioned defaults first set -o allexport +if [ -f build.conf.defaults ]; then + echo "Loading build.conf.defaults..." + . ./build.conf.defaults +fi + +# Source local overrides second (can override anything) if [ -f build.conf ]; then - echo "Loading build.conf file..." + echo "Loading build.conf (local overrides)..." . ./build.conf fi set +o allexport @@ -20,10 +26,11 @@ OWRT_VERSION=${OWRT_VERSION:?Missing OWRT_VERSION environment variable} NETHSECURITY_VERSION=${NETHSECURITY_VERSION:?Missing NETHSECURITY_VERSION environment variable} REPO_CHANNEL=${REPO_CHANNEL:-dev} TARGET=${TARGET:-x86_64} +BUILD_SEMVER_SUFFIX=${BUILD_SEMVER_SUFFIX:-} -if [ -f "./key-build" ] && [ -f "./key-build.pub" ]; then - USIGN_PRIV_KEY="$(cat ./key-build)" - USIGN_PUB_KEY="$(cat ./key-build.pub)" +if [ -f "./private-key.pem" ] && [ -f "./public-key.pem" ]; then + APK_PRIV_KEY="$(cat ./private-key.pem)" + APK_PUB_KEY="$(cat ./public-key.pem)" fi @@ -32,20 +39,20 @@ podman build \ --layers \ --file builder/Containerfile \ --tag nethsecurity-next \ - --target builder \ --jobs 0 \ --build-arg OWRT_VERSION="$OWRT_VERSION" \ --build-arg REPO_CHANNEL="$REPO_CHANNEL" \ --build-arg TARGET="$TARGET" \ --build-arg NETHSECURITY_VERSION="$NETHSECURITY_VERSION" \ + --build-arg BUILD_SEMVER_SUFFIX="$BUILD_SEMVER_SUFFIX" \ . set +e status=0 podman run \ - --env USIGN_PRIV_KEY="$USIGN_PRIV_KEY" \ - --env USIGN_PUB_KEY="$USIGN_PUB_KEY" \ + --env APK_PRIV_KEY="$APK_PRIV_KEY" \ + --env APK_PUB_KEY="$APK_PUB_KEY" \ --name nethsecurity-builder \ --interactive \ --tty \ diff --git a/build.conf.defaults b/build.conf.defaults new file mode 100644 index 000000000..15e0198cd --- /dev/null +++ b/build.conf.defaults @@ -0,0 +1,4 @@ +OWRT_VERSION=v25.12.3 +NETHSECURITY_VERSION=8.8.0 +TARGET=x86_64 +REPO_CHANNEL=dev diff --git a/build.conf.example b/build.conf.example deleted file mode 100644 index 0f05f1e39..000000000 --- a/build.conf.example +++ /dev/null @@ -1,4 +0,0 @@ -OWRT_VERSION=v24.10.5 -NETHSECURITY_VERSION=8.7.2 -TARGET=x86_64 -REPO_CHANNEL=dev diff --git a/builder/Containerfile b/builder/Containerfile index 351dda0ef..7d44137ae 100644 --- a/builder/Containerfile +++ b/builder/Containerfile @@ -1,53 +1,59 @@ # -# Copyright (C) 2025 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # -# 2025-06-30 -FROM debian:12.11 AS base -ARG SOURCE_DATE_EPOCH=1751241600 +FROM debian:13.4 AS base RUN apt-get update \ && apt-get install --yes --no-install-recommends --no-install-suggests \ + # openwrt build dependencies https://openwrt.org/docs/guide-developer/toolchain/install-buildsystem#debianubuntumint bison \ build-essential \ - ca-certificates \ clang \ - cmake \ - curl \ file \ flex \ g++ \ + g++-multilib \ gawk \ gcc-multilib \ gettext \ git \ libncurses5-dev \ libssl-dev \ - python3-distutils \ + python3-setuptools \ rsync \ - sudo \ + swig \ unzip \ - vim \ wget \ - zlib1g-dev - -FROM base AS usign_build -RUN git clone --depth 1 https://git.openwrt.org/project/usign.git /tmp/usign \ - && cd /tmp/usign \ - && cmake . \ - && make + zlib1g-dev \ + # other dependencies + ca-certificates \ + cmake \ + curl \ + less \ + quilt \ + sudo \ + vim -FROM base AS builder RUN groupadd -g 1000 'buildbot' \ && useradd -m -s '/bin/bash' -u 1000 -g 1000 'buildbot' \ - && echo 'buildbot ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/buildbot + && echo 'buildbot ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/buildbot \ + && echo "QUILT_DIFF_ARGS=--no-timestamps --no-index -p ab --color=auto" \ + "QUILT_REFRESH_ARGS=--no-timestamps --no-index -p ab" \ + "QUILT_SERIES_ARGS=--color=auto" \ + "QUILT_PATCH_OPTS=--unified" \ + "QUILT_DIFF_OPTS=-p" \ + "EDITOR=vim" \ + > /home/buildbot/.quiltrc USER buildbot WORKDIR /home/buildbot # OpenWRT repo ARG OWRT_VERSION RUN git clone --branch "${OWRT_VERSION}" https://github.com/openwrt/openwrt.git WORKDIR /home/buildbot/openwrt -RUN sed -i '/telephony/d' feeds.conf.default +RUN sed -i '/telephony/d' feeds.conf.default \ + && sed -i '/routing/d' feeds.conf.default \ + && sed -i '/video/d' feeds.conf.default RUN ./scripts/feeds update -a COPY --chmod=777 builder/apply-patches.sh /usr/local/bin/apply-patches COPY --chown=buildbot:buildbot patches patches @@ -63,19 +69,19 @@ COPY --chown=buildbot:buildbot config config ARG REPO_CHANNEL ARG TARGET ARG NETHSECURITY_VERSION +ARG BUILD_SEMVER_SUFFIX COPY --chmod=777 builder/configure-build.sh /usr/local/bin/configure-build RUN /usr/local/bin/configure-build COPY --chmod=777 builder/entrypoint.sh /usr/local/bin/entrypoint.sh ENTRYPOINT [ "/usr/local/bin/entrypoint.sh" ] -COPY --from=usign_build /tmp/usign/usign /usr/local/bin/usign RUN mkdir -p \ - .ccache \ - build_dir \ - dl \ - download \ - staging_dir + .ccache \ + build_dir \ + dl \ + download \ + staging_dir VOLUME "/home/buildbot/openwrt/.ccache" \ - "/home/buildbot/openwrt/build_dir" \ - "/home/buildbot/openwrt/dl" \ - "/home/buildbot/openwrt/download" \ - "/home/buildbot/openwrt/staging_dir" + "/home/buildbot/openwrt/build_dir" \ + "/home/buildbot/openwrt/dl" \ + "/home/buildbot/openwrt/download" \ + "/home/buildbot/openwrt/staging_dir" diff --git a/builder/configure-build.sh b/builder/configure-build.sh index 8f1cf189a..f21b34fe1 100644 --- a/builder/configure-build.sh +++ b/builder/configure-build.sh @@ -12,6 +12,13 @@ nethsecurity_version=${NETHSECURITY_VERSION:?Missing NETHSECURITY_VERSION enviro repo_channel=${REPO_CHANNEL:?Missing REPO_CHANNEL environment variable} target=${TARGET:?Missing TARGET environment variable} owrt_version=${OWRT_VERSION:?Missing OWRT_VERSION environment variable} +build_semver_suffix=${BUILD_SEMVER_SUFFIX:-} + +if [ -n "$build_semver_suffix" ]; then + image_version="${nethsecurity_version}${build_semver_suffix}" +else + image_version="${nethsecurity_version}" +fi # For each file inside the config directory, cat the content into a .config file for file in config/*.conf; do @@ -31,7 +38,7 @@ CONFIG_VERSION_DIST="NethSecurity" CONFIG_VERSION_HOME_URL="https://github.com/nethserver/nethsecurity" CONFIG_VERSION_MANUFACTURER="Nethesis" CONFIG_VERSION_MANUFACTURER_URL="https://www.nethesis.it" -CONFIG_VERSION_NUMBER="${nethsecurity_version}" +CONFIG_VERSION_NUMBER="${image_version}" CONFIG_VERSION_CODE="${owrt_version}" CONFIG_VERSION_PRODUCT="NethSecurity" CONFIG_VERSION_REPO="https://updates.nethsecurity.nethserver.org/${repo_channel}/${nethsecurity_version}" diff --git a/builder/entrypoint.sh b/builder/entrypoint.sh index ff3b2a27f..09f4f5409 100644 --- a/builder/entrypoint.sh +++ b/builder/entrypoint.sh @@ -7,12 +7,9 @@ set -e -if [ -n "$USIGN_PUB_KEY" ] && [ -n "$USIGN_PRIV_KEY" ]; then - echo "$USIGN_PUB_KEY" > /home/buildbot/openwrt/key-build.pub - echo "$USIGN_PRIV_KEY" > /home/buildbot/openwrt/key-build -else - echo "No signing keys found. Generating dummy keys..." - usign -G -s ./key-build -p ./key-build.pub -c "Local build key" +if [ -n "$APK_PRIV_KEY" ] && [ -n "$APK_PUB_KEY" ]; then + echo "$APK_PRIV_KEY" > /home/buildbot/openwrt/private-key.pem + echo "$APK_PUB_KEY" > /home/buildbot/openwrt/public-key.pem fi # if command $1 is a file or a executable, run it diff --git a/config/avahi.conf b/config/avahi.conf new file mode 100644 index 000000000..c06fb1e5c --- /dev/null +++ b/config/avahi.conf @@ -0,0 +1 @@ +CONFIG_PACKAGE_avahi-nodbus-daemon=m diff --git a/config/monitoring.conf b/config/monitoring.conf new file mode 100644 index 000000000..f5e4fa381 --- /dev/null +++ b/config/monitoring.conf @@ -0,0 +1,3 @@ +CONFIG_PACKAGE_victoria-metrics=y +CONFIG_PACKAGE_victoria-logs=m +CONFIG_PACKAGE_telegraf=y diff --git a/docs/build/index.md b/docs/build/index.md index 1afbbc4dc..a0c4e9ea9 100644 --- a/docs/build/index.md +++ b/docs/build/index.md @@ -39,7 +39,15 @@ By default, the CI will build the `x86_64` target. To build a different target, To build locally, it's recommended to populate the `build.conf` file with the options you want to use for the build. This file is ignored by Git and should not be committed to the repository. -You can use the `build.conf.example` file as a starting point. Refer to [Environment variables](#environment-variables) for more details on the available options. +The `build.conf.defaults` file contains the versioned defaults and is always tracked by Git. + +You can create a local `build.conf` override that inherits from `build.conf.defaults`: +```bash +cp build.conf.defaults build.conf +# Edit build.conf to override any variables as needed +``` + +Refer to [Environment variables](#environment-variables) for more details on the available options. To build images locally on your machine, make sure these minimum requirements are met: @@ -65,23 +73,30 @@ During the start-up, the container will download netifyd plugins if configuratio ### Environment variables -The `build-nethsec.sh` script behavior can be changed by giving the following environment variables or setting them inside the `build.conf` file: +The `build-nethsec.sh` script behavior can be changed by setting environment variables or by populating the `build.conf` file (git-ignored, local overrides only). + +**Variable loading order:** +1. `build.conf.defaults` (versioned, always loaded first — contains canonical defaults) +2. `build.conf` (git-ignored, optional — can override any variable) +3. Environment variables set before calling `./build-nethsec.sh` (highest priority) + +**Available variables:** - `OWRT_VERSION`: specify the OpenWrt version to build, it can be either a TAG or a branch in the [GitHub OpenWRT repo](https://github.com/openwrt/openwrt); **required** - `NETHSECURITY_VERSION`: specify what to call the NethSecurity image; **required** - `TARGET`: specify the target to build; if not set default is `x86_64` - `REPO_CHANNEL`: specify the channel to publish the image to; if not set default is `dev` -- `USIGN_PUB_KEY` and `USIGN_PRIV_KEY`: see [package signing section](#package-signing) - with the given keys +- `BUILD_SEMVER_SUFFIX`: optional semver suffix appended to the image version only (not the distfeed URL). Use pre-release format (`-rc.1`, `-beta.2`) or metadata format (`+hotfix.1`, `+testing`) or both (`-rc.1+fix.1`). +- `APK_PUB_KEY` and `APK_PRIV_KEY`: see [package signing section](#package-signing) -The `USIGN_PUB_KEY`, `USIGN_PRIV_KEY` variables are always set as secrets inside the CI pipeline, but +The `APK_PUB_KEY`, `APK_PRIV_KEY` variables are always set as secrets inside the CI pipeline, but for [security reasons](https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#accessing-secrets) they are not accessible when building pull requests from forks. ### Build locally for a release If you need to build some packages locally for a release, make sure the following environment variables are set: -- `USIGN_PUB_KEY` and `USIGN_PRIV_KEY`: refer to the [package signing section](#package-signing) for more info +- `APK_PUB_KEY` and `APK_PRIV_KEY`: refer to the [package signing section](#package-signing) for more info Then execute the build as described in the [Build locally](#build-locally) section. @@ -133,7 +148,8 @@ Development version example: ## Upstream version change -To change the OpenWrt version used by NethSecurity, you can just replace the `OWRT_VERSION` variable inside the `build.conf.example` file with the new OpenWrt version. +To change the OpenWrt version used by NethSecurity, update the `OWRT_VERSION` variable inside the `build.conf.defaults` file (versioned, always tracked by Git). +This ensures all developers and CI get the same default version. ## Release new image checklist @@ -256,26 +272,26 @@ To replace an upstream package just create a new package with the same name insi ### Package signing -All packages are signed with the following public key generated with [OpenBSD signify](nethsecurity-pub.key). - -Public key fingerprint: `7640d16662de3b89` +Packages are signed using an EC prime256v1 key pair (PEM format) via the APK package manager. -Public key content: +To generate a new signing key pair: ``` -untrusted comment: NethSecurity sign key -RWR2QNFmYt47ieK7g/zEPwgk+MN8bHsA2vFnPThSpnLZ48L7sh6wxB/f +openssl ecparam -name prime256v1 -genkey -noout -out private-key.pem +openssl ec -in private-key.pem -pubout -out public-key.pem ``` -To sign the packages, just execute the `build-nethsec.sh` script with the following environment variables: -- `USIGN_PUB_KEY` -- `USIGN_PRIV_KEY` +To sign the packages, execute the `build-nethsec.sh` script with the following environment variables: +- `APK_PUB_KEY` +- `APK_PRIV_KEY` Usage example: ``` -USIGN_PUB_KEY=$(cat nethsecurity-pub.key) USIGN_PRIV_KEY=$(cat nethsecurity-priv.key) ./build-nethsec.sh +APK_PUB_KEY=$(cat public-key.pem) APK_PRIV_KEY=$(cat private-key.pem) ./build-nethsec.sh ``` -Or you can have the keys as two files named `key-build` and `key-build.pub` in the root of the repository. They will be automatically used by the build script. +Or you can place the keys as two files named `private-key.pem` and `public-key.pem` in the root of the repository. They will be automatically used by the build script. + +If no keys are provided, OpenWrt will auto-generate a throwaway key pair at build time. Builds executed inside CI will sign the packages with the correct key. diff --git a/files/bin/ipcalc.sh b/files/bin/ipcalc.sh deleted file mode 100755 index e8c7a07df..000000000 --- a/files/bin/ipcalc.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/sh - -. /lib/functions/ipv4.sh - -PROG="$(basename "$0")" - -# wrapper to convert an integer to an address, unless we're using -# decimal output format. -# hook for library function -_ip2str() { - local var="$1" n="$2" - assert_uint32 "$n" || exit 1 - - if [ "$decimal" -ne 0 ]; then - export -- "$var=$n" - elif [ "$hexadecimal" -ne 0 ]; then - export -- "$var=$(printf "%x" "$n")" - else - ip2str "$@" - fi -} - -usage() { - echo "Usage: $PROG [ -d | -x ] address/prefix [ start limit ]" >&2 - exit 1 -} - -decimal=0 -hexadecimal=0 -if [ "$1" = "-d" ]; then - decimal=1 - shift -elif [ "$1" = "-x" ]; then - hexadecimal=1 - shift -fi - -if [ $# -eq 0 ]; then - usage -fi - -case "$1" in -*/*.*) - # data is n.n.n.n/m.m.m.m format, like on a Cisco router - str2ip ipaddr "${1%/*}" || exit 1 - str2ip netmask "${1#*/}" || exit 1 - netmask2prefix prefix "$netmask" || exit 1 - shift - ;; -*/*) - # more modern prefix notation of n.n.n.n/p - str2ip ipaddr "${1%/*}" || exit 1 - prefix="${1#*/}" - assert_uint32 "$prefix" || exit 1 - if [ "$prefix" -gt 32 ]; then - printf "Prefix out of range (%s)\n" "$prefix" >&2 - exit 1 - fi - prefix2netmask netmask "$prefix" || exit 1 - shift - ;; -*) - # address and netmask as two separate arguments - str2ip ipaddr "$1" || exit 1 - str2ip netmask "$2" || exit 1 - netmask2prefix prefix "$netmask" || exit 1 - shift 2 - ;; -esac - -# we either have no arguments left, or we have a range start and length -if [ $# -ne 0 ] && [ $# -ne 2 ]; then - usage -fi - -# complement of the netmask, i.e. the hostmask -hostmask=$((netmask ^ 0xffffffff)) -network=$((ipaddr & netmask)) -broadcast=$((network | hostmask)) -count=$((hostmask + 1)) - -_ip2str IP "$ipaddr" -_ip2str NETMASK "$netmask" -_ip2str NETWORK "$network" - -echo "IP=$IP" -echo "NETMASK=$NETMASK" -# don't include this-network or broadcast addresses -if [ "$prefix" -le 30 ]; then - _ip2str BROADCAST "$broadcast" - echo "BROADCAST=$BROADCAST" -fi -echo "NETWORK=$NETWORK" -echo "PREFIX=$prefix" -echo "COUNT=$count" - -# if there's no range, we're done -[ $# -eq 0 ] && exit 0 -[ -z "$1$2" ] && exit 0 - -if [ "$prefix" -le 30 ]; then - lower=$((network + 1)) -else - lower="$network" -fi - -start="$1" -assert_uint32 "$start" || exit 1 -start=$((network | (start & hostmask))) -[ "$start" -lt "$lower" ] && start="$lower" -[ "$start" -eq "$ipaddr" ] && start=$((start + 1)) - -if [ "$prefix" -le 30 ]; then - upper=$(((network | hostmask) - 1)) -elif [ "$prefix" -eq 31 ]; then - upper=$((network | hostmask)) -else - upper="$network" -fi - -range="$2" -assert_uint32 "$range" || exit 1 -end=$((start + range - 1)) -[ "$end" -gt "$upper" ] && end="$upper" -[ "$end" -eq "$ipaddr" ] && end=$((end - 1)) - -if [ "$start" -gt "$end" ]; then - echo "network ($NETWORK/$prefix) too small" >&2 - exit 1 -fi - -_ip2str START "$start" -_ip2str END "$end" - -if [ "$start" -le "$ipaddr" ] && [ "$ipaddr" -le "$end" ]; then - echo "warning: address $IP inside range $START..$END" >&2 -fi - -echo "START=$START" -echo "END=$END" - -exit 0 diff --git a/files/etc/profile.d/busybox-history-file.sh b/files/etc/profile.d/busybox-history-file.sh new file mode 100644 index 000000000..9f2c9ab5f --- /dev/null +++ b/files/etc/profile.d/busybox-history-file.sh @@ -0,0 +1 @@ +export HISTFILE=~/.bash_history diff --git a/packages/checkmk-agent/Makefile b/packages/checkmk-agent/Makefile index 8f33df9f8..30053f9f0 100644 --- a/packages/checkmk-agent/Makefile +++ b/packages/checkmk-agent/Makefile @@ -6,7 +6,9 @@ include $(TOPDIR)/rules.mk PKG_NAME:=checkmk-agent -PKG_VERSION:=2.4.0p24 +# renovate: datasource=github-tags depName=Checkmk/checkmk +CHECKMK_UPSTREAM_VERSION:=2.5.0 +PKG_VERSION:=$(subst p,_p,$(CHECKMK_UPSTREAM_VERSION)) PKG_RELEASE:=1 PKG_BUILD_DIR:=$(BUILD_DIR)/checkmk-agent-$(PKG_VERSION) @@ -32,7 +34,7 @@ endef # Download the Check_MK agent binary define Download/checkmk-agent-binary - URL:=https://raw.githubusercontent.com/Checkmk/checkmk/v$(PKG_VERSION)/agents + URL:=https://raw.githubusercontent.com/Checkmk/checkmk/v$(CHECKMK_UPSTREAM_VERSION)/agents URL_FILE:=check_mk_agent.openwrt FILE:=check_mk_agent.openwrt HASH:=skip diff --git a/packages/netifyd/Makefile b/packages/netifyd/Makefile index 54a598df9..291baa833 100644 --- a/packages/netifyd/Makefile +++ b/packages/netifyd/Makefile @@ -8,8 +8,8 @@ PKG_MAINTAINER:=Darryl Sokoloski PKG_LICENSE:=Unlicensed # Base URL for downloads -NETIFYD_BASE_URL:=https://updates.nethsecurity.nethserver.org/netifyd-dist/netifyd-$(NETIFYD_VERSION)/$(ARCH) -DL_DIR:=$(DL_DIR)/netifyd-$(NETIFYD_VERSION)-$(ARCH) +NETIFYD_BASE_URL:=https://updates.nethsecurity.nethserver.org/netifyd-dist/netifyd-$(PKG_VERSION)/25.12.2/$(ARCH) +DL_DIR:=$(DL_DIR)/netifyd-$(PKG_VERSION)-$(ARCH) include $(INCLUDE_DIR)/package.mk @@ -66,7 +66,7 @@ define Download/libnetifyd URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetifyd.so.4.0.0 FILE:=libnetifyd.so.4.0.0 - HASH:=a7bef78717e200eef177da8ee94557a565828ce796d3fd1a1f078e3f55959dd5 + HASH:=ffbbe5078a2b6575db4c478cf1b4d257f9b7241797c84aa1bcbca17ee1b23f3b endef $(eval $(call Download,libnetifyd)) @@ -74,7 +74,7 @@ define Download/libnetify-plm URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-plm.so.1.0.0 FILE:=libnetify-plm.so.1.0.0 - HASH:=b5c8994dc5f497ef1ccf8aae3685c1541702031726bc9f2819489cacfcb67e8f + HASH:=4320235873539f561238c92094702e74b75b8939a301b694255a5512fa036a00 endef $(eval $(call Download,libnetify-plm)) @@ -82,7 +82,7 @@ define Download/libnetify-proc-aggregator URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-aggregator.so.0.0.0 FILE:=libnetify-proc-aggregator.so.0.0.0 - HASH:=c9e5981fd4410776a1808902fbe284f40882823fc4c711f54e14421c80e94c6b + HASH:=736acb4c22d891da66694eee5507ea70679885f0747dec7a07eec8c926dd76fc endef $(eval $(call Download,libnetify-proc-aggregator)) @@ -90,7 +90,7 @@ define Download/libnetify-proc-core URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-core.so.0.0.0 FILE:=libnetify-proc-core.so.0.0.0 - HASH:=725415498d05fc29695a6f1fe14aed4c6b5967ff0d7d4703d06b80489c852222 + HASH:=8c9a1f26a498a6d1c88d76e6aa0367749779ca5f44ea3fa614bc1814387258ed endef $(eval $(call Download,libnetify-proc-core)) @@ -98,7 +98,7 @@ define Download/libnetify-proc-dev-discovery URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-dev-discovery.so.0.0.0 FILE:=libnetify-proc-dev-discovery.so.0.0.0 - HASH:=c25c4a497925cc0cf9d48f353de76fd2b5c9aa223390c87f2163feb1b0d61efb + HASH:=756c01e22c3724311eb6952ca6d580c728592771948207b93bab9b9795c38d1c endef $(eval $(call Download,libnetify-proc-dev-discovery)) @@ -106,7 +106,7 @@ define Download/libnetify-proc-flow-actions URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-proc-flow-actions.so.0.0.0 FILE:=libnetify-proc-flow-actions.so.0.0.0 - HASH:=06ed65a3ec5d840edff008aaebbe294e4b0f7545ab295b287f6ed286bf28b7d7 + HASH:=1d6f03ead8760e314f98f5fe083515b19bf9c60e720d36bc03f70fa60a9b0d2b endef $(eval $(call Download,libnetify-proc-flow-actions)) @@ -114,7 +114,7 @@ define Download/libnetify-sink-http URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-http.so.0.0.0 FILE:=libnetify-sink-http.so.0.0.0 - HASH:=aaaadb9179f3df215f08d29f3a46db314d3990adafa5f3c4abeff305f4bb9583 + HASH:=95d0d6fa2b47b3764318a2f8680441d126c94bb39c84440b3e056767e330991d endef $(eval $(call Download,libnetify-sink-http)) @@ -122,7 +122,7 @@ define Download/libnetify-sink-log URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-log.so.0.0.0 FILE:=libnetify-sink-log.so.0.0.0 - HASH:=26d136562f7416ce4fe7c70d8023ae3f578a2d2dff4239791c65ad4289d28765 + HASH:=f9cafc30e150109dc3ab5e57eb203d51525a4ad0241fe76724206edf73c575b1 endef $(eval $(call Download,libnetify-sink-log)) @@ -130,7 +130,7 @@ define Download/libnetify-sink-socket URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-socket.so.0.0.0 FILE:=libnetify-sink-socket.so.0.0.0 - HASH:=8772335da50b702edd2a1412c98c82e32058dee2158db6e874ba3964ac66590a + HASH:=7ed9fe8d4d952ccbedca1c2be505fe288080f19c5c60d4db46932078c97d5364 endef $(eval $(call Download,libnetify-sink-socket)) @@ -138,7 +138,7 @@ define Download/libnetify-sink-sqlite URL:=$(NETIFYD_BASE_URL)/usr/lib URL_FILE:=libnetify-sink-sqlite.so.0.0.0 FILE:=libnetify-sink-sqlite.so.0.0.0 - HASH:=6c2879cb8c9da36d592c85b9f8b4042339f0322fe2ebf335a5d15d06562ec65b + HASH:=0de0ef72cbd092fedd5cc278a4b57977891650aa10712f5ae0e6381a5a02fa52 endef $(eval $(call Download,libnetify-sink-sqlite)) @@ -146,7 +146,7 @@ define Download/netifyd URL:=$(NETIFYD_BASE_URL)/usr/sbin URL_FILE:=netifyd FILE:=netifyd - HASH:=02a8647d779f7dc4506fe9dd10295a06f6f1cb7b701edbfe791e6f319c4d7fd3 + HASH:=a1d8f40f87ba58876c7652dc0f82e699d6f8139793b8ede9d33cea043a693c72 endef $(eval $(call Download,netifyd)) diff --git a/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf b/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf index 61266c775..687f9d690 100644 --- a/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf +++ b/packages/netifyd/files/etc/netifyd/profiles.d/00-default.conf @@ -158,5 +158,6 @@ all = include [netlink] # Set the Netlink buffer size buffer_size = 32768 +bridge_pvid_discovery = no # vim: set ft=dosini : diff --git a/packages/ns-api/files/ns.dashboard b/packages/ns-api/files/ns.dashboard index 449fec46e..1e3954eed 100644 --- a/packages/ns-api/files/ns.dashboard +++ b/packages/ns-api/files/ns.dashboard @@ -154,7 +154,7 @@ def check_adblock(): if not adb_enabled: return "disabled" pa = subprocess.run(["service", "adblock", "status"], check=False, capture_output=True, text=True) - if adb_enabled and re.search('adblock_status\s+:\s+enabled', pa.stdout): + if adb_enabled and re.search(r'adblock_status\s+:\s+enabled', pa.stdout): return "ok" else: return "error" @@ -165,7 +165,7 @@ def check_banip(): if not bip_enabled: return "disabled" pa = subprocess.run(["service", "banip", "status"], check=False, capture_output=True, text=True) - if bip_enabled and re.search('status\s+:\s+active', pa.stdout): + if bip_enabled and re.search(r'status\s+:\s+active', pa.stdout): return "ok" def check_ts_ip(): diff --git a/packages/ns-api/files/ns.nathelpers b/packages/ns-api/files/ns.nathelpers index c674078ac..15afe300e 100755 --- a/packages/ns-api/files/ns.nathelpers +++ b/packages/ns-api/files/ns.nathelpers @@ -58,12 +58,12 @@ DEFAULT_PARAMS = { def get_nat_helper_names(): nat_helpers = [] - proc = subprocess.run("/bin/opkg files kmod-nf-nathelper | grep -e '\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, + proc = subprocess.run("/bin/opkg files kmod-nf-nathelper | grep -e '\\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, capture_output=True, text=True) nat_helpers = proc.stdout.splitlines() nat_helpers_extra = [] - proc = subprocess.run("/bin/opkg files kmod-nf-nathelper-extra | grep -e '\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, + proc = subprocess.run("/bin/opkg files kmod-nf-nathelper-extra | grep -e '\\.ko$' | cut -d'/' -f 5 | cut -d'.' -f1", shell=True, check=True, capture_output=True, text=True) nat_helpers_extra = proc.stdout.splitlines() return nat_helpers + nat_helpers_extra diff --git a/packages/ns-api/files/ns.ovpnrw b/packages/ns-api/files/ns.ovpnrw index 147b7e86e..a43e24e10 100755 --- a/packages/ns-api/files/ns.ovpnrw +++ b/packages/ns-api/files/ns.ovpnrw @@ -63,8 +63,7 @@ def add_tap_to_bridge(u, bridge, interface): u.save("network") except: pass - finally: - return + return def remove_tap_from_bridge(u, bridge, interface): if bridge is None or interface is None: @@ -85,8 +84,7 @@ def remove_tap_from_bridge(u, bridge, interface): u.save("network") except: pass - finally: - return + return def get_ip_and_mask(bridge): u = EUci() diff --git a/packages/ns-api/files/ns.redirects b/packages/ns-api/files/ns.redirects index 8356c065c..4af2f2837 100755 --- a/packages/ns-api/files/ns.redirects +++ b/packages/ns-api/files/ns.redirects @@ -21,7 +21,7 @@ def get_services(): line = line.strip() if not line: continue - tmp = re.split("\s+", line) + tmp = re.split(r"\s+", line) port = tmp[1][0:tmp[1].index("/")] services[port] = tmp[0] return services diff --git a/packages/ns-api/files/ns.update b/packages/ns-api/files/ns.update index b1fef000d..9121fdffd 100755 --- a/packages/ns-api/files/ns.update +++ b/packages/ns-api/files/ns.update @@ -86,7 +86,7 @@ def check_system_update(): response = requests.get(f"{url}/latest_release", headers={"Accept": "application/json"}, timeout=5) response.raise_for_status() version = response.text.strip() - if semver.compare(version, current_version) > 0: + if semver.Version.compare(version, current_version) > 0: data["lastVersion"] = f'NethSecurity {version}' except requests.exceptions.ConnectionError: return utils.generic_error("connection_error") diff --git a/packages/ns-migration/files/scripts/openvpn b/packages/ns-migration/files/scripts/openvpn index 06908f81a..09cf6a1b2 100755 --- a/packages/ns-migration/files/scripts/openvpn +++ b/packages/ns-migration/files/scripts/openvpn @@ -62,8 +62,7 @@ def add_tap_to_bridge(u, bridge, interface): u.commit("network") except: pass - finally: - return + return iname="ns_roadwarrior1" diff --git a/packages/python-jinja2/Makefile b/packages/python-jinja2/Makefile deleted file mode 100644 index f29df71ec..000000000 --- a/packages/python-jinja2/Makefile +++ /dev/null @@ -1,50 +0,0 @@ -# This is free software, licensed under the GNU General Public License v2. -# See /LICENSE for more information. -# - -include $(TOPDIR)/rules.mk - -PKG_NAME:=python-jinja2 -PKG_VERSION:=3.1.2 -PKG_RELEASE:=2 - -PYPI_NAME:=Jinja2 -PYTHON3_PKG_WHEEL_NAME:=jinja2 -PKG_HASH:=31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852 - -PKG_MAINTAINER:=Michal Vasilek -PKG_LICENSE:=BSD-3-Clause -PKG_LICENSE_FILES:=LICENSE.rst -PKG_CPE_ID:=cpe:/a:pocoo:jinja2 -HOST_BUILD_DEPENDS:= python-markupsafe/host - -include $(TOPDIR)/feeds/packages/lang/python/pypi.mk -include $(INCLUDE_DIR)/package.mk -include $(INCLUDE_DIR)/host-build.mk -include $(TOPDIR)/feeds/packages/lang/python/python3-package.mk -include $(TOPDIR)/feeds/packages/lang/python/python3-host-build.mk - -define Package/python3-jinja2 - SECTION:=lang - CATEGORY:=Languages - SUBMENU:=Python - TITLE:=Very fast and expressive template engine - URL:=https://palletsprojects.com/p/jinja/ - DEPENDS:= \ - +python3-light \ - +python3-asyncio \ - +python3-logging \ - +python3-urllib \ - +python3-markupsafe -endef - -define Package/python3-jinja2/description -Jinja2 is a full featured template engine for Python. It has full -unicode support, an optional integrated sandboxed execution -environment, widely used and BSD licensed. -endef - -$(eval $(call Py3Package,python3-jinja2)) -$(eval $(call BuildPackage,python3-jinja2)) -$(eval $(call BuildPackage,python3-jinja2-src)) -$(eval $(call HostBuild)) \ No newline at end of file diff --git a/packages/python-semver/Makefile b/packages/python-semver/Makefile index 9132e194e..4affdbf63 100644 --- a/packages/python-semver/Makefile +++ b/packages/python-semver/Makefile @@ -5,11 +5,11 @@ include $(TOPDIR)/rules.mk PKG_NAME:=python-semver -PKG_VERSION:=2.13.0 +PKG_VERSION:=3.0.4 PKG_RELEASE:=1 PYPI_NAME:=semver -PKG_HASH:=fa0fe2722ee1c3f57eac478820c3a5ae2f624af8264cbdf9000c980ff7f75e3f +PKG_HASH:=afc7d8c584a5ed0a11033af086e8af226a9c0b206f313e0301f8dd7b6b589602 PKG_MAINTAINER:=Tommaso Bailetti PKG_LICENSE:=BSD-3-Clause diff --git a/packages/telegraf/Makefile b/packages/telegraf/Makefile new file mode 100644 index 000000000..e63822acb --- /dev/null +++ b/packages/telegraf/Makefile @@ -0,0 +1,90 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +include $(TOPDIR)/rules.mk + +PKG_NAME:=telegraf +# renovate: datasource=github-tags depName=influxdata/telegraf +PKG_VERSION:=1.38.3 +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/influxdata/telegraf/tar.gz/v$(PKG_VERSION)? +PKG_SOURCE_SUBDIR:=telegraf-$(PKG_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) + +PKG_HASH:=skip +PKG_MAINTAINER:=Tommaso Bailetti +PKG_LICENSE:=MIT + +PKG_BUILD_DEPENDS:=golang/host +PKG_BUILD_PARALLEL:=1 +PKG_BUILD_FLAGS:=no-mips16 + +GO_PKG:=github.com/influxdata/telegraf/cmd/$(PKG_NAME) +GO_BUILD_PKG:=github.com/influxdata/telegraf/cmd/$(PKG_NAME) +GO_PKG_LDFLAGS_X:=github.com/influxdata/telegraf/internal.Version=$(PKG_VERSION) +GO_PKG_TAGS:= \ + custom \ + inputs.bond \ + inputs.cpu \ + inputs.disk \ + inputs.ethtool \ + inputs.mem \ + inputs.net \ + inputs.netstat \ + inputs.nstat \ + inputs.processes \ + inputs.sensors \ + inputs.system \ + outputs.influxdb \ + outputs.prometheus_client + +include $(INCLUDE_DIR)/package.mk +include $(TOPDIR)/feeds/packages/lang/golang/golang-package.mk + +define Package/telegraf + SECTION:=base + CATEGORY:=NethServer + TITLE:=Telegraf + URL:=https://github.com/influxdata/telegraf + DEPENDS:= \ + $(GO_ARCH_DEPENDS) \ + +lm-sensors \ + +victoria-metrics \ + +python3-uci \ + +python3-jinja2 +endef + +define Package/telegraf/description + Telegraf is an agent for collecting, processing, aggregating, and writing metrics. +endef + +define Package/telegraf/conffiles +/etc/config/telegraf +endef + +define Package/telegraf/install + $(call GoPackage/Package/Install/Bin,$(1)) + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/telegraf.initd $(1)/etc/init.d/telegraf + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_CONF) ./files/telegraf.uci $(1)/etc/config/telegraf + $(INSTALL_DIR) $(1)/etc/telegraf + $(INSTALL_DATA) ./files/telegraf.conf $(1)/etc/telegraf.conf + $(INSTALL_DIR) $(1)/etc/telegraf.conf.d + $(INSTALL_DATA) ./files/telegraf.conf.d/os.conf $(1)/etc/telegraf.conf.d/os.conf + $(INSTALL_DIR) $(1)/usr/sbin + $(INSTALL_BIN) ./files/telegraf-config $(1)/usr/sbin/telegraf-config +endef + +define Package/telegraf/postinst +#!/bin/sh +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/telegraf restart +exit 0 +endef + +$(eval $(call GoBinPackage,telegraf)) +$(eval $(call BuildPackage,telegraf)) diff --git a/packages/telegraf/files/telegraf-config b/packages/telegraf/files/telegraf-config new file mode 100644 index 000000000..235476d86 --- /dev/null +++ b/packages/telegraf/files/telegraf-config @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +import os +import subprocess +import glob +from jinja2 import Environment, BaseLoader +from euci import EUci + +PROMETHEUS_TEMPLATE = """# This file is automatically generated by /usr/sbin/telegraf-config. +# Do not edit manually — changes will be overwritten. + +# Prometheus client output +{% if enabled == '1' -%} +[[outputs.prometheus_client]] + ## Address to listen on. + ## ex: + ## listen = ":9273" + ## listen = "vsock://:9273" + listen = "{{ listen_addr }}" + + ## Maximum duration before timing out read of the request + # read_timeout = "10s" + ## Maximum duration before timing out write of the response + # write_timeout = "10s" + + ## Metric version controls the mapping from Prometheus metrics into Telegraf metrics. + ## See "Metric Format Configuration" in plugins/inputs/prometheus/README.md for details. + ## Valid options: 1, 2 + # metric_version = 1 + + ## Use HTTP Basic Authentication. +{%- if basic_auth_username and basic_auth_password %} + basic_username = "{{ basic_auth_username }}" + basic_password = "{{ basic_auth_password }}" +{%- else %} + # basic_username = "Foo" + # basic_password = "Bar" +{%- endif %} + + ## If set, the IP Ranges which are allowed to access metrics. + ## ex: ip_range = ["192.168.0.0/24", "192.168.1.0/30"] + # ip_range = [] + + ## Path to publish the metrics on. + # path = "/metrics" + + ## Expiration interval for each metric. 0 == no expiration + # expiration_interval = "60s" + + ## Collectors to enable, valid entries are "gocollector" and "process". + ## If unset, both are enabled. + # collectors_exclude = ["gocollector", "process"] + + ## Send string metrics as Prometheus labels. + ## Unless set to false all string metrics will be sent as labels. + # string_as_label = true + + ## If set, enable TLS with the given certificate. + # tls_cert = "/etc/ssl/telegraf.crt" + # tls_key = "/etc/ssl/telegraf.key" + + ## Set one or more allowed client CA certificate file names to + ## enable mutually authenticated TLS connections + # tls_allowed_cacerts = ["/etc/telegraf/clientca.pem"] + + ## Export metric collection time. + # export_timestamp = false + + ## Specify the metric type explicitly. + ## This overrides the metric-type of the Telegraf metric. Globbing is allowed. + # [outputs.prometheus_client.metric_types] + # counter = [] + # gauge = [] +{%- endif %} +""" + +SENSORS_TEMPLATE = """# This file is automatically generated by /usr/sbin/telegraf-config. +# Do not edit manually — changes will be overwritten. + +# Monitor sensors, requires lm-sensors package +# This plugin ONLY supports Linux +{% if sensors_available -%} +[[inputs.sensors]] + ## Remove numbers from field names. + ## If true, a field name like 'temp1_input' will be changed to 'temp_input'. + # remove_numbers = true + + ## Timeout is the maximum amount of time that the sensors command can run. + # timeout = "5s" + [inputs.sensors.tags] + influxdb_db = "os-metrics" +{%- endif %} +""" + + +def generate_prometheus_config(): + """Read UCI config and render Prometheus client output section.""" + e_uci = EUci() + + try: + enabled = e_uci.get('telegraf', 'output_prometheus', 'enabled', dtype=str, default='0') + listen_addr = e_uci.get('telegraf', 'output_prometheus', 'listen_addr', dtype=str, default=':9273') + basic_auth_username = e_uci.get('telegraf', 'output_prometheus', 'basic_auth_username', dtype=str, default='') + basic_auth_password = e_uci.get('telegraf', 'output_prometheus', 'basic_auth_password', dtype=str, default='') + except Exception as e: + print(f"Error reading UCI config: {e}") + return False + + # Render template + template = Environment(loader=BaseLoader()).from_string(PROMETHEUS_TEMPLATE) + rendered = template.render( + enabled=enabled, + listen_addr=listen_addr, + basic_auth_username=basic_auth_username, + basic_auth_password=basic_auth_password + ) + + # Write to drop-in directory + config_file = '/etc/telegraf.conf.d/prometheus.conf' + os.makedirs(os.path.dirname(config_file), exist_ok=True) + + try: + with open(config_file, 'w') as f: + f.write(rendered) + return True + except Exception as e: + print(f"Error writing config file: {e}") + return False + + +def sensors_available(): + """Check if sensors command works by running it with -A -u flags.""" + try: + result = subprocess.run(['sensors', '-A', '-u'], capture_output=True, timeout=5) + return result.returncode == 0 + except (subprocess.TimeoutExpired, FileNotFoundError): + return False + + +def generate_sensors_config(): + """Render sensors input section based on system availability.""" + has_sensors = sensors_available() + + # Render template + template = Environment(loader=BaseLoader()).from_string(SENSORS_TEMPLATE) + rendered = template.render(sensors_available=has_sensors) + + # Write to drop-in directory + config_file = '/etc/telegraf.conf.d/sensors.conf' + os.makedirs(os.path.dirname(config_file), exist_ok=True) + + try: + with open(config_file, 'w') as f: + f.write(rendered) + return True + except Exception as e: + print(f"Error writing config file: {e}") + return False + + +if __name__ == '__main__': + generate_prometheus_config() + generate_sensors_config() + exit(0) diff --git a/packages/telegraf/files/telegraf.conf b/packages/telegraf/files/telegraf.conf new file mode 100644 index 000000000..90f4b4061 --- /dev/null +++ b/packages/telegraf/files/telegraf.conf @@ -0,0 +1,24 @@ +# Telegraf Configuration +# Managed by NethSecurity — do not edit manually. + +[global_tags] + +[agent] + interval = "10s" + round_interval = true + metric_batch_size = 1000 + metric_buffer_limit = 10000 + collection_jitter = "0s" + flush_interval = "10s" + flush_jitter = "0s" + precision = "0s" + omit_hostname = true + skip_processors_after_aggregators = true + +# Victoria metrics output plugin configuration +[[outputs.influxdb]] + urls = ["http://127.0.0.1:8428"] + database = "nethsecurity" + database_tag = "influxdb_db" + exclude_database_tag = true + content_encoding = "gzip" diff --git a/packages/telegraf/files/telegraf.conf.d/os.conf b/packages/telegraf/files/telegraf.conf.d/os.conf new file mode 100644 index 000000000..04f84000e --- /dev/null +++ b/packages/telegraf/files/telegraf.conf.d/os.conf @@ -0,0 +1,60 @@ +# OS and system metrics collection +# Includes CPU, memory, disk, network, and kernel statistics +# All metrics from this section are tagged with influxdb_db=os-metrics + +[[inputs.cpu]] + percpu = true + totalcpu = true + collect_cpu_time = false + report_active = false + core_tags = false + [inputs.cpu.tags] + influxdb_db = "os-metrics" + +[[inputs.disk]] + ignore_fs = ["tmpfs", "devtmpfs", "devfs", "iso9660", "overlay", "aufs", "squashfs"] + [inputs.disk.tags] + influxdb_db = "os-metrics" + +[[inputs.mem]] + [inputs.mem.tags] + influxdb_db = "os-metrics" + +[[inputs.processes]] + [inputs.processes.tags] + influxdb_db = "os-metrics" + +[[inputs.system]] + [inputs.system.tags] + influxdb_db = "os-metrics" + +[[inputs.bond]] + [inputs.bond.tags] + influxdb_db = "os-metrics" + +[[inputs.ethtool]] + ## List of interfaces to include (default: all up interfaces) + # interface_include = ["eth0"] + ## List of interfaces to ignore (default: none) + interface_exclude = ["wg*", "ipsec*", "tun*", "br*"] + + [inputs.ethtool.tags] + influxdb_db = "os-metrics" + +[[inputs.net]] + ## Skip protocol stats, use inputs.nstat instead (fixes deprecation warning) + ignore_protocol_stats = true + [inputs.net.tags] + influxdb_db = "os-metrics" + +[[inputs.netstat]] + [inputs.netstat.tags] + influxdb_db = "os-metrics" + +[[inputs.nstat]] + proc_net_netstat = "/proc/net/netstat" + proc_net_snmp = "/proc/net/snmp" + proc_net_snmp6 = "/proc/net/snmp6" + dump_zeros = true + [inputs.nstat.tags] + influxdb_db = "os-metrics" diff --git a/packages/telegraf/files/telegraf.initd b/packages/telegraf/files/telegraf.initd new file mode 100644 index 000000000..49197475e --- /dev/null +++ b/packages/telegraf/files/telegraf.initd @@ -0,0 +1,36 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# shellcheck disable=SC3043 + +START=99 +USE_PROCD=1 + +PROG="/usr/bin/telegraf" + +start_service() { + /usr/sbin/telegraf-config + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 0 + procd_set_param command $PROG + procd_append_param command --watch-config notify + procd_append_param command --config /etc/telegraf.conf + procd_append_param command --config-directory /etc/telegraf.conf.d + procd_close_instance +} + +reload_service() +{ + /usr/sbin/telegraf-config +} + +service_triggers() +{ + procd_add_reload_trigger telegraf +} diff --git a/packages/telegraf/files/telegraf.uci b/packages/telegraf/files/telegraf.uci new file mode 100644 index 000000000..d1225d452 --- /dev/null +++ b/packages/telegraf/files/telegraf.uci @@ -0,0 +1,5 @@ +config output_prometheus 'output_prometheus' + option enabled '0' + option listen_addr ':9273' + option basic_auth_username '' + option basic_auth_password '' diff --git a/packages/victoria-logs/Makefile b/packages/victoria-logs/Makefile new file mode 100644 index 000000000..813fe68a3 --- /dev/null +++ b/packages/victoria-logs/Makefile @@ -0,0 +1,76 @@ +include $(TOPDIR)/rules.mk + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +PKG_NAME:=victoria-logs +# renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaLogs +PKG_VERSION:=1.49.0 +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/VictoriaMetrics/VictoriaLogs/tar.gz/v$(PKG_VERSION)? +PKG_SOURCE_SUBDIR:=VictoriaLogs-$(PKG_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) + +PKG_HASH:=skip +PKG_MAINTAINER:=Tommaso Bailetti +PKG_LICENSE:=Apache-2.0 + +PKG_BUILD_DEPENDS:=golang/host +PKG_BUILD_PARALLEL:=1 +PKG_BUILD_FLAGS:=no-mips16 + +GO_PKG:=github.com/VictoriaMetrics/VictoriaLogs +GO_PKG_BUILD_PKG:=github.com/VictoriaMetrics/VictoriaLogs/app/victoria-logs \ + github.com/VictoriaMetrics/VictoriaLogs/app/vlogscli +GO_PKG_LDFLAGS:= \ + -extldflags \ + -static +GO_PKG_LDFLAGS_X:=github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo.Version=$(PKG_NAME)-v$(PKG_VERSION) +GO_PKG_TAGS:= \ + netgo \ + osusergo \ + musl + +include $(INCLUDE_DIR)/package.mk +include $(TOPDIR)/feeds/packages/lang/golang/golang-package.mk + +define Package/victoria-logs + SECTION:=base + CATEGORY:=NethServer + TITLE:=Victoria Logs + URL:=https://github.com/VictoriaMetrics/VictoriaLogs + DEPENDS:=$(GO_ARCH_DEPENDS) +rsyslog +endef + +define Package/victoria-logs/description + VictoriaLogs — fast and easy-to-use database for logs. +endef + +define Package/victoria-logs/conffiles +/etc/config/victoria-logs +endef + +define Package/victoria-logs/install + $(call GoPackage/Package/Install/Bin,$(1)) + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/victoria-logs.initd $(1)/etc/init.d/victoria-logs + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_DATA) ./files/victoria-logs.conf $(1)/etc/config/victoria-logs + $(INSTALL_DIR) $(1)/etc/rsyslog.d + $(INSTALL_CONF) ./files/rsyslog-victoria-logs.conf $(1)/etc/rsyslog.d/victoria-logs.conf + $(INSTALL_DIR) $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/35_victoria-logs $(1)/etc/uci-defaults/35_victoria-logs +endef + +define Package/victoria-logs/postinst +#!/bin/sh +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/victoria-logs restart +exit 0 +endef + +$(eval $(call GoPackage,victoria-logs)) +$(eval $(call BuildPackage,victoria-logs)) diff --git a/packages/victoria-logs/files/35_victoria-logs b/packages/victoria-logs/files/35_victoria-logs new file mode 100644 index 000000000..079e1a4eb --- /dev/null +++ b/packages/victoria-logs/files/35_victoria-logs @@ -0,0 +1,14 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +RSYSLOG_CONF="/etc/rsyslog.d/victoria-logs.conf" + +# Register the victoria-logs rsyslog drop-in in rsyslog UCI includes +if ! uci -q get rsyslog.syslog.includes | grep -qF "${RSYSLOG_CONF}"; then + uci add_list rsyslog.syslog.includes="${RSYSLOG_CONF}" + uci commit rsyslog +fi diff --git a/packages/victoria-logs/files/rsyslog-victoria-logs.conf b/packages/victoria-logs/files/rsyslog-victoria-logs.conf new file mode 100644 index 000000000..c2a66de3b --- /dev/null +++ b/packages/victoria-logs/files/rsyslog-victoria-logs.conf @@ -0,0 +1,18 @@ +# Rsyslog configuration for VictoriaLogs + +ruleset(name="victoria-logs") { + *.* action( + type="omfwd" + target="127.0.0.1" + port="5514" + protocol="tcp" + TCP_Framing="octet-counted" + Template="RSYSLOG_SyslogProtocol23Format" + + action.resumeRetryCount="-1" + queue.type="linkedList" + queue.size="10000" + ) +} + +*.* call victoria-logs diff --git a/packages/victoria-logs/files/victoria-logs.conf b/packages/victoria-logs/files/victoria-logs.conf new file mode 100644 index 000000000..84aba8dbe --- /dev/null +++ b/packages/victoria-logs/files/victoria-logs.conf @@ -0,0 +1,4 @@ +config victorialogs 'main' + option storage_path '/var/lib/victoria-logs' + option max_disk_usage '50MB' + option http_listen_addr '127.0.0.1:9428' diff --git a/packages/victoria-logs/files/victoria-logs.initd b/packages/victoria-logs/files/victoria-logs.initd new file mode 100644 index 000000000..820ae7877 --- /dev/null +++ b/packages/victoria-logs/files/victoria-logs.initd @@ -0,0 +1,43 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# shellcheck disable=SC3043 + +START=99 +USE_PROCD=1 + +PROG="/usr/bin/victoria-logs" + +start_service() { + config_load victoria-logs + local storage_path max_disk_usage http_listen_addr + config_get storage_path main storage_path /var/lib/victoria-logs + config_get max_disk_usage main max_disk_usage 50MB + config_get http_listen_addr main http_listen_addr 127.0.0.1:9428 + + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 0 + procd_set_param command $PROG + procd_append_param command -storageDataPath="$storage_path" + procd_append_param command -retention.maxDiskSpaceUsageBytes="$max_disk_usage" + procd_append_param command -httpListenAddr="$http_listen_addr" + procd_append_param command -syslog.listenAddr.tcp=127.0.0.1:5514 + procd_close_instance +} + +service_triggers() +{ + procd_add_reload_trigger victoria-logs +} + +reload_service() +{ + stop + start +} diff --git a/packages/victoria-metrics/Makefile b/packages/victoria-metrics/Makefile new file mode 100644 index 000000000..5fe85f5a8 --- /dev/null +++ b/packages/victoria-metrics/Makefile @@ -0,0 +1,72 @@ +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +include $(TOPDIR)/rules.mk + +PKG_NAME:=victoria-metrics +# renovate: datasource=github-tags depName=VictoriaMetrics/VictoriaMetrics +PKG_VERSION:=1.139.0 +PKG_RELEASE:=1 + +PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz +PKG_SOURCE_URL:=https://codeload.github.com/VictoriaMetrics/VictoriaMetrics/tar.gz/v$(PKG_VERSION)? +PKG_SOURCE_SUBDIR:=VictoriaMetrics-$(PKG_VERSION) +PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) + +PKG_HASH:=skip +PKG_MAINTAINER:=Tommaso Bailetti +PKG_LICENSE:=Apache-2.0 + +PKG_BUILD_DEPENDS:=golang/host +PKG_BUILD_PARALLEL:=1 +PKG_BUILD_FLAGS:=no-mips16 + +GO_PKG:=github.com/VictoriaMetrics/VictoriaMetrics +GO_PKG_BUILD_PKG:=github.com/VictoriaMetrics/VictoriaMetrics/app/victoria-metrics \ + github.com/VictoriaMetrics/VictoriaMetrics/app/vmalert +GO_PKG_LDFLAGS:= \ + -extldflags \ + -static +GO_PKG_LDFLAGS_X:=github.com/VictoriaMetrics/VictoriaMetrics/lib/buildinfo.Version=$(PKG_NAME)-v$(PKG_VERSION) +GO_PKG_TAGS:= \ + netgo \ + osusergo \ + musl + +include $(INCLUDE_DIR)/package.mk +include $(TOPDIR)/feeds/packages/lang/golang/golang-package.mk + +define Package/victoria-metrics + SECTION:=base + CATEGORY:=NethServer + TITLE:=Victoria Metrics + URL:=https://github.com/VictoriaMetrics/VictoriaMetrics + DEPENDS:=$(GO_ARCH_DEPENDS) +endef + +define Package/victoria-metrics/description + VictoriaMetrics time series database / single-node server. +endef + +define Package/victoria-metrics/conffiles +/etc/config/victoria-metrics +endef + +define Package/victoria-metrics/install + $(call GoPackage/Package/Install/Bin,$(1)) + $(INSTALL_DIR) $(1)/etc/init.d + $(INSTALL_BIN) ./files/victoria-metrics.initd $(1)/etc/init.d/victoria-metrics + $(INSTALL_DIR) $(1)/etc/config + $(INSTALL_DATA) ./files/victoria-metrics.conf $(1)/etc/config/victoria-metrics +endef + +define Package/victoria-metrics/postinst +#!/bin/sh +[ -z "$${IPKG_INSTROOT}" ] && /etc/init.d/victoria-metrics restart +exit 0 +endef + +$(eval $(call GoBinPackage,victoria-metrics)) +$(eval $(call BuildPackage,victoria-metrics)) \ No newline at end of file diff --git a/packages/victoria-metrics/files/victoria-metrics.conf b/packages/victoria-metrics/files/victoria-metrics.conf new file mode 100644 index 000000000..5b6813a89 --- /dev/null +++ b/packages/victoria-metrics/files/victoria-metrics.conf @@ -0,0 +1,4 @@ +config victoriametrics 'main' + option storage_path '/var/lib/victoriametrics' + option retention_period '1d' + option http_listen_addr '127.0.0.1:8428' diff --git a/packages/victoria-metrics/files/victoria-metrics.initd b/packages/victoria-metrics/files/victoria-metrics.initd new file mode 100644 index 000000000..fd2cee851 --- /dev/null +++ b/packages/victoria-metrics/files/victoria-metrics.initd @@ -0,0 +1,42 @@ +#!/bin/sh /etc/rc.common + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# shellcheck disable=SC3043 + +START=99 +USE_PROCD=1 + +PROG="/usr/bin/victoria-metrics" + +start_service() { + config_load victoria-metrics + local storage_path retention_period http_listen_addr + config_get storage_path main storage_path /var/lib/victoriametrics + config_get retention_period main retention_period 1d + config_get http_listen_addr main http_listen_addr 127.0.0.1:8428 + + procd_open_instance + procd_set_param stdout 1 + procd_set_param stderr 1 + procd_set_param respawn 3600 5 0 + procd_set_param command $PROG + procd_append_param command -storageDataPath="$storage_path" + procd_append_param command -retentionPeriod="$retention_period" + procd_append_param command -httpListenAddr="$http_listen_addr" + procd_close_instance +} + +service_triggers() +{ + procd_add_reload_trigger victoria-metrics +} + +reload_service() +{ + stop + start +} diff --git a/patches/package/100-pppd-edit-bcopy.patch b/patches/package/100-pppd-edit-bcopy.patch deleted file mode 100644 index 2874da9fa..000000000 --- a/patches/package/100-pppd-edit-bcopy.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git a/network/services/ppp/patches/200-use-memmove-for-bcopy.patch b/network/services/ppp/patches/200-use-memmove-for-bcopy.patch -new file mode 100644 -index 0000000000..1632eb27b8 ---- /dev/null -+++ b/package/network/services/ppp/patches/200-use-memmove-for-bcopy.patch -@@ -0,0 +1,11 @@ -+--- a/pppd/pppd-private.h -++++ b/pppd/pppd-private.h -+@@ -523,7 +523,7 @@ int parse_dotted_ip(char *, u_int32_t *) -+ #define TIMEOUT(r, f, t) ppp_timeout((r), (f), (t), 0) -+ #define UNTIMEOUT(r, f) ppp_untimeout((r), (f)) -+ -+-#define BCOPY(s, d, l) memcpy(d, s, l) -++#define BCOPY(s, d, l) memmove(d, s, l) -+ #define BZERO(s, n) memset(s, 0, n) -+ #define BCMP(s1, s2, l) memcmp(s1, s2, l) -+ diff --git a/scripts/Readme.md b/scripts/Readme.md index f7b16b6cb..32fa85a37 100644 --- a/scripts/Readme.md +++ b/scripts/Readme.md @@ -99,7 +99,14 @@ This script retrieves open issues labeled "testing" from the NethServer/nethsecu ## netifyd-packages.sh -This scripts pulls the `.ipk` packages from the `netifyd-ipks` directory, unpacks them and puts the files inside the `netifyd` package. This script is useful when an update of `netifyd` requires changes in the integration meta package. +This script extracts Netify `.apk` packages from the `netifyd-apks` directory, unpacks them, and merges the files for analysis and integration. This script is useful when an update of `netifyd` requires changes in the integration meta package. + +### Prerequisites + +- **apk-tools**: The Alpine package extraction tool must be installed on your system. + - On Fedora/RHEL: `dnf install apk-tools` + - On Debian/Ubuntu: `apt-get install apk-tools` + - On Alpine Linux: Already included ### Usage @@ -107,4 +114,12 @@ This scripts pulls the `.ipk` packages from the `netifyd-ipks` directory, unpack ./netifyd-packages.sh ``` -All process will be handled by the script, there will be an output that lists the files that were being copied in the process, this output needs to be copied and pasted inside the `packages/netifyd/Makefile` file, inside the `define Package/netifyd/install` section. +The script will process all `.apk` files in the `netifyd-apks/{arch}/` directories and extract their contents into `netifyd-apks/tmp/{arch}/netifyd/`. The merged files can then be copied into the `packages/netifyd/` directory as needed. + +### How It Works + +1. Iterates over each architecture subdirectory in `netifyd-apks/` +2. For each `.apk` file found, extracts its contents to a temporary location +3. Merges all extracted files into a single `netifyd` output directory per architecture +4. Skips metadata files (`.PKGINFO`, install scripts, etc.) during extraction +5. Outputs the merged file structure in `netifyd-apks/tmp/{arch}/netifyd/` diff --git a/scripts/netifyd-packages.sh b/scripts/netifyd-packages.sh index 8425d380c..d3261af1c 100755 --- a/scripts/netifyd-packages.sh +++ b/scripts/netifyd-packages.sh @@ -1,28 +1,32 @@ #!/bin/bash -# This is a helper script to extract all Netify IPK packages -# into netifyd-ipks/tmp/{arch} directories for analysis and integration. +# This is a helper script to extract all Netify APK packages +# into netifyd-apks/tmp/{arch} directories for analysis and integration. # # Usage: ./netifyd-packages.sh # -# Input: netifyd-ipks/{arch}/*.ipk -# Output: netifyd-ipks/tmp/{arch}/netifyd/ — merged contents of all IPKs for that arch -# netifyd-ipks/tmp/{arch}/{pkg}/ — per-package extraction (intermediate) +# Input: netifyd-apks/{arch}/*.apk +# Output: netifyd-apks/tmp/{arch}/netifyd/ — merged contents of all APKs for that arch +# netifyd-apks/tmp/{arch}/{pkg}/ — per-package extraction (intermediate) set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -IPKS_DIR="${SCRIPT_DIR}/netifyd-ipks" -TMP_DIR="${IPKS_DIR}/tmp" +APKS_DIR="${SCRIPT_DIR}/netifyd-apks" +TMP_DIR="${APKS_DIR}/tmp" -if [ ! -d "${IPKS_DIR}" ]; then - echo "ERROR: IPKs directory not found: ${IPKS_DIR}" >&2 +if [ ! -d "${APKS_DIR}" ]; then + echo "ERROR: APKs directory not found: ${APKS_DIR}" >&2 exit 1 fi # Iterate over each arch subdirectory -for arch_dir in "${IPKS_DIR}"/*/; do +for arch_dir in "${APKS_DIR}"/*/; do [ -d "${arch_dir}" ] || continue + + # Skip the tmp directory + [ "$(basename "${arch_dir}")" = "tmp" ] && continue + arch="$(basename "${arch_dir}")" echo "==> Processing arch: ${arch}" @@ -31,13 +35,13 @@ for arch_dir in "${IPKS_DIR}"/*/; do rm -rf "${TMP_DIR:?}/${arch}" mkdir -p "${output_dir}" - # Extract each IPK into its own per-package subdirectory, then merge - for ipk_file in "${arch_dir}"*.ipk; do - [ -f "${ipk_file}" ] || continue + # Extract each APK into its own per-package subdirectory, then merge + for apk_file in "${arch_dir}"*.apk; do + [ -f "${apk_file}" ] || continue # Derive package name: strip version and arch suffix - # e.g. netify-plm_2026-01-01-v1.2.1-r8_x86_64.ipk -> netify-plm - filename="$(basename "${ipk_file}" .ipk)" + # e.g. netify-plm_2026-01-01-v1.2.1-r8_x86_64.apk -> netify-plm + filename="$(basename "${apk_file}" .apk)" pkg_name="${filename%%_*}" # Extract to a temporary directory first to avoid conflicts @@ -45,7 +49,7 @@ for arch_dir in "${IPKS_DIR}"/*/; do mkdir -p "${pkg_extract_dir}" echo " Extracting ${filename} -> ${pkg_name}/" - tar -xf "${ipk_file}" ./data.tar.gz -O | tar -xzf - -C "${pkg_extract_dir}" + apk extract --allow-untrusted --destination "${pkg_extract_dir}" "${apk_file}" # Merge extracted files into the single netifyd output directory cp -a "${pkg_extract_dir}/." "${output_dir}/" diff --git a/tools/cleanup/cleanup.py b/tools/cleanup/cleanup.py index 4c413e494..caa163bf4 100755 --- a/tools/cleanup/cleanup.py +++ b/tools/cleanup/cleanup.py @@ -2,13 +2,13 @@ # # Cleanup old development builds from DigitalOcean Spaces: -# - Keep all tagged releases -# - Keep at least 3 versions of each sub release +# - Keep the latest 5 versions of each channel # +# Version format: 8.7.2-dev.. or 8.7.2-branch.. import os import boto3 -from semver import VersionInfo +from semver import Version as VersionInfo region = "ams3" bucket_name = "nethsecurity" @@ -27,20 +27,35 @@ parsed_files = [] for file in files: file_name = file.lstrip(f'{prefix}/').rstrip('/') - version_parsed = VersionInfo.parse(file_name) - if version_parsed.build is None: + try: + version_parsed = VersionInfo.parse(file_name) + except ValueError: + print(f'Skipping {file_name} - not a valid semver version.') + continue + + # Check if it's a development build (has prerelease segment) + if version_parsed.prerelease is None: print(f'Skipping {file_name} as it is not a development build.') continue - build_split = version_parsed.build.split('.') + + # Extract timestamp from prerelease segment: "dev.." or "branch.." + prerelease_parts = version_parsed.prerelease.split('.') + if len(prerelease_parts) < 3: + print(f'Skipping {file_name} - prerelease segment does not contain timestamp.') + continue + + timestamp = prerelease_parts[1] # middle part is the timestamp parsed_files.append({ - 'timestamp': build_split[1], - 'file': file + 'timestamp': timestamp, + 'file': file, + 'version': file_name }) -# keep only the latest 5 dev builds +# Keep only the latest 5 dev builds, sorted by timestamp (descending) to_delete = sorted(parsed_files, key=lambda k: k['timestamp'], reverse=True)[min_versions:] for d in to_delete: - print(f"Deleting {d['file']} ...") + print(f"Deleting {d['version']} ...") objects_to_delete = s3_client.list_objects(Bucket=bucket_name, Prefix=d['file']) delete_keys = {'Objects': [{'Key': k['Key']} for k in objects_to_delete.get('Contents', [])]} s3_client.delete_objects(Bucket=bucket_name, Delete=delete_keys) +